diff --git a/AUTHORS b/AUTHORS index 2431fe61e14c..fd965302bba4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -600,6 +600,7 @@ answer newbie questions, and generally made Django that much better: mattycakes@gmail.com Max Burstein Max Derkachev + Max Smolens Maxime Lorant Maxime Turcotte Maximilian Merz diff --git a/django/__init__.py b/django/__init__.py index e2082dfde8bc..c245e9b41023 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,6 +1,6 @@ from django.utils.version import get_version -VERSION = (3, 0, 7, 'final', 0) +VERSION = (3, 0, 14, 'final', 0) __version__ = get_version(VERSION) diff --git a/django/contrib/gis/db/backends/oracle/introspection.py b/django/contrib/gis/db/backends/oracle/introspection.py index 5e71c84fdd07..80899b33d7d9 100644 --- a/django/contrib/gis/db/backends/oracle/introspection.py +++ b/django/contrib/gis/db/backends/oracle/introspection.py @@ -1,14 +1,19 @@ import cx_Oracle from django.db.backends.oracle.introspection import DatabaseIntrospection +from django.utils.functional import cached_property class OracleIntrospection(DatabaseIntrospection): # Associating any OBJECTVAR instances with GeometryField. Of course, # this won't work right on Oracle objects that aren't MDSYS.SDO_GEOMETRY, # but it is the only object type supported within Django anyways. - data_types_reverse = DatabaseIntrospection.data_types_reverse.copy() - data_types_reverse[cx_Oracle.OBJECT] = 'GeometryField' + @cached_property + def data_types_reverse(self): + return { + **super().data_types_reverse, + cx_Oracle.OBJECT: 'GeometryField', + } def get_geometry_type(self, table_name, description): with self.connection.cursor() as cursor: diff --git a/django/contrib/messages/storage/cookie.py b/django/contrib/messages/storage/cookie.py index 9e0c93e436f3..057d573d3fa8 100644 --- a/django/contrib/messages/storage/cookie.py +++ b/django/contrib/messages/storage/cookie.py @@ -89,7 +89,11 @@ def _update_cookie(self, encoded_data, response): samesite=settings.SESSION_COOKIE_SAMESITE, ) else: - response.delete_cookie(self.cookie_name, domain=settings.SESSION_COOKIE_DOMAIN) + response.delete_cookie( + self.cookie_name, + domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) def _store(self, messages, response, remove_oldest=True, *args, **kwargs): """ diff --git a/django/contrib/sessions/middleware.py b/django/contrib/sessions/middleware.py index d36be4eca8bd..7a0f9e030b2f 100644 --- a/django/contrib/sessions/middleware.py +++ b/django/contrib/sessions/middleware.py @@ -38,6 +38,7 @@ def process_response(self, request, response): settings.SESSION_COOKIE_NAME, path=settings.SESSION_COOKIE_PATH, domain=settings.SESSION_COOKIE_DOMAIN, + samesite=settings.SESSION_COOKIE_SAMESITE, ) patch_vary_headers(response, ('Cookie',)) else: diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index 86a7aca5757a..875689bfaaad 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -286,6 +286,6 @@ def memcache_key_warnings(key): if ord(char) < 33 or ord(char) == 127: yield ( 'Cache key contains characters that will cause errors if ' - 'used with memcached: %r' % key, CacheKeyWarning + 'used with memcached: %r' % key ) break diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index 48b8df90abd9..64cc175b1835 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -113,7 +113,13 @@ def _cull(self): self._delete(fname) def _createdir(self): - os.makedirs(self._dir, 0o700, exist_ok=True) + # Set the umask because os.makedirs() doesn't apply the "mode" argument + # to intermediate-level directories. + old_umask = os.umask(0o077) + try: + os.makedirs(self._dir, 0o700, exist_ok=True) + finally: + os.umask(old_umask) def _key_to_file(self, key, version=None): """ diff --git a/django/core/files/storage.py b/django/core/files/storage.py index 4c27fce6052e..af70b3fa1584 100644 --- a/django/core/files/storage.py +++ b/django/core/files/storage.py @@ -237,9 +237,9 @@ def _save(self, name, content): directory = os.path.dirname(full_path) try: if self.directory_permissions_mode is not None: - # os.makedirs applies the global umask, so we reset it, - # for consistency with file_permissions_mode behavior. - old_umask = os.umask(0) + # Set the umask because os.makedirs() doesn't apply the "mode" + # argument to intermediate-level directories. + old_umask = os.umask(0o777 & ~self.directory_permissions_mode) try: os.makedirs(directory, self.directory_permissions_mode, exist_ok=True) finally: diff --git a/django/core/mail/message.py b/django/core/mail/message.py index 607eb4af0b85..963542cd6239 100644 --- a/django/core/mail/message.py +++ b/django/core/mail/message.py @@ -10,7 +10,7 @@ from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.utils import formatdate, getaddresses, make_msgid +from email.utils import formataddr, formatdate, getaddresses, make_msgid from io import BytesIO, StringIO from pathlib import Path @@ -96,16 +96,24 @@ def sanitize_address(addr, encoding): nm, address = addr localpart, domain = address.rsplit('@', 1) - nm = Header(nm, encoding).encode() + address_parts = nm + localpart + domain + if '\n' in address_parts or '\r' in address_parts: + raise ValueError('Invalid address; address parts cannot contain newlines.') + # Avoid UTF-8 encode, if it's possible. + try: + nm.encode('ascii') + nm = Header(nm).encode() + except UnicodeEncodeError: + nm = Header(nm, encoding).encode() try: localpart.encode('ascii') except UnicodeEncodeError: localpart = Header(localpart, encoding).encode() domain = punycode(domain) - parsed_address = Address(nm, username=localpart, domain=domain) - return str(parsed_address) + parsed_address = Address(username=localpart, domain=domain) + return formataddr((nm, parsed_address.addr_spec)) class MIMEMixin: diff --git a/django/db/backends/base/schema.py b/django/db/backends/base/schema.py index 98afbcc05a8b..5d2ee4eb04eb 100644 --- a/django/db/backends/base/schema.py +++ b/django/db/backends/base/schema.py @@ -461,8 +461,10 @@ def add_field(self, model, field): if self.sql_create_column_inline_fk: to_table = field.remote_field.model._meta.db_table to_column = field.remote_field.model._meta.get_field(field.remote_field.field_name).column + namespace, _ = split_identifier(model._meta.db_table) definition += " " + self.sql_create_column_inline_fk % { 'name': self._fk_constraint_name(model, field, constraint_suffix), + 'namespace': '%s.' % self.quote_name(namespace) if namespace else '', 'column': self.quote_name(field.column), 'to_table': self.quote_name(to_table), 'to_column': self.quote_name(to_column), diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 86650a894ea5..ec993bbcfd49 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -310,6 +310,10 @@ def is_usable(self): else: return True + @cached_property + def cx_oracle_version(self): + return tuple(int(x) for x in Database.version.split('.')) + @cached_property def oracle_version(self): with self.temporary_connection(): diff --git a/django/db/backends/oracle/introspection.py b/django/db/backends/oracle/introspection.py index 2322ae0b5d6a..1ad0c9e8d7ad 100644 --- a/django/db/backends/oracle/introspection.py +++ b/django/db/backends/oracle/introspection.py @@ -6,29 +6,48 @@ from django.db.backends.base.introspection import ( BaseDatabaseIntrospection, FieldInfo as BaseFieldInfo, TableInfo, ) +from django.utils.functional import cached_property FieldInfo = namedtuple('FieldInfo', BaseFieldInfo._fields + ('is_autofield',)) class DatabaseIntrospection(BaseDatabaseIntrospection): - # Maps type objects to Django Field types. - data_types_reverse = { - cx_Oracle.BLOB: 'BinaryField', - cx_Oracle.CLOB: 'TextField', - cx_Oracle.DATETIME: 'DateField', - cx_Oracle.FIXED_CHAR: 'CharField', - cx_Oracle.FIXED_NCHAR: 'CharField', - cx_Oracle.INTERVAL: 'DurationField', - cx_Oracle.NATIVE_FLOAT: 'FloatField', - cx_Oracle.NCHAR: 'CharField', - cx_Oracle.NCLOB: 'TextField', - cx_Oracle.NUMBER: 'DecimalField', - cx_Oracle.STRING: 'CharField', - cx_Oracle.TIMESTAMP: 'DateTimeField', - } - cache_bust_counter = 1 + # Maps type objects to Django Field types. + @cached_property + def data_types_reverse(self): + if self.connection.cx_oracle_version < (8,): + return { + cx_Oracle.BLOB: 'BinaryField', + cx_Oracle.CLOB: 'TextField', + cx_Oracle.DATETIME: 'DateField', + cx_Oracle.FIXED_CHAR: 'CharField', + cx_Oracle.FIXED_NCHAR: 'CharField', + cx_Oracle.INTERVAL: 'DurationField', + cx_Oracle.NATIVE_FLOAT: 'FloatField', + cx_Oracle.NCHAR: 'CharField', + cx_Oracle.NCLOB: 'TextField', + cx_Oracle.NUMBER: 'DecimalField', + cx_Oracle.STRING: 'CharField', + cx_Oracle.TIMESTAMP: 'DateTimeField', + } + else: + return { + cx_Oracle.DB_TYPE_DATE: 'DateField', + cx_Oracle.DB_TYPE_BINARY_DOUBLE: 'FloatField', + cx_Oracle.DB_TYPE_BLOB: 'BinaryField', + cx_Oracle.DB_TYPE_CHAR: 'CharField', + cx_Oracle.DB_TYPE_CLOB: 'TextField', + cx_Oracle.DB_TYPE_INTERVAL_DS: 'DurationField', + cx_Oracle.DB_TYPE_NCHAR: 'CharField', + cx_Oracle.DB_TYPE_NCLOB: 'TextField', + cx_Oracle.DB_TYPE_NVARCHAR: 'CharField', + cx_Oracle.DB_TYPE_NUMBER: 'DecimalField', + cx_Oracle.DB_TYPE_TIMESTAMP: 'DateTimeField', + cx_Oracle.DB_TYPE_VARCHAR: 'CharField', + } + def get_field_type(self, data_type, description): if data_type == cx_Oracle.NUMBER: precision, scale = description[4:6] diff --git a/django/db/backends/postgresql/schema.py b/django/db/backends/postgresql/schema.py index 62e0bf95fddf..233eb10eb39f 100644 --- a/django/db/backends/postgresql/schema.py +++ b/django/db/backends/postgresql/schema.py @@ -23,7 +23,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): # transaction. sql_create_column_inline_fk = ( 'CONSTRAINT %(name)s REFERENCES %(to_table)s(%(to_column)s)%(deferrable)s' - '; SET CONSTRAINTS %(name)s IMMEDIATE' + '; SET CONSTRAINTS %(namespace)s%(name)s IMMEDIATE' ) # Setting the constraint to IMMEDIATE runs any deferred checks to allow # dropping it in the same transaction. diff --git a/django/db/models/base.py b/django/db/models/base.py index 63801da3c299..814e6ee28083 100644 --- a/django/db/models/base.py +++ b/django/db/models/base.py @@ -543,7 +543,10 @@ def __reduce__(self): def __getstate__(self): """Hook to allow choosing the attributes to pickle.""" - return self.__dict__ + state = self.__dict__.copy() + state['_state'] = copy.copy(state['_state']) + state['_state'].fields_cache = state['_state'].fields_cache.copy() + return state def __setstate__(self, state): msg = None diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 7b68e1108ead..29649313e5a6 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -380,7 +380,9 @@ def select_format(self, compiler, sql, params): Custom format for select clauses. For example, EXISTS expressions need to be wrapped in CASE WHEN on Oracle. """ - return self.output_field.select_format(compiler, sql, params) + if hasattr(self.output_field, 'select_format'): + return self.output_field.select_format(compiler, sql, params) + return sql, params @cached_property def identity(self): diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index ccec6fdc380c..18f8a5120b1b 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -973,7 +973,8 @@ def get_select_for_update_of_arguments(self): the query. """ def _get_parent_klass_info(klass_info): - for parent_model, parent_link in klass_info['model']._meta.parents.items(): + concrete_model = klass_info['model']._meta.concrete_model + for parent_model, parent_link in concrete_model._meta.parents.items(): parent_list = parent_model._meta.get_parent_list() yield { 'model': parent_model, @@ -998,8 +999,9 @@ def _get_first_selected_col_from_model(klass_info): select_fields is filled recursively, so it also contains fields from the parent models. """ + concrete_model = klass_info['model']._meta.concrete_model for select_index in klass_info['select_fields']: - if self.select[select_index][0].target.model == klass_info['model']: + if self.select[select_index][0].target.model == concrete_model: return self.select[select_index][0] def _get_field_choices(): diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index f550d5e28bbf..9504105807d6 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -1127,7 +1127,10 @@ def check_related_objects(self, field, value, opts): def check_filterable(self, expression): """Raise an error if expression cannot be used in a WHERE clause.""" - if not getattr(expression, 'filterable', 'True'): + if ( + hasattr(expression, 'resolve_expression') and + not getattr(expression, 'filterable', True) + ): raise NotSupportedError( expression.__class__.__name__ + ' is disallowed in the filter ' 'clause.' @@ -2131,8 +2134,10 @@ def set_values(self, fields): field_names.append(f) self.set_extra_mask(extra_names) self.set_annotation_mask(annotation_names) + selected = frozenset(field_names + extra_names + annotation_names) else: field_names = [f.attname for f in self.model._meta.concrete_fields] + selected = frozenset(field_names) # Selected annotations must be known before setting the GROUP BY # clause. if self.group_by is True: @@ -2146,7 +2151,7 @@ def set_values(self, fields): # the selected fields anymore. group_by = [] for expr in self.group_by: - if isinstance(expr, Ref) and expr.refs not in field_names: + if isinstance(expr, Ref) and expr.refs not in selected: expr = self.annotations[expr.refs] group_by.append(expr) self.group_by = tuple(group_by) diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index fd8fce8b4d46..db1b5ce8b956 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -9,6 +9,7 @@ import cgi import collections import html +import os from urllib.parse import unquote from django.conf import settings @@ -209,7 +210,7 @@ def parse(self): file_name = disposition.get('filename') if file_name: file_name = force_str(file_name, encoding, errors='replace') - file_name = self.IE_sanitize(html.unescape(file_name)) + file_name = self.sanitize_file_name(file_name) if not file_name: continue @@ -297,9 +298,13 @@ def handle_file_complete(self, old_field_name, counters): self._files.appendlist(force_str(old_field_name, self._encoding, errors='replace'), file_obj) break - def IE_sanitize(self, filename): - """Cleanup filename from Internet Explorer full paths.""" - return filename and filename[filename.rfind("\\") + 1:].strip() + def sanitize_file_name(self, file_name): + file_name = html.unescape(file_name) + # Cleanup Windows-style path separators. + file_name = file_name[file_name.rfind('\\') + 1:].strip() + return os.path.basename(file_name) + + IE_sanitize = sanitize_file_name def _close_files(self): # Free up all file handles. diff --git a/django/http/response.py b/django/http/response.py index c33feb97c43b..d98c040d9124 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -209,13 +209,13 @@ def set_signed_cookie(self, key, value, salt='', **kwargs): value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) - def delete_cookie(self, key, path='/', domain=None): + def delete_cookie(self, key, path='/', domain=None, samesite=None): # Most browsers ignore the Set-Cookie header if the cookie name starts # with __Host- or __Secure- and the cookie doesn't use the secure flag. secure = key.startswith(('__Secure-', '__Host-')) self.set_cookie( key, max_age=0, path=path, domain=domain, secure=secure, - expires='Thu, 01 Jan 1970 00:00:00 GMT', + expires='Thu, 01 Jan 1970 00:00:00 GMT', samesite=samesite, ) # Common methods used by subclasses diff --git a/django/utils/archive.py b/django/utils/archive.py index 235809f2ad0e..d5a0cf044657 100644 --- a/django/utils/archive.py +++ b/django/utils/archive.py @@ -27,6 +27,8 @@ import tarfile import zipfile +from django.core.exceptions import SuspiciousOperation + class ArchiveException(Exception): """ @@ -133,6 +135,13 @@ def has_leading_dir(self, paths): return False return True + def target_filename(self, to_path, name): + target_path = os.path.abspath(to_path) + filename = os.path.abspath(os.path.join(target_path, name)) + if not filename.startswith(target_path): + raise SuspiciousOperation("Archive contains invalid path: '%s'" % name) + return filename + def extract(self): raise NotImplementedError('subclasses of BaseArchive must provide an extract() method') @@ -155,7 +164,7 @@ def extract(self, to_path): name = member.name if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) + filename = self.target_filename(to_path, name) if member.isdir(): if filename: os.makedirs(filename, exist_ok=True) @@ -198,8 +207,10 @@ def extract(self, to_path): info = self._archive.getinfo(name) if leading: name = self.split_leading_dir(name)[1] - filename = os.path.join(to_path, name) - if filename.endswith(('/', '\\')): + if not name: + continue + filename = self.target_filename(to_path, name) + if name.endswith(('/', '\\')): # A directory os.makedirs(filename, exist_ok=True) else: diff --git a/django/utils/http.py b/django/utils/http.py index ff2f08ac1e84..c8f2c49a821a 100644 --- a/django/utils/http.py +++ b/django/utils/http.py @@ -41,7 +41,7 @@ RFC3986_GENDELIMS = ":/?#[]@" RFC3986_SUBDELIMS = "!$&'()*+,;=" -FIELDS_MATCH = re.compile('[&;]') +FIELDS_MATCH = re.compile('&') @keep_lazy_text diff --git a/docs/_ext/djangodocs.py b/docs/_ext/djangodocs.py index e9f5edc34078..1194c1ab5add 100644 --- a/docs/_ext/djangodocs.py +++ b/docs/_ext/djangodocs.py @@ -11,11 +11,10 @@ from sphinx import addnodes from sphinx.builders.html import StandaloneHTMLBuilder from sphinx.directives.code import CodeBlock -from sphinx.errors import SphinxError from sphinx.domains.std import Cmdoption from sphinx.errors import ExtensionError from sphinx.util import logging -from sphinx.util.console import bold, red +from sphinx.util.console import bold from sphinx.writers.html import HTMLTranslator logger = logging.getLogger(__name__) @@ -379,8 +378,9 @@ def default_role_error( name, rawtext, text, lineno, inliner, options=None, content=None ): msg = ( - "Default role used (`single backticks`) at line %s: %s. Did you mean " - "to use two backticks for ``code``, or miss an underscore for a " - "`link`_ ?" % (lineno, rawtext) + "Default role used (`single backticks`): %s. Did you mean to use two " + "backticks for ``code``, or miss an underscore for a `link`_ ?" + % rawtext ) - raise SphinxError(red(msg)) + logger.warning(msg, location=(inliner.document.current_source, lineno)) + return [nodes.Text(text)], [] diff --git a/docs/conf.py b/docs/conf.py index 972cc029d7da..525970262983 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -95,8 +95,9 @@ def django_release(): django_next_version = '3.1' extlinks = { + 'bpo': ('https://bugs.python.org/issue%s', 'bpo-'), 'commit': ('https://github.com/django/django/commit/%s', ''), - 'cve': ('https://nvd.nist.gov/view/vuln/detail?vulnId=%s', 'CVE-'), + 'cve': ('https://nvd.nist.gov/vuln/detail/CVE-%s', 'CVE-'), # A file or directory. GitHub redirects from blob to tree if needed. 'source': ('https://github.com/django/django/blob/master/%s', ''), 'ticket': ('https://code.djangoproject.com/ticket/%s', '#'), diff --git a/docs/faq/install.txt b/docs/faq/install.txt index fc8f2ec7dcb3..972fb73cd963 100644 --- a/docs/faq/install.txt +++ b/docs/faq/install.txt @@ -53,8 +53,8 @@ Django version Python versions 1.11 2.7, 3.4, 3.5, 3.6, 3.7 (added in 1.11.17) 2.0 3.4, 3.5, 3.6, 3.7 2.1 3.5, 3.6, 3.7 -2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8) -3.0 3.6, 3.7, 3.8 +2.2 3.5, 3.6, 3.7, 3.8 (added in 2.2.8), 3.9 (added in 2.2.17) +3.0 3.6, 3.7, 3.8, 3.9 (added in 3.0.11) ============== =============== For each version of Python, only the latest micro release (A.B.C) is officially diff --git a/docs/howto/custom-model-fields.txt b/docs/howto/custom-model-fields.txt index 8a8da037f3ee..9064ea11a53a 100644 --- a/docs/howto/custom-model-fields.txt +++ b/docs/howto/custom-model-fields.txt @@ -541,8 +541,8 @@ Converting Python objects to query values ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since using a database requires conversion in both ways, if you override -:meth:`~Field.to_python` you also have to override :meth:`~Field.get_prep_value` -to convert Python objects back to query values. +:meth:`~Field.from_db_value` you also have to override +:meth:`~Field.get_prep_value` to convert Python objects back to query values. For example:: diff --git a/docs/howto/windows.txt b/docs/howto/windows.txt index fecdbf3fba68..9149509604a1 100644 --- a/docs/howto/windows.txt +++ b/docs/howto/windows.txt @@ -4,27 +4,28 @@ How to install Django on Windows .. highlight:: doscon -This document will guide you through installing Python 3.7 and Django on +This document will guide you through installing Python 3.8 and Django on Windows. It also provides instructions for setting up a virtual environment, which makes it easier to work on Python projects. This is meant as a beginner's guide for users working on Django projects and does not reflect how Django should be installed when developing patches for Django itself. -The steps in this guide have been tested with Windows 7, 8, and 10. In other +The steps in this guide have been tested with Windows 10. In other versions, the steps would be similar. You will need to be familiar with using the Windows command prompt. +.. _install_python_windows: + Install Python ============== Django is a Python web framework, thus requiring Python to be installed on your -machine. At the time of writing, Python 3.7 is the latest version. +machine. At the time of writing, Python 3.8 is the latest version. To install Python on your machine go to https://python.org/downloads/. The website should offer you a download button for the latest Python version. -Download the executable installer and run it. Check the boxes next to ``Install -launcher for all users (recommended)`` and ``Add Python 3.7 to PATH`` then -click ``Install Now``. +Download the executable installer and run it. Check the boxes next to "Install +launcher for all users (recommended)" then click "Install Now". After installation, open the command prompt and check that the Python version matches the version you installed by executing:: @@ -38,13 +39,10 @@ matches the version you installed by executing:: About ``pip`` ============= -`pip`_ is a package manage for Python. It makes installing and uninstalling -Python packages (such as Django!) very easy. For the rest of the installation, -we'll use ``pip`` to install Python packages from the command line. - -To install pip on your machine, go to -https://pip.pypa.io/en/latest/installing/, and follow the ``Installing with -get-pip.py`` instructions. +`pip`_ is a package manager for Python and is included by default with the +Python installer. It helps to install and uninstall Python packages +(such as Django!). For the rest of the installation, we'll use ``pip`` to +install Python packages from the command line. .. _pip: https://pypi.org/project/pip/ diff --git a/docs/internals/contributing/writing-code/javascript.txt b/docs/internals/contributing/writing-code/javascript.txt index 09017af63a33..07bea197a26a 100644 --- a/docs/internals/contributing/writing-code/javascript.txt +++ b/docs/internals/contributing/writing-code/javascript.txt @@ -83,13 +83,13 @@ Django's JavaScript tests use `QUnit`_. Here is an example test module: QUnit.module('magicTricks', { beforeEach: function() { - var $ = django.jQuery; + const $ = django.jQuery; $('#qunit-fixture').append(''); } }); QUnit.test('removeOnClick removes button on click', function(assert) { - var $ = django.jQuery; + const $ = django.jQuery; removeOnClick('.button'); assert.equal($('.button').length, 1); $('.button').click(); @@ -97,7 +97,7 @@ Django's JavaScript tests use `QUnit`_. Here is an example test module: }); QUnit.test('copyOnClick adds button on click', function(assert) { - var $ = django.jQuery; + const $ = django.jQuery; copyOnClick('.button'); assert.equal($('.button').length, 1); $('.button').click(); diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index efc862dc76e4..ad7c5b494e93 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -277,7 +277,7 @@ dependencies: * geoip2_ * jinja2_ 2.7+ * numpy_ -* Pillow_ +* Pillow_ 4.2.0+ * PyYAML_ * pytz_ (required) * pywatchman_ diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt index 72a1d2a414a3..1ed21f9ba467 100644 --- a/docs/intro/contributing.txt +++ b/docs/intro/contributing.txt @@ -77,8 +77,7 @@ probably got the answers. .. admonition:: For Windows users - When installing Python on Windows, make sure you check the option "Add - python.exe to Path", so that it is always available on the command line. + See :ref:`install_python_windows` on Windows docs for additional guidance. Code of Conduct =============== diff --git a/docs/ref/class-based-views/mixins-single-object.txt b/docs/ref/class-based-views/mixins-single-object.txt index 4fb33f6ce8a7..1378e10823df 100644 --- a/docs/ref/class-based-views/mixins-single-object.txt +++ b/docs/ref/class-based-views/mixins-single-object.txt @@ -66,7 +66,7 @@ Single object mixins non-sequential arguments. Using a unique slug may serve the same purpose, but this scheme allows you to have non-unique slugs. - .. _insecure direct object reference: https://www.owasp.org/index.php/Top_10_2013-A4-Insecure_Direct_Object_References + .. _insecure direct object reference: https://wiki.owasp.org/index.php/Top_10_2013-A4-Insecure_Direct_Object_References .. method:: get_object(queryset=None) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index e68466a8c67a..14187891d9c2 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -109,7 +109,7 @@ Other topics The ``register`` decorator -------------------------- -.. function:: register(*models, site=django.admin.sites.site) +.. function:: register(*models, site=django.contrib.admin.sites.site) There is also a decorator for registering your ``ModelAdmin`` classes:: @@ -1088,9 +1088,9 @@ subclass:: .. attribute:: ModelAdmin.preserve_filters - The admin now preserves filters on the list view after creating, editing - or deleting an object. You can restore the previous behavior of clearing - filters by setting this attribute to ``False``. + By default, applied filters are preserved on the list view after creating, + editing, or deleting an object. You can have filters cleared by setting + this attribute to ``False``. .. attribute:: ModelAdmin.radio_fields @@ -1782,6 +1782,18 @@ templates used by the :class:`ModelAdmin` views: This uses the ``HttpRequest`` instance to filter the ``Car`` foreign key field to only display the cars owned by the ``User`` instance. + For more complex filters, you can use ``ModelForm.__init__()`` method to + filter based on an ``instance`` of your model (see + :ref:`fields-which-handle-relationships`). For example:: + + class CountryAdminForm(forms.ModelForm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['capital'].queryset = self.instance.cities.all() + + class CountryAdmin(admin.ModelAdmin): + form = CountryAdminForm + .. method:: ModelAdmin.formfield_for_manytomany(db_field, request, **kwargs) Like the ``formfield_for_foreignkey`` method, the diff --git a/docs/ref/contrib/gis/tutorial.txt b/docs/ref/contrib/gis/tutorial.txt index 8cf95622f49a..39d2b6787991 100644 --- a/docs/ref/contrib/gis/tutorial.txt +++ b/docs/ref/contrib/gis/tutorial.txt @@ -203,7 +203,7 @@ model to represent this data:: name = models.CharField(max_length=50) area = models.IntegerField() pop2005 = models.IntegerField('Population 2005') - fips = models.CharField('FIPS Code', max_length=2) + fips = models.CharField('FIPS Code', max_length=2, null=True) iso2 = models.CharField('2 Digit ISO', max_length=2) iso3 = models.CharField('3 Digit ISO', max_length=3) un = models.IntegerField('United Nations Code') diff --git a/docs/ref/contrib/postgres/aggregates.txt b/docs/ref/contrib/postgres/aggregates.txt index c70b6ec3b316..46d56d77773e 100644 --- a/docs/ref/contrib/postgres/aggregates.txt +++ b/docs/ref/contrib/postgres/aggregates.txt @@ -77,6 +77,20 @@ General-purpose aggregation functions Returns ``True``, if all input values are true, ``None`` if all values are null or if there are no values, otherwise ``False`` . + Usage example:: + + class Comment(models.Model): + body = models.TextField() + published = models.BooleanField() + rank = models.IntegerField() + + >>> from django.db.models import BooleanField, Q + >>> from django.contrib.postgres.aggregates import BoolAnd + >>> Comment.objects.aggregate(booland=BoolAnd('published')) + {'booland': False} + >>> Comment.objects.aggregate(booland=BoolAnd(Q(rank__lt=100), output_field=BooleanField())) + {'booland': True} + ``BoolOr`` ---------- @@ -85,6 +99,20 @@ General-purpose aggregation functions Returns ``True`` if at least one input value is true, ``None`` if all values are null or if there are no values, otherwise ``False``. + Usage example:: + + class Comment(models.Model): + body = models.TextField() + published = models.BooleanField() + rank = models.IntegerField() + + >>> from django.db.models import BooleanField, Q + >>> from django.contrib.postgres.aggregates import BoolOr + >>> Comment.objects.aggregate(boolor=BoolOr('published')) + {'boolor': True} + >>> Comment.objects.aggregate(boolor=BoolOr(Q(rank__gt=2), output_field=BooleanField())) + {'boolor': False} + ``JSONBAgg`` ------------ diff --git a/docs/ref/contrib/postgres/forms.txt b/docs/ref/contrib/postgres/forms.txt index bb78e5bc0571..62af57538660 100644 --- a/docs/ref/contrib/postgres/forms.txt +++ b/docs/ref/contrib/postgres/forms.txt @@ -255,4 +255,4 @@ Widgets Takes a single "compressed" value of a field, for example a :class:`~django.contrib.postgres.fields.DateRangeField`, - and returns a tuple representing and lower and upper bound. + and returns a tuple representing a lower and upper bound. diff --git a/docs/ref/csrf.txt b/docs/ref/csrf.txt index 20a8ddb433d9..0e4423248d23 100644 --- a/docs/ref/csrf.txt +++ b/docs/ref/csrf.txt @@ -85,11 +85,11 @@ You can acquire the token like this: .. code-block:: javascript function getCookie(name) { - var cookieValue = null; + let cookieValue = null; if (document.cookie && document.cookie !== '') { - var cookies = document.cookie.split(';'); - for (var i = 0; i < cookies.length; i++) { - var cookie = cookies[i].trim(); + const cookies = document.cookie.split(';'); + for (let i = 0; i < cookies.length; i++) { + const cookie = cookies[i].trim(); // Does this cookie string begin with the name we want? if (cookie.substring(0, name.length + 1) === (name + '=')) { cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); @@ -99,14 +99,14 @@ You can acquire the token like this: } return cookieValue; } - var csrftoken = getCookie('csrftoken'); + const csrftoken = getCookie('csrftoken'); The above code could be simplified by using the `JavaScript Cookie library `_ to replace ``getCookie``: .. code-block:: javascript - var csrftoken = Cookies.get('csrftoken'); + const csrftoken = Cookies.get('csrftoken'); .. note:: @@ -138,7 +138,7 @@ and read the token from the DOM with JavaScript: {% csrf_token %} Setting the token on the AJAX request diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 16a4c2c52680..1e761d263734 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -1037,9 +1037,7 @@ by 3rd parties that allow you to use other databases with Django: * `CockroachDB`_ * `Firebird`_ -* `IBM DB2`_ * `Microsoft SQL Server`_ -* `ODBC`_ The Django versions and ORM features supported by these unofficial backends vary considerably. Queries regarding the specific capabilities of these @@ -1048,6 +1046,4 @@ the support channels provided by each 3rd party project. .. _CockroachDB: https://pypi.org/project/django-cockroachdb/ .. _Firebird: https://pypi.org/project/django-firebird/ -.. _IBM DB2: https://pypi.org/project/ibm_db_django/ .. _Microsoft SQL Server: https://pypi.org/project/django-mssql-backend/ -.. _ODBC: https://pypi.org/project/django-pyodbc/ diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index b66ca5f5510f..2094c60ac9c6 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -156,7 +156,7 @@ are excluded. .. django-admin-option:: --use-fuzzy, -f -Includes fuzzy translations into compiled files. +Includes `fuzzy translations`_ into compiled files. Example usage:: @@ -169,6 +169,8 @@ Example usage:: django-admin compilemessages -x pt_BR django-admin compilemessages -x pt_BR -x fr +.. _fuzzy translations: https://www.gnu.org/software/gettext/manual/html_node/Fuzzy-Entries.html + .. django-admin-option:: --ignore PATTERN, -i PATTERN .. versionadded:: 3.0 diff --git a/docs/ref/exceptions.txt b/docs/ref/exceptions.txt index 83869d6b4247..8f276a993c71 100644 --- a/docs/ref/exceptions.txt +++ b/docs/ref/exceptions.txt @@ -26,12 +26,12 @@ Django core exception classes are defined in ``django.core.exceptions``. .. exception:: ObjectDoesNotExist - The base class for :exc:`~django.db.models.Model.DoesNotExist` exceptions; - a ``try/except`` for ``ObjectDoesNotExist`` will catch + The base class for :exc:`Model.DoesNotExist + ` exceptions. A ``try/except`` for + ``ObjectDoesNotExist`` will catch :exc:`~django.db.models.Model.DoesNotExist` exceptions for all models. - See :meth:`~django.db.models.query.QuerySet.get()` for further information - on :exc:`ObjectDoesNotExist` and :exc:`~django.db.models.Model.DoesNotExist`. + See :meth:`~django.db.models.query.QuerySet.get()`. ``EmptyResultSet`` ------------------ @@ -56,13 +56,13 @@ Django core exception classes are defined in ``django.core.exceptions``. .. exception:: MultipleObjectsReturned - The :exc:`MultipleObjectsReturned` exception is raised by a query if only - one object is expected, but multiple objects are returned. A base version - of this exception is provided in :mod:`django.core.exceptions`; each model - class contains a subclassed version that can be used to identify the - specific object type that has returned multiple objects. + The base class for :exc:`Model.MultipleObjectsReturned + ` exceptions. A + ``try/except`` for ``MultipleObjectsReturned`` will catch + :exc:`~django.db.models.Model.MultipleObjectsReturned` exceptions for all + models. - See :meth:`~django.db.models.query.QuerySet.get()` for further information. + See :meth:`~django.db.models.query.QuerySet.get()`. ``SuspiciousOperation`` ----------------------- diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 1d08e2d069fd..89b9c697f110 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -408,12 +408,12 @@ For each field, we describe the default widget used if you don't specify .. attribute:: choices Either an :term:`iterable` of 2-tuples to use as choices for this - field, or a callable that returns such an iterable. This argument - accepts the same formats as the ``choices`` argument to a model field. - See the :ref:`model field reference documentation on choices - ` for more details. If the argument is a callable, it is - evaluated each time the field's form is initialized. Defaults to an - empty list. + field, :ref:`enumeration ` choices, or a + callable that returns such an iterable. This argument accepts the same + formats as the ``choices`` argument to a model field. See the + :ref:`model field reference documentation on choices ` + for more details. If the argument is a callable, it is evaluated each + time the field's form is initialized. Defaults to an empty list. ``TypedChoiceField`` -------------------- @@ -1098,6 +1098,8 @@ Slightly complex built-in ``Field`` classes If no ``input_time_formats`` argument is provided, the default input formats for :class:`TimeField` are used. +.. _fields-which-handle-relationships: + Fields which handle relationships ================================= diff --git a/docs/ref/models/class.txt b/docs/ref/models/class.txt index 0183a82093b2..81414973a939 100644 --- a/docs/ref/models/class.txt +++ b/docs/ref/models/class.txt @@ -11,6 +11,34 @@ reference guides `. Attributes ========== +``DoesNotExist`` +---------------- + +.. exception:: Model.DoesNotExist + + This exception is raised by the ORM when an expected object is not found. + For example, :meth:`.QuerySet.get` will raise it when no object is found + for the given lookups. + + Django provides a ``DoesNotExist`` exception as an attribute of each model + class to identify the class of object that could not be found, allowing you + to catch exceptions for a particular model class. The exception is a + subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. + +``MultipleObjectsReturned`` +--------------------------- + +.. exception:: Model.MultipleObjectsReturned + + This exception is raised by :meth:`.QuerySet.get` when multiple objects are + found for the given lookups. + + Django provides a ``MultipleObjectsReturned`` exception as an attribute of + each model class to identify the class of object for which multiple objects + were found, allowing you to catch exceptions for a particular model class. + The exception is a subclass of + :exc:`django.core.exceptions.MultipleObjectsReturned`. + ``objects`` ----------- diff --git a/docs/ref/models/constraints.txt b/docs/ref/models/constraints.txt index f6d1842c14c5..1cb23e808039 100644 --- a/docs/ref/models/constraints.txt +++ b/docs/ref/models/constraints.txt @@ -65,7 +65,8 @@ ensures the age field is never less than 18. .. attribute:: CheckConstraint.name -The name of the constraint. +The name of the constraint. You must always specify a unique name for the +constraint. .. versionchanged:: 3.0 @@ -95,7 +96,8 @@ date. .. attribute:: UniqueConstraint.name -The name of the constraint. +The name of the constraint. You must always specify a unique name for the +constraint. .. versionchanged:: 3.0 diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 2fad2d64b7bd..03b713704baf 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -1713,6 +1713,13 @@ that control how the relationship functions. :ref:`extra data with a many-to-many relationship `. + .. note:: + + If you don't want multiple associations between the same instances, add + a :class:`~django.db.models.UniqueConstraint` including the from and to + fields. Django's automatically generated many-to-many tables include + such a constraint. + .. note:: Recursive relationships using an intermediary model and defined as @@ -1746,7 +1753,9 @@ that control how the relationship functions. points (i.e. the target model instance). This class can be used to query associated records for a given model - instance like a normal model. + instance like a normal model:: + + Model.m2mfield.through.objects.all() .. attribute:: ManyToManyField.through_fields @@ -1885,14 +1894,16 @@ your resulting ``User`` model will have the following attributes:: >>> hasattr(user, 'supervisor_of') True -A ``DoesNotExist`` exception is raised when accessing the reverse relationship -if an entry in the related table doesn't exist. For example, if a user doesn't -have a supervisor designated by ``MySpecialUser``:: +A ``RelatedObjectDoesNotExist`` exception is raised when accessing the reverse +relationship if an entry in the related table doesn't exist. This is a subclass +of the target model's :exc:`Model.DoesNotExist +` exception. For example, if a user +doesn't have a supervisor designated by ``MySpecialUser``:: >>> user.supervisor_of Traceback (most recent call last): ... - DoesNotExist: User matching query does not exist. + RelatedObjectDoesNotExist: User has no supervisor_of. .. _onetoone-arguments: diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index aef2e0e1ddc7..b439098674fe 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -81,7 +81,7 @@ fields are present, then ``values`` are guaranteed to be in the order to each of the missing fields. In addition to creating the new model, the ``from_db()`` method must set the -``adding`` and ``db`` flags in the new instance's ``_state`` attribute. +``adding`` and ``db`` flags in the new instance's :attr:`~Model._state` attribute. Below is an example showing how to record the initial values of fields that are loaded from the database:: @@ -841,16 +841,20 @@ duplicated. That also means you cannot use those methods on unsaved objects. Other attributes ================ -``DoesNotExist`` ----------------- +``_state`` +---------- -.. exception:: Model.DoesNotExist +.. attribute:: Model._state - This exception is raised by the ORM in a couple places, for example by - :meth:`QuerySet.get() ` when an object - is not found for the given query parameters. + The ``_state`` attribute refers to a ``ModelState`` object that tracks + the lifecycle of the model instance. - Django provides a ``DoesNotExist`` exception as an attribute of each model - class to identify the class of object that could not be found and to allow - you to catch a particular model class with ``try/except``. The exception is - a subclass of :exc:`django.core.exceptions.ObjectDoesNotExist`. + The ``ModelState`` object has two attributes: ``adding``, a flag which is + ``True`` if the model has not been saved to the database yet, and ``db``, + a string referring to the database alias the instance was loaded from or + saved to. + + Newly instantiated instances have ``adding=True`` and ``db=None``, + since they are yet to be saved. Instances fetched from a ``QuerySet`` + will have ``adding=False`` and ``db`` set to the alias of the associated + database. diff --git a/docs/ref/models/options.txt b/docs/ref/models/options.txt index 275f077bc842..b0ea93f9c941 100644 --- a/docs/ref/models/options.txt +++ b/docs/ref/models/options.txt @@ -157,11 +157,12 @@ Django quotes column and table names behind the scenes. part of a :djadmin:`flush` management command. That is, Django *manages* the database tables' lifecycles. - If ``False``, no database table creation or deletion operations will be - performed for this model. This is useful if the model represents an existing - table or a database view that has been created by some other means. This is - the *only* difference when ``managed=False``. All other aspects of - model handling are exactly the same as normal. This includes + If ``False``, no database table creation, modification, or deletion + operations will be performed for this model. This is useful if the model + represents an existing table or a database view that has been created by + some other means. This is the *only* difference when ``managed=False``. All + other aspects of model handling are exactly the same as normal. This + includes #. Adding an automatic primary key field to the model if you don't declare it. To avoid confusion for later code readers, it's diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index a64d986dbb40..c72c7885d0b3 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -851,8 +851,8 @@ duplicate values, use the ``all=True`` argument. of the type of the first ``QuerySet`` even if the arguments are ``QuerySet``\s of other models. Passing different models works as long as the ``SELECT`` list is the same in all ``QuerySet``\s (at least the types, the names don't matter -as long as the types in the same order). In such cases, you must use the column -names from the first ``QuerySet`` in ``QuerySet`` methods applied to the +as long as the types are in the same order). In such cases, you must use the +column names from the first ``QuerySet`` in ``QuerySet`` methods applied to the resulting ``QuerySet``. For example:: >>> qs1 = Author.objects.values_list('name') @@ -1823,34 +1823,42 @@ they query the database each time they're called. .. method:: get(**kwargs) Returns the object matching the given lookup parameters, which should be in -the format described in `Field lookups`_. +the format described in `Field lookups`_. You should use lookups that are +guaranteed unique, such as the primary key or fields in a unique constraint. +For example:: + + Entry.objects.get(id=1) + Entry.objects.get(blog=blog, entry_number=1) + +If you expect a queryset to already return one row, you can use ``get()`` +without any arguments to return the object for that row:: -``get()`` raises :exc:`~django.core.exceptions.MultipleObjectsReturned` if more -than one object was found. The -:exc:`~django.core.exceptions.MultipleObjectsReturned` exception is an -attribute of the model class. + Entry.objects.filter(pk=1).get() -``get()`` raises a :exc:`~django.db.models.Model.DoesNotExist` exception if an -object wasn't found for the given parameters. This exception is an attribute -of the model class. Example:: +If ``get()`` doesn't find any object, it raises a :exc:`Model.DoesNotExist +` exception:: - Entry.objects.get(id='foo') # raises Entry.DoesNotExist + Entry.objects.get(id=-999) # raises Entry.DoesNotExist -The :exc:`~django.db.models.Model.DoesNotExist` exception inherits from -:exc:`django.core.exceptions.ObjectDoesNotExist`, so you can target multiple -:exc:`~django.db.models.Model.DoesNotExist` exceptions. Example:: +If ``get()`` finds more than one object, it raises a +:exc:`Model.MultipleObjectsReturned +` exception:: + + Entry.objects.get(name='A Duplicated Name') # raises Entry.MultipleObjectsReturned + +Both these exception classes are attributes of the model class, and specific to +that model. If you want to handle such exceptions from several ``get()`` calls +for different models, you can use their generic base classes. For example, you +can use :exc:`django.core.exceptions.ObjectDoesNotExist` to handle +:exc:`~django.db.models.Model.DoesNotExist` exceptions from multiple models:: from django.core.exceptions import ObjectDoesNotExist + try: - e = Entry.objects.get(id=3) - b = Blog.objects.get(id=1) + blog = Blog.objects.get(id=1) + entry = Entry.objects.get(blog=blog, entry_number=1) except ObjectDoesNotExist: - print("Either the entry or blog doesn't exist.") - -If you expect a queryset to return one row, you can use ``get()`` without any -arguments to return the object for that row:: - - entry = Entry.objects.filter(...).exclude(...).get() + print("Either the blog or entry doesn't exist.") ``create()`` ~~~~~~~~~~~~ @@ -2100,6 +2108,17 @@ that fail constraints such as duplicate unique values. Enabling this parameter disables setting the primary key on each model instance (if the database normally supports it). +.. warning:: + + On MySQL and MariaDB, setting the ``ignore_conflicts`` parameter to + ``True`` turns certain types of errors, other than duplicate key, into + warnings. Even with Strict Mode. For example: invalid values or + non-nullable violations. See the `MySQL documentation`_ and + `MariaDB documentation`_ for more details. + +.. _MySQL documentation: https://dev.mysql.com/doc/refman/en/sql-mode.html#ignore-strict-comparison +.. _MariaDB documentation: https://mariadb.com/kb/en/ignore/ + .. versionchanged:: 2.2 The ``ignore_conflicts`` parameter was added. diff --git a/docs/ref/request-response.txt b/docs/ref/request-response.txt index 7b20f324a0c4..99878f6ab97c 100644 --- a/docs/ref/request-response.txt +++ b/docs/ref/request-response.txt @@ -43,8 +43,10 @@ All attributes should be considered read-only, unless stated otherwise. XML payload etc. For processing conventional form data, use :attr:`HttpRequest.POST`. - You can also read from an ``HttpRequest`` using a file-like interface. See - :meth:`HttpRequest.read()`. + You can also read from an ``HttpRequest`` using a file-like interface with + :meth:`HttpRequest.read` or :meth:`HttpRequest.readline`. Accessing + the ``body`` attribute *after* reading the request with either of these I/O + stream methods will produce a ``RawPostDataException``. .. attribute:: HttpRequest.path @@ -836,7 +838,7 @@ Methods isn't supported by all browsers, so it's not a replacement for Django's CSRF protection, but rather a defense in depth measure. - .. _HttpOnly: https://www.owasp.org/index.php/HttpOnly + .. _HttpOnly: https://owasp.org/www-community/HttpOnly .. _SameSite: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite .. warning:: @@ -856,7 +858,7 @@ Methods you will need to remember to pass it to the corresponding :meth:`HttpRequest.get_signed_cookie` call. -.. method:: HttpResponse.delete_cookie(key, path='/', domain=None) +.. method:: HttpResponse.delete_cookie(key, path='/', domain=None, samesite=None) Deletes the cookie with the given key. Fails silently if the key doesn't exist. @@ -865,6 +867,10 @@ Methods values you used in ``set_cookie()`` -- otherwise the cookie may not be deleted. + .. versionchanged:: 2.2.15 + + The ``samesite`` argument was added. + .. method:: HttpResponse.close() This method is called at the end of the request directly by the WSGI @@ -1078,7 +1084,7 @@ The :class:`StreamingHttpResponse` is not a subclass of :class:`HttpResponse`, because it features a slightly different API. However, it is almost identical, with the following notable differences: -* It should be given an iterator that yields strings as content. +* It should be given an iterator that yields bytestrings as content. * You cannot access its content, except by iterating the response object itself. This should only occur when the response is returned to the client. diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 42c0600b1c8e..cfabdf7d3979 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -246,7 +246,7 @@ See the :ref:`cache documentation ` for more information. ``CACHE_MIDDLEWARE_ALIAS`` -------------------------- -Default: ``default`` +Default: ``'default'`` The cache connection to use for the :ref:`cache middleware `. @@ -1489,10 +1489,9 @@ The numeric mode (i.e. ``0o644``) to set newly uploaded files to. For more information about what these modes mean, see the documentation for :func:`os.chmod`. -If this isn't given or is ``None``, you'll get operating-system -dependent behavior. On most platforms, temporary files will have a mode -of ``0o600``, and files saved from memory will be saved using the -system's standard umask. +If ``None``, you'll get operating-system dependent behavior. On most platforms, +temporary files will have a mode of ``0o600``, and files saved from memory will +be saved using the system's standard umask. For security reasons, these permissions aren't applied to the temporary files that are stored in :setting:`FILE_UPLOAD_TEMP_DIR`. @@ -2711,11 +2710,10 @@ See also :setting:`LANGUAGE_CODE`, :setting:`USE_I18N` and :setting:`USE_TZ`. Default: ``False`` A boolean that specifies whether to display numbers using a thousand separator. -When :setting:`USE_L10N` is set to ``True`` and if this is also set to -``True``, Django will use the values of :setting:`THOUSAND_SEPARATOR` and -:setting:`NUMBER_GROUPING` to format numbers unless the locale already has an -existing thousands separator. If there is a thousands separator in the locale -format, it will have higher precedence and will be applied instead. +When set to ``True`` and :setting:`USE_L10N` is also ``True``, Django will +format numbers using the :setting:`NUMBER_GROUPING` and +:setting:`THOUSAND_SEPARATOR` settings. These settings may also be dictated by +the locale, which takes precedence. See also :setting:`DECIMAL_SEPARATOR`, :setting:`NUMBER_GROUPING` and :setting:`THOUSAND_SEPARATOR`. @@ -3081,7 +3079,7 @@ vulnerability into full hijacking of a user's session. There aren't many good reasons for turning this off. Your code shouldn't read session cookies from JavaScript. -.. _HttpOnly: https://www.owasp.org/index.php/HttpOnly +.. _HttpOnly: https://owasp.org/www-community/HttpOnly .. setting:: SESSION_COOKIE_NAME diff --git a/docs/ref/signals.txt b/docs/ref/signals.txt index dc8c53ac2d20..a4ed248769d1 100644 --- a/docs/ref/signals.txt +++ b/docs/ref/signals.txt @@ -101,10 +101,10 @@ Arguments sent with this signal: .. note:: - ``instance._state`` isn't set before sending the ``post_init`` signal, - so ``_state`` attributes always have their default values. For example, - ``_state.db`` is ``None`` and cannot be used to check an ``instance`` - database. + :attr:`instance._state ` isn't set + before sending the ``post_init`` signal, so ``_state`` attributes + always have their default values. For example, ``_state.db`` is + ``None``. .. warning:: diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 387ad09a83df..ae62727266b5 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1816,7 +1816,7 @@ The resulting data can be accessed in JavaScript like this: .. code-block:: javascript - var value = JSON.parse(document.getElementById('hello-data').textContent); + const value = JSON.parse(document.getElementById('hello-data').textContent); XSS attacks are mitigated by escaping the characters "<", ">" and "&". For example if ``value`` is ``{'hello': 'world&'}``, the output is: diff --git a/docs/releases/1.3.txt b/docs/releases/1.3.txt index 7e02bcd0a52c..37b3cc7311ae 100644 --- a/docs/releases/1.3.txt +++ b/docs/releases/1.3.txt @@ -315,7 +315,7 @@ requests. These include: * Support for combining :class:`F expressions ` with ``timedelta`` values when retrieving or updating database values. -.. _HttpOnly: https://www.owasp.org/index.php/HttpOnly +.. _HttpOnly: https://owasp.org/www-community/HttpOnly .. _backwards-incompatible-changes-1.3: diff --git a/docs/releases/1.5.1.txt b/docs/releases/1.5.1.txt index cc961ac304bf..66d78997626a 100644 --- a/docs/releases/1.5.1.txt +++ b/docs/releases/1.5.1.txt @@ -10,10 +10,8 @@ compatible with Django 1.5, but includes a handful of fixes. The biggest fix is for a memory leak introduced in Django 1.5. Under certain circumstances, repeated iteration over querysets could leak memory - sometimes quite a bit of it. If you'd like more information, the details are in -:ticket:`our ticket tracker <19895>` (and in `a related issue`__ in Python -itself). - -__ https://bugs.python.org/issue17468 +:ticket:`our ticket tracker <19895>` (and in :bpo:`a related issue <17468>` in +Python itself). If you've noticed memory problems under Django 1.5, upgrading to 1.5.1 should fix those issues. diff --git a/docs/releases/1.6.11.txt b/docs/releases/1.6.11.txt index 8cf81f89bfdb..1bf2bf89110b 100644 --- a/docs/releases/1.6.11.txt +++ b/docs/releases/1.6.11.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that diff --git a/docs/releases/1.7.7.txt b/docs/releases/1.7.7.txt index f20ee127bcc5..bfd54563a1ee 100644 --- a/docs/releases/1.7.7.txt +++ b/docs/releases/1.7.7.txt @@ -13,9 +13,9 @@ Last year :func:`~django.utils.html.strip_tags` was changed to work iteratively. The problem is that the size of the input it's processing can increase on each iteration which results in an infinite loop in ``strip_tags()``. This issue only affects versions of Python that haven't -received `a bugfix in HTMLParser `_; namely -Python < 2.7.7 and 3.3.5. Some operating system vendors have also backported -the fix for the Python bug into their packages of earlier versions. +received :bpo:`a bugfix in HTMLParser <20288>`; namely Python < 2.7.7 and +3.3.5. Some operating system vendors have also backported the fix for the +Python bug into their packages of earlier versions. To remedy this issue, ``strip_tags()`` will now return the original input if it detects the length of the string it's processing increases. Remember that diff --git a/docs/releases/2.2.14.txt b/docs/releases/2.2.14.txt new file mode 100644 index 000000000000..38683cf30198 --- /dev/null +++ b/docs/releases/2.2.14.txt @@ -0,0 +1,13 @@ +=========================== +Django 2.2.14 release notes +=========================== + +*July 1, 2020* + +Django 2.2.14 fixes a bug in 2.2.13. + +Bugfixes +======== + +* Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` + warnings raised by cache key validation (:ticket:`31654`). diff --git a/docs/releases/2.2.15.txt b/docs/releases/2.2.15.txt new file mode 100644 index 000000000000..c36d746d5db9 --- /dev/null +++ b/docs/releases/2.2.15.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.15 release notes +=========================== + +*August 3, 2020* + +Django 2.2.15 fixes two bugs in 2.2.14. + +Bugfixes +======== + +* Allowed setting the ``SameSite`` cookie flag in + :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). + +* Fixed crash when sending emails to addresses with display names longer than + 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+ (:ticket:`31784`). diff --git a/docs/releases/2.2.16.txt b/docs/releases/2.2.16.txt new file mode 100644 index 000000000000..31231fb0655d --- /dev/null +++ b/docs/releases/2.2.16.txt @@ -0,0 +1,36 @@ +=========================== +Django 2.2.16 release notes +=========================== + +*September 1, 2020* + +Django 2.2.16 fixes two security issues and two data loss bugs in 2.2.15. + +CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ +====================================================================================== + +On Python 3.7+, :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` mode was not +applied to intermediate-level directories created in the process of uploading +files and to intermediate-level collected static directories when using the +:djadmin:`collectstatic` management command. + +You should review and manually fix permissions on existing intermediate-level +directories. + +CVE-2020-24584: Permission escalation in intermediate-level directories of the file system cache on Python 3.7+ +=============================================================================================================== + +On Python 3.7+, the intermediate-level directories of the file system cache had +the system's standard umask rather than ``0o077`` (no group or others +permissions). + +Bugfixes +======== + +* Fixed a data loss possibility in the + :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using + related fields pointing to a proxy model in the ``of`` argument, the + corresponding model was not locked (:ticket:`31866`). + +* Fixed a data loss possibility, following a regression in Django 2.0, when + copying model instances with a cached fields value (:ticket:`31863`). diff --git a/docs/releases/2.2.17.txt b/docs/releases/2.2.17.txt new file mode 100644 index 000000000000..4bea2eaed43a --- /dev/null +++ b/docs/releases/2.2.17.txt @@ -0,0 +1,7 @@ +=========================== +Django 2.2.17 release notes +=========================== + +*November 2, 2020* + +Django 2.2.17 adds compatibility with Python 3.9. diff --git a/docs/releases/2.2.18.txt b/docs/releases/2.2.18.txt new file mode 100644 index 000000000000..45df4fb83c9f --- /dev/null +++ b/docs/releases/2.2.18.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.18 release notes +=========================== + +*February 1, 2021* + +Django 2.2.18 fixes a security issue with severity "low" in 2.2.17. + +CVE-2021-3281: Potential directory-traversal via ``archive.extract()`` +====================================================================== + +The ``django.utils.archive.extract()`` function, used by +:option:`startapp --template` and :option:`startproject --template`, allowed +directory-traversal via an archive with absolute paths or relative paths with +dot segments. diff --git a/docs/releases/2.2.19.txt b/docs/releases/2.2.19.txt new file mode 100644 index 000000000000..feaffd996cac --- /dev/null +++ b/docs/releases/2.2.19.txt @@ -0,0 +1,16 @@ +=========================== +Django 2.2.19 release notes +=========================== + +*February 19, 2021* + +Django 2.2.19 fixes a security issue in 2.2.18. + +CVE-2021-23336: Web cache poisoning via ``django.utils.http.limited_parse_qsl()`` +================================================================================= + +Django contains a copy of :func:`urllib.parse.parse_qsl` which was added to +backport some security fixes. A further security fix has been issued recently +such that ``parse_qsl()`` no longer allows using ``;`` as a query parameter +separator by default. Django now includes this fix. See :bpo:`42967` for +further details. diff --git a/docs/releases/2.2.20.txt b/docs/releases/2.2.20.txt new file mode 100644 index 000000000000..a67c51502181 --- /dev/null +++ b/docs/releases/2.2.20.txt @@ -0,0 +1,15 @@ +=========================== +Django 2.2.20 release notes +=========================== + +*April 6, 2021* + +Django 2.2.20 fixes a security issue with severity "low" in 2.2.19. + +CVE-2021-28658: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser`` allowed directory-traversal via uploaded files with +suitably crafted file names. + +Built-in upload handlers were not affected by this vulnerability. diff --git a/docs/releases/2.2.txt b/docs/releases/2.2.txt index 195665b158e0..7f9822c8cbb2 100644 --- a/docs/releases/2.2.txt +++ b/docs/releases/2.2.txt @@ -23,9 +23,9 @@ end in April 2020. Python compatibility ==================== -Django 2.2 supports Python 3.5, 3.6, 3.7, and 3.8 (as of 2.2.8). We -**highly recommend** and only officially support the latest release of each -series. +Django 2.2 supports Python 3.5, 3.6, 3.7, 3.8 (as of 2.2.8), and 3.9 (as of +2.2.17). We **highly recommend** and only officially support the latest release +of each series. .. _whats-new-2.2: @@ -331,7 +331,7 @@ change shouldn't have an impact on your tests unless you've customized ``sqlparse`` is required dependency ----------------------------------- -To simplify a few parts of Django's database handling, `sqlparse +To simplify a few parts of Django's database handling, `sqlparse 0.2.2+ `_ is now a required dependency. It's automatically installed along with Django. @@ -475,6 +475,14 @@ Miscellaneous * Providing an integer in the ``key`` argument of the :meth:`.cache.delete` or :meth:`.cache.get` now raises :exc:`ValueError`. +* Plural equations for some languages are changed, because the latest versions + from Transifex are incorporated. + + .. note:: + + The ability to handle ``.po`` files containing different plural equations + for the same language was added in Django 2.2.12. + .. _deprecated-features-2.2: Features deprecated in 2.2 diff --git a/docs/releases/3.0.10.txt b/docs/releases/3.0.10.txt new file mode 100644 index 000000000000..0583f8d5aa5d --- /dev/null +++ b/docs/releases/3.0.10.txt @@ -0,0 +1,36 @@ +=========================== +Django 3.0.10 release notes +=========================== + +*September 1, 2020* + +Django 3.0.10 fixes two security issues and two data loss bugs in 3.0.9. + +CVE-2020-24583: Incorrect permissions on intermediate-level directories on Python 3.7+ +====================================================================================== + +On Python 3.7+, :setting:`FILE_UPLOAD_DIRECTORY_PERMISSIONS` mode was not +applied to intermediate-level directories created in the process of uploading +files and to intermediate-level collected static directories when using the +:djadmin:`collectstatic` management command. + +You should review and manually fix permissions on existing intermediate-level +directories. + +CVE-2020-24584: Permission escalation in intermediate-level directories of the file system cache on Python 3.7+ +=============================================================================================================== + +On Python 3.7+, the intermediate-level directories of the file system cache had +the system's standard umask rather than ``0o077`` (no group or others +permissions). + +Bugfixes +======== + +* Fixed a data loss possibility in the + :meth:`~django.db.models.query.QuerySet.select_for_update()`. When using + related fields pointing to a proxy model in the ``of`` argument, the + corresponding model was not locked (:ticket:`31866`). + +* Fixed a data loss possibility, following a regression in Django 2.0, when + copying model instances with a cached fields value (:ticket:`31863`). diff --git a/docs/releases/3.0.11.txt b/docs/releases/3.0.11.txt new file mode 100644 index 000000000000..a5a45b2ab710 --- /dev/null +++ b/docs/releases/3.0.11.txt @@ -0,0 +1,14 @@ +=========================== +Django 3.0.11 release notes +=========================== + +*November 2, 2020* + +Django 3.0.11 fixes a regression in 3.0.7 and adds compatibility with Python +3.9. + +Bugfixes +======== + +* Fixed a regression in Django 3.0.7 that didn't use ``Subquery()`` aliases in + the ``GROUP BY`` clause (:ticket:`32152`). diff --git a/docs/releases/3.0.12.txt b/docs/releases/3.0.12.txt new file mode 100644 index 000000000000..20d9459feae2 --- /dev/null +++ b/docs/releases/3.0.12.txt @@ -0,0 +1,15 @@ +=========================== +Django 3.0.12 release notes +=========================== + +*February 1, 2021* + +Django 3.0.12 fixes a security issue with severity "low" in 3.0.11. + +CVE-2021-3281: Potential directory-traversal via ``archive.extract()`` +====================================================================== + +The ``django.utils.archive.extract()`` function, used by +:option:`startapp --template` and :option:`startproject --template`, allowed +directory-traversal via an archive with absolute paths or relative paths with +dot segments. diff --git a/docs/releases/3.0.13.txt b/docs/releases/3.0.13.txt new file mode 100644 index 000000000000..c78b8a04fd15 --- /dev/null +++ b/docs/releases/3.0.13.txt @@ -0,0 +1,16 @@ +=========================== +Django 3.0.13 release notes +=========================== + +*February 19, 2021* + +Django 3.0.13 fixes a security issue in 3.0.12. + +CVE-2021-23336: Web cache poisoning via ``django.utils.http.limited_parse_qsl()`` +================================================================================= + +Django contains a copy of :func:`urllib.parse.parse_qsl` which was added to +backport some security fixes. A further security fix has been issued recently +such that ``parse_qsl()`` no longer allows using ``;`` as a query parameter +separator by default. Django now includes this fix. See :bpo:`42967` for +further details. diff --git a/docs/releases/3.0.14.txt b/docs/releases/3.0.14.txt new file mode 100644 index 000000000000..c32442874523 --- /dev/null +++ b/docs/releases/3.0.14.txt @@ -0,0 +1,15 @@ +=========================== +Django 3.0.14 release notes +=========================== + +*April 6, 2021* + +Django 3.0.14 fixes a security issue with severity "low" in 3.0.13. + +CVE-2021-28658: Potential directory-traversal via uploaded files +================================================================ + +``MultiPartParser`` allowed directory-traversal via uploaded files with +suitably crafted file names. + +Built-in upload handlers were not affected by this vulnerability. diff --git a/docs/releases/3.0.8.txt b/docs/releases/3.0.8.txt new file mode 100644 index 000000000000..f8fc8ab96141 --- /dev/null +++ b/docs/releases/3.0.8.txt @@ -0,0 +1,26 @@ +========================== +Django 3.0.8 release notes +========================== + +*July 1, 2020* + +Django 3.0.8 fixes several bugs in 3.0.7. + +Bugfixes +======== + +* Fixed messages of ``InvalidCacheKey`` exceptions and ``CacheKeyWarning`` + warnings raised by cache key validation (:ticket:`31654`). + +* Fixed a regression in Django 3.0.7 that caused a queryset crash when grouping + by a many-to-one relationship (:ticket:`31660`). + +* Reallowed, following a regression in Django 3.0, non-expressions having a + ``filterable`` attribute to be used as the right-hand side in queryset + filters (:ticket:`31664`). + +* Fixed a regression in Django 3.0.2 that caused a migration crash on + PostgreSQL when adding a foreign key to a model with a namespaced + ``db_table`` (:ticket:`31735`). + +* Added compatibility for ``cx_Oracle`` 8 (:ticket:`31751`). diff --git a/docs/releases/3.0.9.txt b/docs/releases/3.0.9.txt new file mode 100644 index 000000000000..7317ead794f5 --- /dev/null +++ b/docs/releases/3.0.9.txt @@ -0,0 +1,16 @@ +========================== +Django 3.0.9 release notes +========================== + +*August 3, 2020* + +Django 3.0.9 fixes several bugs in 3.0.8. + +Bugfixes +======== + +* Allowed setting the ``SameSite`` cookie flag in + :meth:`.HttpResponse.delete_cookie` (:ticket:`31790`). + +* Fixed crash when sending emails to addresses with display names longer than + 75 chars on Python 3.6.11+, 3.7.8+, and 3.8.4+ (:ticket:`31784`). diff --git a/docs/releases/3.0.txt b/docs/releases/3.0.txt index 51ca584ecb5d..1c39980a91ed 100644 --- a/docs/releases/3.0.txt +++ b/docs/releases/3.0.txt @@ -19,8 +19,8 @@ project. Python compatibility ==================== -Django 3.0 supports Python 3.6, 3.7, and 3.8. We **highly recommend** and only -officially support the latest release of each series. +Django 3.0 supports Python 3.6, 3.7, 3.8, and 3.9 (as of 3.0.11). We **highly +recommend** and only officially support the latest release of each series. The Django 2.2.x series is the last to support Python 3.5. @@ -513,7 +513,7 @@ In older versions, the :setting:`FILE_UPLOAD_PERMISSIONS` setting defaults to uploaded files having different permissions depending on their size and which upload handler is used. -``FILE_UPLOAD_PERMISSION`` now defaults to ``0o644`` to avoid this +``FILE_UPLOAD_PERMISSIONS`` now defaults to ``0o644`` to avoid this inconsistency. New default values for security settings @@ -571,8 +571,6 @@ Miscellaneous * ``alias=None`` is added to the signature of :meth:`.Expression.get_group_by_cols`. -* Support for ``sqlparse`` < 0.2.2 is removed. - * ``RegexPattern``, used by :func:`~django.urls.re_path`, no longer returns keyword arguments with ``None`` values to be passed to the view for the optional named groups that are missing. diff --git a/docs/releases/index.txt b/docs/releases/index.txt index a9581634e265..bc30665aea64 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -25,6 +25,13 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 3.0.14 + 3.0.13 + 3.0.12 + 3.0.11 + 3.0.10 + 3.0.9 + 3.0.8 3.0.7 3.0.6 3.0.5 @@ -39,6 +46,13 @@ versions of the documentation contain the release notes for any later releases. .. toctree:: :maxdepth: 1 + 2.2.20 + 2.2.19 + 2.2.18 + 2.2.17 + 2.2.16 + 2.2.15 + 2.2.14 2.2.13 2.2.12 2.2.11 diff --git a/docs/releases/security.txt b/docs/releases/security.txt index 340aba041b02..10f871d563fd 100644 --- a/docs/releases/security.txt +++ b/docs/releases/security.txt @@ -1082,3 +1082,83 @@ Versions affected * Django 3.0 :commit:`(patch) <26a5cf834526e291db00385dd33d319b8271fc4c>` * Django 2.2 :commit:`(patch) ` * Django 1.11 :commit:`(patch) <02d97f3c9a88adc890047996e5606180bd1c6166>` + +June 3, 2020 - :cve:`2020-13254` +-------------------------------- + +Potential data leakage via malformed memcached keys. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <84b2da5552e100ae3294f564f6c862fef8d0e693>` +* Django 2.2 :commit:`(patch) <07e59caa02831c4569bbebb9eb773bdd9cb4b206>` + +June 3, 2020 - :cve:`2020-13596` +-------------------------------- + +Possible XSS via admin ``ForeignKeyRawIdWidget``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.0 :commit:`(patch) <1f2dd37f6fcefdd10ed44cb233b2e62b520afb38>` +* Django 2.2 :commit:`(patch) <6d61860b22875f358fac83d903dc629897934815>` + +September 1, 2020 - :cve:`2020-24583` +------------------------------------- + +Incorrect permissions on intermediate-level directories on Python 3.7+. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <934430d22aa5d90c2ba33495ff69a6a1d997d584>` +* Django 3.0 :commit:`(patch) <08892bffd275c79ee1f8f67639eb170aaaf1181e>` +* Django 2.2 :commit:`(patch) <375657a71c889c588f723469bd868bd1d40c369f>` + +September 1, 2020 - :cve:`2020-24584` +------------------------------------- + +Permission escalation in intermediate-level directories of the file system +cache on Python 3.7+. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <2b099caa5923afa8cfb5f1e8c0d56b6e0e81915b>` +* Django 3.0 :commit:`(patch) ` +* Django 2.2 :commit:`(patch) ` + +February 1, 2021 - :cve:`2021-3281` +----------------------------------- + +Potential directory-traversal via ``archive.extract()``. `Full description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.1 :commit:`(patch) <02e6592835b4559909aa3aaaf67988fef435f624>` +* Django 3.0 :commit:`(patch) <52e409ed17287e9aabda847b6afe58be2fa9f86a>` +* Django 2.2 :commit:`(patch) <21e7622dec1f8612c85c2fc37fe8efbfd3311e37>` + +February 19, 2021 - :cve:`2021-23336` +------------------------------------- + +Web cache poisoning via ``django.utils.http.limited_parse_qsl()``. `Full +description +`__ + +Versions affected +~~~~~~~~~~~~~~~~~ + +* Django 3.2 :commit:`(patch) ` +* Django 3.1 :commit:`(patch) <8f6d431b08cbb418d9144b976e7b972546607851>` +* Django 3.0 :commit:`(patch) <326a926beef869d3341bc9ef737887f0449b6b71>` +* Django 2.2 :commit:`(patch) ` diff --git a/docs/topics/db/multi-db.txt b/docs/topics/db/multi-db.txt index be660cfcc2de..afd6e28accc2 100644 --- a/docs/topics/db/multi-db.txt +++ b/docs/topics/db/multi-db.txt @@ -239,9 +239,10 @@ database usage. Whenever a query needs to know which database to use, it calls the master router, providing a model and a hint (if available). Django then tries each router in turn until a database suggestion can be found. If no suggestion can be found, it tries the -current ``_state.db`` of the hint instance. If a hint instance wasn't -provided, or the instance doesn't currently have database state, the -master router will allocate the ``default`` database. +current :attr:`instance._state.db ` of the hint +instance. If a hint instance wasn't provided, or :attr:`instance._state.db +` is ``None``, the master router will allocate +the ``default`` database. An example ---------- diff --git a/docs/topics/i18n/translation.txt b/docs/topics/i18n/translation.txt index f29da625de34..34098a627ee3 100644 --- a/docs/topics/i18n/translation.txt +++ b/docs/topics/i18n/translation.txt @@ -447,7 +447,7 @@ helper function described next. Lazy translations and plural ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When using lazy translation for a plural string (``[u]n[p]gettext_lazy``), you +When using lazy translation for a plural string (``n[p]gettext_lazy``), you generally don't know the ``number`` argument at the time of the string definition. Therefore, you are authorized to pass a key name instead of an integer as the ``number`` argument. Then ``number`` will be looked up in the @@ -1067,9 +1067,12 @@ interface within your Python code:: The ``ngettext`` function provides an interface to pluralize words and phrases:: - var object_count = 1 // or 0, or 2, or 3, ... - s = ngettext('literal for the singular case', - 'literal for the plural case', object_count); + const objectCount = 1 // or 0, or 2, or 3, ... + const string = ngettext( + 'literal for the singular case', + 'literal for the plural case', + objectCount + ); ``interpolate`` ~~~~~~~~~~~~~~~ @@ -1083,23 +1086,29 @@ function supports both positional and named interpolation: corresponding ``fmt`` placeholders in the same order they appear. For example:: - fmts = ngettext('There is %s object. Remaining: %s', - 'There are %s objects. Remaining: %s', 11); - s = interpolate(fmts, [11, 20]); - // s is 'There are 11 objects. Remaining: 20' + const formats = ngettext( + 'There is %s object. Remaining: %s', + 'There are %s objects. Remaining: %s', + 11 + ); + const string = interpolate(formats, [11, 20]); + // string is 'There are 11 objects. Remaining: 20' * Named interpolation: This mode is selected by passing the optional boolean ``named`` parameter as ``true``. ``obj`` contains a JavaScript object or associative array. For example:: - d = { - count: 10, - total: 50 + const data = { + count: 10, + total: 50 }; - fmts = ngettext('Total: %(total)s, there is %(count)s object', - 'there are %(count)s of a total of %(total)s objects', d.count); - s = interpolate(fmts, d, true); + const formats = ngettext( + 'Total: %(total)s, there is %(count)s object', + 'there are %(count)s of a total of %(total)s objects', + data.count + ); + const string = interpolate(formats, data, true); You shouldn't go over the top with string interpolation, though: this is still JavaScript, so the code has to make repeated regular-expression substitutions. @@ -1606,6 +1615,13 @@ otherwise, they'll be tacked together without whitespace! files are created). This means that everybody will be using the same encoding, which is important when Django processes the PO files. +.. admonition:: Fuzzy entries + + :djadmin:`makemessages` sometimes generates translation entries marked as + fuzzy, e.g. when translations are inferred from previously translated + strings. By default, fuzzy entries are **not** processed by + :djadmin:`compilemessages`. + To reexamine all source code and templates for new translation strings and update all message files for **all** languages, run this:: diff --git a/docs/topics/security.txt b/docs/topics/security.txt index ba73f2089913..426c33d03516 100644 --- a/docs/topics/security.txt +++ b/docs/topics/security.txt @@ -294,5 +294,5 @@ security protection of the Web server, operating system and other components. pages also include security principles that apply to any system. .. _LimitRequestBody: https://httpd.apache.org/docs/2.4/mod/core.html#limitrequestbody -.. _Top 10 list: https://www.owasp.org/index.php/Top_10-2017_Top_10 +.. _Top 10 list: https://owasp.org/www-project-top-ten/OWASP_Top_Ten_2017/ .. _web security: https://infosec.mozilla.org/guidelines/web_security.html diff --git a/docs/topics/signals.txt b/docs/topics/signals.txt index ee097f9faa46..33b6a1b3ae32 100644 --- a/docs/topics/signals.txt +++ b/docs/topics/signals.txt @@ -188,7 +188,11 @@ Preventing duplicate signals In some circumstances, the code connecting receivers to signals may run multiple times. This can cause your receiver function to be registered more -than once, and thus called multiple times for a single signal event. +than once, and thus called as many times for a signal event. For example, the +:meth:`~django.apps.AppConfig.ready` method may be executed more than once +during testing. More generally, this occurs everywhere your project imports the +module where you define the signals, because signal registration runs as many +times as it is imported. If this behavior is problematic (such as when using signals to send an email whenever a model is saved), pass a unique identifier as diff --git a/docs/topics/signing.txt b/docs/topics/signing.txt index cd2b4ef2e8aa..c60930c1e5c1 100644 --- a/docs/topics/signing.txt +++ b/docs/topics/signing.txt @@ -167,12 +167,12 @@ and tuples) if you pass in a tuple, you will get a list from >>> signing.loads(value) ['a', 'b', 'c'] -.. function:: dumps(obj, key=None, salt='django.core.signing', compress=False) +.. function:: dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, compress=False) Returns URL-safe, sha1 signed base64 compressed JSON string. Serialized object is signed using :class:`~TimestampSigner`. -.. function:: loads(string, key=None, salt='django.core.signing', max_age=None) +.. function:: loads(string, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None) Reverse of ``dumps()``, raises ``BadSignature`` if signature fails. Checks ``max_age`` (in seconds) if given. diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt index cdff8889ee5e..f49177fe5969 100644 --- a/docs/topics/testing/advanced.txt +++ b/docs/topics/testing/advanced.txt @@ -499,7 +499,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a selection of other methods that are used to by ``run_tests()`` to set up, execute and tear down the test suite. -.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, test_name_patterns=None, **kwargs) +.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, test_name_patterns=None, pdb=False, **kwargs) ``DiscoverRunner`` will search for tests in any file matching ``pattern``. @@ -541,6 +541,9 @@ execute and tear down the test suite. ``test_name_patterns`` can be used to specify a set of patterns for filtering test methods and classes by their names. + If ``pdb`` is ``True``, a debugger (``pdb`` or ``ipdb``) will be spawned at + each test error or failure. + Django may, from time to time, extend the capabilities of the test runner by adding new arguments. The ``**kwargs`` declaration allows for this expansion. If you subclass ``DiscoverRunner`` or write your own test @@ -551,6 +554,10 @@ execute and tear down the test suite. custom arguments by calling ``parser.add_argument()`` inside the method, so that the :djadmin:`test` command will be able to use those arguments. + .. versionadded:: 3.0 + + The ``pdb`` argument was added. + Attributes ~~~~~~~~~~ diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt index 57f2e6b7727f..4f63ab903155 100644 --- a/docs/topics/testing/tools.txt +++ b/docs/topics/testing/tools.txt @@ -1687,10 +1687,11 @@ your test suite. Asserts that a queryset ``qs`` returns a particular list of values ``values``. - The comparison of the contents of ``qs`` and ``values`` is performed using - the function ``transform``; by default, this means that the ``repr()`` of - each value is compared. Any other callable can be used if ``repr()`` doesn't - provide a unique or helpful comparison. + The comparison of the contents of ``qs`` and ``values`` is performed by + applying ``transform`` to ``qs``. By default, this means that the + ``repr()`` of each value in ``qs`` is compared to the ``values``. Any other + callable can be used if ``repr()`` doesn't provide a unique or helpful + comparison. By default, the comparison is also ordering dependent. If ``qs`` doesn't provide an implicit ordering, you can set the ``ordered`` parameter to diff --git a/setup.py b/setup.py index eb133926bafe..2c6170fe5b86 100644 --- a/setup.py +++ b/setup.py @@ -101,6 +101,7 @@ def read(fname): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', diff --git a/tests/annotations/tests.py b/tests/annotations/tests.py index 0064b14ed07e..2ed3474f5d6b 100644 --- a/tests/annotations/tests.py +++ b/tests/annotations/tests.py @@ -1,14 +1,16 @@ import datetime from decimal import Decimal +from unittest import skipIf from django.core.exceptions import FieldDoesNotExist, FieldError +from django.db import connection from django.db.models import ( - BooleanField, CharField, Count, DateTimeField, Exists, ExpressionWrapper, - F, Func, IntegerField, Max, NullBooleanField, OuterRef, Q, Subquery, Sum, - Value, + BooleanField, Case, CharField, Count, DateTimeField, Exists, + ExpressionWrapper, F, Func, IntegerField, Max, NullBooleanField, OuterRef, + Q, Subquery, Sum, Value, When, ) from django.db.models.expressions import RawSQL -from django.db.models.functions import Length, Lower +from django.db.models.functions import ExtractYear, Length, Lower from django.test import TestCase, skipUnlessDBFeature from .models import ( @@ -632,3 +634,41 @@ def test_annotation_exists_aggregate_values_chaining(self): datetime.date(2008, 6, 23), datetime.date(2008, 11, 3), ]) + + @skipUnlessDBFeature('supports_subqueries_in_group_by') + def test_annotation_subquery_and_aggregate_values_chaining(self): + qs = Book.objects.annotate( + pub_year=ExtractYear('pubdate') + ).values('pub_year').annotate( + top_rating=Subquery( + Book.objects.filter( + pubdate__year=OuterRef('pub_year') + ).order_by('-rating').values('rating')[:1] + ), + total_pages=Sum('pages'), + ).values('pub_year', 'total_pages', 'top_rating') + self.assertCountEqual(qs, [ + {'pub_year': 1991, 'top_rating': 5.0, 'total_pages': 946}, + {'pub_year': 1995, 'top_rating': 4.0, 'total_pages': 1132}, + {'pub_year': 2007, 'top_rating': 4.5, 'total_pages': 447}, + {'pub_year': 2008, 'top_rating': 4.0, 'total_pages': 1178}, + ]) + + @skipIf( + connection.vendor == 'mysql', + 'GROUP BY optimization does not work properly when ONLY_FULL_GROUP_BY ' + 'mode is enabled on MySQL, see #31331.', + ) + def test_annotation_aggregate_with_m2o(self): + qs = Author.objects.filter(age__lt=30).annotate( + max_pages=Case( + When(book_contact_set__isnull=True, then=Value(0)), + default=Max(F('book__pages')), + output_field=IntegerField(), + ), + ).values('name', 'max_pages') + self.assertCountEqual(qs, [ + {'name': 'James Bennett', 'max_pages': 300}, + {'name': 'Paul Bissex', 'max_pages': 0}, + {'name': 'Wesley J. Chun', 'max_pages': 0}, + ]) diff --git a/tests/cache/tests.py b/tests/cache/tests.py index a30e4ceeb89d..c0f90b832637 100644 --- a/tests/cache/tests.py +++ b/tests/cache/tests.py @@ -6,11 +6,13 @@ import pickle import re import shutil +import sys import tempfile import threading import time import unittest -from unittest import mock +from pathlib import Path +from unittest import mock, skipIf from django.conf import settings from django.core import management, signals @@ -624,8 +626,9 @@ def func(key, *args): cache.key_func = func try: - with self.assertWarnsMessage(CacheKeyWarning, expected_warning): + with self.assertWarns(CacheKeyWarning) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.warning), expected_warning) finally: cache.key_func = old_func @@ -1261,9 +1264,10 @@ def _perform_invalid_key_test(self, key, expected_warning): Whilst other backends merely warn, memcached should raise for an invalid key. """ - msg = expected_warning.replace(key, ':1:%s' % key) - with self.assertRaisesMessage(InvalidCacheKey, msg): + msg = expected_warning.replace(key, cache.make_key(key)) + with self.assertRaises(InvalidCacheKey) as cm: cache.set(key, 'value') + self.assertEqual(str(cm.exception), msg) def test_default_never_expiring_timeout(self): # Regression test for #22845 @@ -1441,6 +1445,28 @@ def test_get_ignores_enoent(self): # Returns the default instead of erroring. self.assertEqual(cache.get('foo', 'baz'), 'baz') + @skipIf( + sys.platform == 'win32', + 'Windows only partially supports umasks and chmod.', + ) + def test_cache_dir_permissions(self): + os.rmdir(self.dirname) + dir_path = Path(self.dirname) / 'nested' / 'filebasedcache' + for cache_params in settings.CACHES.values(): + cache_params['LOCATION'] = dir_path + setting_changed.send(self.__class__, setting='CACHES', enter=False) + cache.set('foo', 'bar') + self.assertIs(dir_path.exists(), True) + tests = [ + dir_path, + dir_path.parent, + dir_path.parent.parent, + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o700) + def test_get_does_not_ignore_non_filenotfound_exceptions(self): with mock.patch('builtins.open', side_effect=OSError): with self.assertRaises(OSError): diff --git a/tests/file_storage/tests.py b/tests/file_storage/tests.py index 1c4176014c51..e2a1d06b5dc2 100644 --- a/tests/file_storage/tests.py +++ b/tests/file_storage/tests.py @@ -7,6 +7,7 @@ import unittest from datetime import datetime, timedelta from io import StringIO +from pathlib import Path from urllib.request import urlopen from django.core.cache import cache @@ -910,16 +911,19 @@ def test_file_upload_default_permissions(self): @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o765) def test_file_upload_directory_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o765) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + self.assertEqual(file_path.parent.stat().st_mode & 0o777, 0o765) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, 0o765) @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=None) def test_file_upload_directory_default_permissions(self): self.storage = FileSystemStorage(self.storage_dir) - name = self.storage.save("the_directory/the_file", ContentFile("data")) - dir_mode = os.stat(os.path.dirname(self.storage.path(name)))[0] & 0o777 - self.assertEqual(dir_mode, 0o777 & ~self.umask) + name = self.storage.save('the_directory/subdir/the_file', ContentFile('data')) + file_path = Path(self.storage.path(name)) + expected_mode = 0o777 & ~self.umask + self.assertEqual(file_path.parent.stat().st_mode & 0o777, expected_mode) + self.assertEqual(file_path.parent.parent.stat().st_mode & 0o777, expected_mode) class FileStoragePathParsing(SimpleTestCase): diff --git a/tests/file_uploads/tests.py b/tests/file_uploads/tests.py index 7b31d369b2bc..f6ab0d183cf4 100644 --- a/tests/file_uploads/tests.py +++ b/tests/file_uploads/tests.py @@ -22,6 +22,22 @@ MEDIA_ROOT = sys_tempfile.mkdtemp() UPLOAD_TO = os.path.join(MEDIA_ROOT, 'test_upload') +CANDIDATE_TRAVERSAL_FILE_NAMES = [ + '/tmp/hax0rd.txt', # Absolute path, *nix-style. + 'C:\\Windows\\hax0rd.txt', # Absolute path, win-style. + 'C:/Windows/hax0rd.txt', # Absolute path, broken-style. + '\\tmp\\hax0rd.txt', # Absolute path, broken in a different way. + '/tmp\\hax0rd.txt', # Absolute path, broken by mixing. + 'subdir/hax0rd.txt', # Descendant path, *nix-style. + 'subdir\\hax0rd.txt', # Descendant path, win-style. + 'sub/dir\\hax0rd.txt', # Descendant path, mixed. + '../../hax0rd.txt', # Relative path, *nix-style. + '..\\..\\hax0rd.txt', # Relative path, win-style. + '../..\\hax0rd.txt', # Relative path, mixed. + '../hax0rd.txt', # HTML entities. + '../hax0rd.txt', # HTML entities. +] + @override_settings(MEDIA_ROOT=MEDIA_ROOT, ROOT_URLCONF='file_uploads.urls', MIDDLEWARE=[]) class FileUploadTests(TestCase): @@ -204,22 +220,8 @@ def test_dangerous_file_names(self): # a malicious payload with an invalid file name (containing os.sep or # os.pardir). This similar to what an attacker would need to do when # trying such an attack. - scary_file_names = [ - "/tmp/hax0rd.txt", # Absolute path, *nix-style. - "C:\\Windows\\hax0rd.txt", # Absolute path, win-style. - "C:/Windows/hax0rd.txt", # Absolute path, broken-style. - "\\tmp\\hax0rd.txt", # Absolute path, broken in a different way. - "/tmp\\hax0rd.txt", # Absolute path, broken by mixing. - "subdir/hax0rd.txt", # Descendant path, *nix-style. - "subdir\\hax0rd.txt", # Descendant path, win-style. - "sub/dir\\hax0rd.txt", # Descendant path, mixed. - "../../hax0rd.txt", # Relative path, *nix-style. - "..\\..\\hax0rd.txt", # Relative path, win-style. - "../..\\hax0rd.txt" # Relative path, mixed. - ] - payload = client.FakePayload() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): payload.write('\r\n'.join([ '--' + client.BOUNDARY, 'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name), @@ -239,7 +241,7 @@ def test_dangerous_file_names(self): response = self.client.request(**r) # The filenames should have been sanitized by the time it got to the view. received = response.json() - for i, name in enumerate(scary_file_names): + for i, name in enumerate(CANDIDATE_TRAVERSAL_FILE_NAMES): got = received["file%s" % i] self.assertEqual(got, "hax0rd.txt") @@ -517,6 +519,47 @@ def test_filename_case_preservation(self): # shouldn't differ. self.assertEqual(os.path.basename(obj.testfile.path), 'MiXeD_cAsE.txt') + def test_filename_traversal_upload(self): + os.makedirs(UPLOAD_TO, exist_ok=True) + self.addCleanup(shutil.rmtree, MEDIA_ROOT) + tests = [ + '../test.txt', + '../test.txt', + ] + for file_name in tests: + with self.subTest(file_name=file_name): + payload = client.FakePayload() + payload.write( + '\r\n'.join([ + '--' + client.BOUNDARY, + 'Content-Disposition: form-data; name="my_file"; ' + 'filename="%s";' % file_name, + 'Content-Type: text/plain', + '', + 'file contents.\r\n', + '\r\n--' + client.BOUNDARY + '--\r\n', + ]), + ) + r = { + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': client.MULTIPART_CONTENT, + 'PATH_INFO': '/upload_traversal/', + 'REQUEST_METHOD': 'POST', + 'wsgi.input': payload, + } + response = self.client.request(**r) + result = response.json() + self.assertEqual(response.status_code, 200) + self.assertEqual(result['file_name'], 'test.txt') + self.assertIs( + os.path.exists(os.path.join(MEDIA_ROOT, 'test.txt')), + False, + ) + self.assertIs( + os.path.exists(os.path.join(UPLOAD_TO, 'test.txt')), + True, + ) + @override_settings(MEDIA_ROOT=MEDIA_ROOT) class DirectoryCreationTests(SimpleTestCase): @@ -586,6 +629,15 @@ def test_bad_type_content_length(self): }, StringIO('x'), [], 'utf-8') self.assertEqual(multipart_parser._content_length, 0) + def test_sanitize_file_name(self): + parser = MultiPartParser({ + 'CONTENT_TYPE': 'multipart/form-data; boundary=_foo', + 'CONTENT_LENGTH': '1' + }, StringIO('x'), [], 'utf-8') + for file_name in CANDIDATE_TRAVERSAL_FILE_NAMES: + with self.subTest(file_name=file_name): + self.assertEqual(parser.sanitize_file_name(file_name), 'hax0rd.txt') + def test_rfc2231_parsing(self): test_data = ( (b"Content-Type: application/x-stuff; title*=us-ascii'en-us'This%20is%20%2A%2A%2Afun%2A%2A%2A", diff --git a/tests/file_uploads/uploadhandler.py b/tests/file_uploads/uploadhandler.py index 7c6199fd16d3..65d70c648c44 100644 --- a/tests/file_uploads/uploadhandler.py +++ b/tests/file_uploads/uploadhandler.py @@ -1,6 +1,8 @@ """ Upload handlers to test the upload API. """ +import os +from tempfile import NamedTemporaryFile from django.core.files.uploadhandler import FileUploadHandler, StopUpload @@ -35,3 +37,32 @@ class ErroringUploadHandler(FileUploadHandler): """A handler that raises an exception.""" def receive_data_chunk(self, raw_data, start): raise CustomUploadError("Oops!") + + +class TraversalUploadHandler(FileUploadHandler): + """A handler with potential directory-traversal vulnerability.""" + def __init__(self, request=None): + from .views import UPLOAD_TO + + super().__init__(request) + self.upload_dir = UPLOAD_TO + + def file_complete(self, file_size): + self.file.seek(0) + self.file.size = file_size + with open(os.path.join(self.upload_dir, self.file_name), 'wb') as fp: + fp.write(self.file.read()) + return self.file + + def new_file( + self, field_name, file_name, content_type, content_length, charset=None, + content_type_extra=None, + ): + super().new_file( + file_name, file_name, content_length, content_length, charset, + content_type_extra, + ) + self.file = NamedTemporaryFile(suffix='.upload', dir=self.upload_dir) + + def receive_data_chunk(self, raw_data, start): + self.file.write(raw_data) diff --git a/tests/file_uploads/urls.py b/tests/file_uploads/urls.py index 3e7985d2f9db..eaac1dae3d4b 100644 --- a/tests/file_uploads/urls.py +++ b/tests/file_uploads/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path('upload/', views.file_upload_view), + path('upload_traversal/', views.file_upload_traversal_view), path('verify/', views.file_upload_view_verify), path('unicode_name/', views.file_upload_unicode_name), path('echo/', views.file_upload_echo), diff --git a/tests/file_uploads/views.py b/tests/file_uploads/views.py index 36c9fc12a2ba..06c47d18dd7f 100644 --- a/tests/file_uploads/views.py +++ b/tests/file_uploads/views.py @@ -6,7 +6,9 @@ from .models import FileModel from .tests import UNICODE_FILENAME, UPLOAD_TO -from .uploadhandler import ErroringUploadHandler, QuotaUploadHandler +from .uploadhandler import ( + ErroringUploadHandler, QuotaUploadHandler, TraversalUploadHandler, +) def file_upload_view(request): @@ -141,3 +143,11 @@ def file_upload_fd_closing(request, access): if access == 't': request.FILES # Trigger file parsing. return HttpResponse() + + +def file_upload_traversal_view(request): + request.upload_handlers.insert(0, TraversalUploadHandler()) + request.FILES # Trigger file parsing. + return JsonResponse( + {'file_name': request.upload_handlers[0].file_name}, + ) diff --git a/tests/files/tests.py b/tests/files/tests.py index 1c005dde5779..d90b79efe353 100644 --- a/tests/files/tests.py +++ b/tests/files/tests.py @@ -17,9 +17,11 @@ ) try: - from PIL import Image + from PIL import Image, features + HAS_WEBP = features.check('webp') except ImportError: Image = None + HAS_WEBP = False else: from django.core.files import images @@ -343,6 +345,7 @@ def test_valid_image(self): size = images.get_image_dimensions(fh) self.assertEqual(size, (None, None)) + @unittest.skipUnless(HAS_WEBP, 'WEBP not installed') def test_webp(self): img_path = os.path.join(os.path.dirname(__file__), 'test.webp') with open(img_path, 'rb') as fh: diff --git a/tests/forms_tests/field_tests/test_choicefield.py b/tests/forms_tests/field_tests/test_choicefield.py index 465cfd83a862..c27d19210855 100644 --- a/tests/forms_tests/field_tests/test_choicefield.py +++ b/tests/forms_tests/field_tests/test_choicefield.py @@ -1,3 +1,4 @@ +from django.db import models from django.forms import ChoiceField, Form, ValidationError from django.test import SimpleTestCase @@ -86,3 +87,14 @@ def test_choicefield_disabled(self): '' ) + + def test_choicefield_enumeration(self): + class FirstNames(models.TextChoices): + JOHN = 'J', 'John' + PAUL = 'P', 'Paul' + + f = ChoiceField(choices=FirstNames.choices) + self.assertEqual(f.clean('J'), 'J') + msg = "'Select a valid choice. 3 is not one of the available choices.'" + with self.assertRaisesMessage(ValidationError, msg): + f.clean('3') diff --git a/tests/gis_tests/test_geoip2.py b/tests/gis_tests/test_geoip2.py index 294d875ac810..91bba2e8ff81 100644 --- a/tests/gis_tests/test_geoip2.py +++ b/tests/gis_tests/test_geoip2.py @@ -20,8 +20,8 @@ "GeoIP is required along with the GEOIP_PATH setting." ) class GeoIPTest(SimpleTestCase): - addr = '75.41.39.1' - fqdn = 'tmc.edu' + addr = '129.237.192.1' + fqdn = 'ku.edu' def test01_init(self): "GeoIP initialization." @@ -105,7 +105,7 @@ def test03_country(self, gethostbyname): @mock.patch('socket.gethostbyname') def test04_city(self, gethostbyname): "GeoIP city querying methods." - gethostbyname.return_value = '75.41.39.1' + gethostbyname.return_value = '129.237.192.1' g = GeoIP2(country='') for query in (self.fqdn, self.addr): @@ -130,8 +130,8 @@ def test04_city(self, gethostbyname): self.assertEqual('NA', d['continent_code']) self.assertEqual('North America', d['continent_name']) self.assertEqual('US', d['country_code']) - self.assertEqual('Dallas', d['city']) - self.assertEqual('TX', d['region']) + self.assertEqual('Lawrence', d['city']) + self.assertEqual('KS', d['region']) self.assertEqual('America/Chicago', d['time_zone']) self.assertFalse(d['is_in_european_union']) geom = g.geos(query) diff --git a/tests/handlers/test_exception.py b/tests/handlers/test_exception.py index 7afd4acc6b08..0c1e76399045 100644 --- a/tests/handlers/test_exception.py +++ b/tests/handlers/test_exception.py @@ -6,7 +6,7 @@ class ExceptionHandlerTests(SimpleTestCase): def get_suspicious_environ(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') return { 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/i18n/commands/code.sample b/tests/i18n/commands/code.sample index a5f1520ecba5..2c305a3a1dcf 100644 --- a/tests/i18n/commands/code.sample +++ b/tests/i18n/commands/code.sample @@ -1,4 +1,4 @@ from django.utils.translation import gettext -# This will generate an xgettext warning -my_string = gettext("This string contain two placeholders: %s and %s" % ('a', 'b')) +# This will generate an xgettext "Empty msgid" warning. +my_string = gettext('') diff --git a/tests/mail/tests.py b/tests/mail/tests.py index 6de819965af6..accdba5e4a13 100644 --- a/tests/mail/tests.py +++ b/tests/mail/tests.py @@ -722,14 +722,14 @@ def test_sanitize_address(self): ( ('A name', 'to@example.com'), 'utf-8', - '=?utf-8?q?A_name?= ', + 'A name ', ), ('localpartonly', 'ascii', 'localpartonly'), # ASCII addresses with display names. ('A name ', 'ascii', 'A name '), - ('A name ', 'utf-8', '=?utf-8?q?A_name?= '), + ('A name ', 'utf-8', 'A name '), ('"A name" ', 'ascii', 'A name '), - ('"A name" ', 'utf-8', '=?utf-8?q?A_name?= '), + ('"A name" ', 'utf-8', 'A name '), # Unicode addresses (supported per RFC-6532). ('tó@example.com', 'utf-8', '=?utf-8?b?dMOz?=@example.com'), ('to@éxample.com', 'utf-8', 'to@xn--xample-9ua.com'), @@ -748,20 +748,45 @@ def test_sanitize_address(self): ( 'To Example ', 'utf-8', - '=?utf-8?q?To_Example?= ', + 'To Example ', ), # Addresses with two @ signs. ('"to@other.com"@example.com', 'utf-8', r'"to@other.com"@example.com'), ( '"to@other.com" ', 'utf-8', - '=?utf-8?q?to=40other=2Ecom?= ', + '"to@other.com" ', ), ( ('To Example', 'to@other.com@example.com'), 'utf-8', - '=?utf-8?q?To_Example?= <"to@other.com"@example.com>', + 'To Example <"to@other.com"@example.com>', + ), + # Addresses with long unicode display names. + ( + 'Tó Example very long' * 4 + ' ', + 'utf-8', + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT' + '=C3=B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + '', + ), + ( + ('Tó Example very long' * 4, 'to@example.com'), + 'utf-8', + '=?utf-8?q?T=C3=B3_Example_very_longT=C3=B3_Example_very_longT' + '=C3=B3_Example_?=\n' + ' =?utf-8?q?very_longT=C3=B3_Example_very_long?= ' + '', ), + # Address with long display name and unicode domain. + ( + ('To Example very long' * 4, 'to@exampl€.com'), + 'utf-8', + 'To Example very longTo Example very longTo Example very longT' + 'o Example very\n' + ' long ' + ) ): with self.subTest(email_address=email_address, encoding=encoding): self.assertEqual(sanitize_address(email_address, encoding), expected_result) @@ -781,6 +806,19 @@ def test_sanitize_address_invalid(self): with self.assertRaises(ValueError): sanitize_address(email_address, encoding='utf-8') + def test_sanitize_address_header_injection(self): + msg = 'Invalid address; address parts cannot contain newlines.' + tests = [ + 'Name\nInjection ', + ('Name\nInjection', 'to@xample.com'), + 'Name ', + ('Name', 'to\ninjection@example.com'), + ] + for email_address in tests: + with self.subTest(email_address=email_address): + with self.assertRaisesMessage(ValueError, msg): + sanitize_address(email_address, encoding='utf-8') + @requires_tz_support class MailTimeZoneTests(SimpleTestCase): diff --git a/tests/messages_tests/test_cookie.py b/tests/messages_tests/test_cookie.py index 211d33f04c54..7456e03a70fc 100644 --- a/tests/messages_tests/test_cookie.py +++ b/tests/messages_tests/test_cookie.py @@ -1,5 +1,6 @@ import json +from django.conf import settings from django.contrib.messages import constants from django.contrib.messages.storage.base import Message from django.contrib.messages.storage.cookie import ( @@ -85,6 +86,10 @@ def test_cookie_setings(self): self.assertEqual(response.cookies['messages'].value, '') self.assertEqual(response.cookies['messages']['domain'], '.example.com') self.assertEqual(response.cookies['messages']['expires'], 'Thu, 01 Jan 1970 00:00:00 GMT') + self.assertEqual( + response.cookies['messages']['samesite'], + settings.SESSION_COOKIE_SAMESITE, + ) def test_get_bad_cookie(self): request = self.get_request() diff --git a/tests/model_regress/tests.py b/tests/model_regress/tests.py index 28eed87008a4..87df240d8153 100644 --- a/tests/model_regress/tests.py +++ b/tests/model_regress/tests.py @@ -1,3 +1,4 @@ +import copy import datetime from operator import attrgetter @@ -256,3 +257,17 @@ def test_model_with_evaluate_method(self): dept = Department.objects.create(pk=1, name='abc') dept.evaluate = 'abc' Worker.objects.filter(department=dept) + + +class ModelFieldsCacheTest(TestCase): + def test_fields_cache_reset_on_copy(self): + department1 = Department.objects.create(id=1, name='department1') + department2 = Department.objects.create(id=2, name='department2') + worker1 = Worker.objects.create(name='worker', department=department1) + worker2 = copy.copy(worker1) + + self.assertEqual(worker2.department, department1) + # Changing related fields doesn't mutate the base object. + worker2.department = department2 + self.assertEqual(worker2.department, department2) + self.assertEqual(worker1.department, department1) diff --git a/tests/queries/models.py b/tests/queries/models.py index e9eec5718dde..6bed09c9bfdd 100644 --- a/tests/queries/models.py +++ b/tests/queries/models.py @@ -77,6 +77,7 @@ class ExtraInfo(models.Model): note = models.ForeignKey(Note, models.CASCADE, null=True) value = models.IntegerField(null=True) date = models.ForeignKey(DateTimePK, models.SET_NULL, null=True) + filterable = models.BooleanField(default=True) class Meta: ordering = ['info'] diff --git a/tests/queries/tests.py b/tests/queries/tests.py index 081fb89d5466..83a2875f2aec 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -53,12 +53,12 @@ def setUpTestData(cls): # Create these out of order so that sorting by 'id' will be different to sorting # by 'info'. Helps detect some problems later. - cls.e2 = ExtraInfo.objects.create(info='e2', note=cls.n2, value=41) + cls.e2 = ExtraInfo.objects.create(info='e2', note=cls.n2, value=41, filterable=False) e1 = ExtraInfo.objects.create(info='e1', note=cls.n1, value=42) cls.a1 = Author.objects.create(name='a1', num=1001, extra=e1) cls.a2 = Author.objects.create(name='a2', num=2002, extra=e1) - a3 = Author.objects.create(name='a3', num=3003, extra=cls.e2) + cls.a3 = Author.objects.create(name='a3', num=3003, extra=cls.e2) cls.a4 = Author.objects.create(name='a4', num=4004, extra=cls.e2) cls.time1 = datetime.datetime(2007, 12, 19, 22, 25, 0) @@ -74,7 +74,7 @@ def setUpTestData(cls): i4.tags.set([t4]) cls.r1 = Report.objects.create(name='r1', creator=cls.a1) - Report.objects.create(name='r2', creator=a3) + Report.objects.create(name='r2', creator=cls.a3) Report.objects.create(name='r3') # Ordering by 'rank' gives us rank2, rank1, rank3. Ordering by the Meta.ordering @@ -1185,6 +1185,12 @@ def test_excluded_intermediary_m2m_table_joined(self): [], ) + def test_field_with_filterable(self): + self.assertSequenceEqual( + Author.objects.filter(extra=self.e2), + [self.a3, self.a4], + ) + class Queries2Tests(TestCase): @classmethod diff --git a/tests/requests/test_data_upload_settings.py b/tests/requests/test_data_upload_settings.py index f60f1850ea25..44897cc9fa97 100644 --- a/tests/requests/test_data_upload_settings.py +++ b/tests/requests/test_data_upload_settings.py @@ -11,7 +11,7 @@ class DataUploadMaxMemorySizeFormPostTests(SimpleTestCase): def setUp(self): - payload = FakePayload('a=1&a=2;a=3\r\n') + payload = FakePayload('a=1&a=2&a=3\r\n') self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', @@ -117,7 +117,7 @@ def test_get_max_fields_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -126,7 +126,7 @@ def test_get_max_fields_not_exceeded(self): request = WSGIRequest({ 'REQUEST_METHOD': 'GET', 'wsgi.input': BytesIO(b''), - 'QUERY_STRING': 'a=1&a=2;a=3', + 'QUERY_STRING': 'a=1&a=2&a=3', }) request.GET['a'] @@ -168,7 +168,7 @@ def test_no_limit(self): class DataUploadMaxNumberOfFieldsFormPost(SimpleTestCase): def setUp(self): - payload = FakePayload("\r\n".join(['a=1&a=2;a=3', ''])) + payload = FakePayload("\r\n".join(['a=1&a=2&a=3', ''])) self.request = WSGIRequest({ 'REQUEST_METHOD': 'POST', 'CONTENT_TYPE': 'application/x-www-form-urlencoded', diff --git a/tests/requirements/py3.txt b/tests/requirements/py3.txt index fd677713f880..8b8f2a6f33da 100644 --- a/tests/requirements/py3.txt +++ b/tests/requirements/py3.txt @@ -5,7 +5,7 @@ docutils geoip2 jinja2 >= 2.9.2 numpy -Pillow != 5.4.0 +Pillow >=4.2.0, != 5.4.0 # pylibmc/libmemcached can't be built on Windows. pylibmc; sys.platform != 'win32' python-memcached >= 1.59 diff --git a/tests/responses/test_cookie.py b/tests/responses/test_cookie.py index a46d910f3482..3470e9bf28be 100644 --- a/tests/responses/test_cookie.py +++ b/tests/responses/test_cookie.py @@ -102,6 +102,7 @@ def test_default(self): self.assertEqual(cookie['path'], '/') self.assertEqual(cookie['secure'], '') self.assertEqual(cookie['domain'], '') + self.assertEqual(cookie['samesite'], '') def test_delete_cookie_secure_prefix(self): """ @@ -115,3 +116,8 @@ def test_delete_cookie_secure_prefix(self): cookie_name = '__%s-c' % prefix response.delete_cookie(cookie_name) self.assertEqual(response.cookies[cookie_name]['secure'], True) + + def test_delete_cookie_samesite(self): + response = HttpResponse() + response.delete_cookie('c', samesite='lax') + self.assertEqual(response.cookies['c']['samesite'], 'lax') diff --git a/tests/schema/tests.py b/tests/schema/tests.py index 074f2521975c..9a84969447f9 100644 --- a/tests/schema/tests.py +++ b/tests/schema/tests.py @@ -2,7 +2,7 @@ import itertools import unittest from copy import copy -from unittest import mock +from unittest import mock, skipIf from django.core.management.color import no_style from django.db import ( @@ -94,8 +94,12 @@ def delete_tables(self): with connection.schema_editor() as editor: connection.disable_constraint_checking() table_names = connection.introspection.table_names() + if connection.features.ignores_table_name_case: + table_names = [table_name.lower() for table_name in table_names] for model in itertools.chain(SchemaTests.models, self.local_models): tbl = converter(model._meta.db_table) + if connection.features.ignores_table_name_case: + tbl = tbl.lower() if tbl in table_names: editor.delete_model(model) table_names.remove(tbl) @@ -694,6 +698,12 @@ class Meta: editor.alter_field(Foo, old_field, new_field, strict=True) Foo.objects.create() + @skipIf( + connection.vendor == 'mysql' and + connection.mysql_is_mariadb and + (10, 4, 3) < connection.mysql_version < (10, 5, 2), + 'https://jira.mariadb.org/browse/MDEV-19598', + ) def test_alter_not_unique_field_to_primary_key(self): # Create the table. with connection.schema_editor() as editor: @@ -2879,6 +2889,12 @@ def test_alter_field_add_index_to_integerfield(self): editor.alter_field(Author, new_field, old_field, strict=True) self.assertEqual(self.get_constraints_for_column(Author, 'weight'), []) + @skipIf( + connection.vendor == 'mysql' and + connection.mysql_is_mariadb and + (10, 4, 12) < connection.mysql_version < (10, 5), + 'https://jira.mariadb.org/browse/MDEV-22775', + ) def test_alter_pk_with_self_referential_field(self): """ Changing the primary key field name of a model with a self-referential @@ -2999,6 +3015,35 @@ class Meta: student = Student.objects.create(name='Some man') doc.students.add(student) + @isolate_apps('schema') + @unittest.skipUnless(connection.vendor == 'postgresql', 'PostgreSQL specific db_table syntax.') + def test_namespaced_db_table_foreign_key_reference(self): + with connection.cursor() as cursor: + cursor.execute('CREATE SCHEMA django_schema_tests') + + def delete_schema(): + with connection.cursor() as cursor: + cursor.execute('DROP SCHEMA django_schema_tests CASCADE') + + self.addCleanup(delete_schema) + + class Author(Model): + class Meta: + app_label = 'schema' + + class Book(Model): + class Meta: + app_label = 'schema' + db_table = '"django_schema_tests"."schema_book"' + + author = ForeignKey(Author, CASCADE) + author.set_attributes_from_name('author') + + with connection.schema_editor() as editor: + editor.create_model(Author) + editor.create_model(Book) + editor.add_field(Book, author) + def test_rename_table_renames_deferred_sql_references(self): atomic_rename = connection.features.supports_atomic_references_rename with connection.schema_editor(atomic=atomic_rename) as editor: diff --git a/tests/select_for_update/models.py b/tests/select_for_update/models.py index 305e8cac490b..0a6396bc7066 100644 --- a/tests/select_for_update/models.py +++ b/tests/select_for_update/models.py @@ -23,6 +23,20 @@ class EUCity(models.Model): country = models.ForeignKey(EUCountry, models.CASCADE) +class CountryProxy(Country): + class Meta: + proxy = True + + +class CountryProxyProxy(CountryProxy): + class Meta: + proxy = True + + +class CityCountryProxy(models.Model): + country = models.ForeignKey(CountryProxyProxy, models.CASCADE) + + class Person(models.Model): name = models.CharField(max_length=30) born = models.ForeignKey(City, models.CASCADE, related_name='+') diff --git a/tests/select_for_update/tests.py b/tests/select_for_update/tests.py index 3622a95c11a7..70511b09a123 100644 --- a/tests/select_for_update/tests.py +++ b/tests/select_for_update/tests.py @@ -15,7 +15,9 @@ ) from django.test.utils import CaptureQueriesContext -from .models import City, Country, EUCity, EUCountry, Person, PersonProfile +from .models import ( + City, CityCountryProxy, Country, EUCity, EUCountry, Person, PersonProfile, +) class SelectForUpdateTests(TransactionTestCase): @@ -195,6 +197,21 @@ def test_for_update_sql_multilevel_model_inheritance_ptr_generated_of(self): expected = [connection.ops.quote_name(value) for value in expected] self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') + def test_for_update_sql_model_proxy_generated_of(self): + with transaction.atomic(), CaptureQueriesContext(connection) as ctx: + list(CityCountryProxy.objects.select_related( + 'country', + ).select_for_update( + of=('country',), + )) + if connection.features.select_for_update_of_column: + expected = ['select_for_update_country"."entity_ptr_id'] + else: + expected = ['select_for_update_country'] + expected = [connection.ops.quote_name(value) for value in expected] + self.assertTrue(self.has_for_update_sql(ctx.captured_queries, of=expected)) + @skipUnlessDBFeature('has_select_for_update_of') def test_for_update_of_followed_by_values(self): with transaction.atomic(): @@ -353,6 +370,19 @@ def test_model_inheritance_of_argument_raises_error_ptr_in_choices(self): with transaction.atomic(): EUCountry.objects.select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') + def test_model_proxy_of_argument_raises_error_proxy_field_in_choices(self): + msg = ( + 'Invalid field name(s) given in select_for_update(of=(...)): ' + 'name. Only relational fields followed in the query are allowed. ' + 'Choices are: self, country, country__entity_ptr.' + ) + with self.assertRaisesMessage(FieldError, msg): + with transaction.atomic(): + CityCountryProxy.objects.select_related( + 'country', + ).select_for_update(of=('name',)).get() + @skipUnlessDBFeature('has_select_for_update', 'has_select_for_update_of') def test_reverse_one_to_one_of_arguments(self): """ diff --git a/tests/sessions_tests/tests.py b/tests/sessions_tests/tests.py index 24e4e0c81b5b..3254013d501a 100644 --- a/tests/sessions_tests/tests.py +++ b/tests/sessions_tests/tests.py @@ -755,8 +755,9 @@ def test_session_delete_on_end(self): # Set-Cookie: sessionid=; expires=Thu, 01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/ self.assertEqual( 'Set-Cookie: {}=""; expires=Thu, 01 Jan 1970 00:00:00 GMT; ' - 'Max-Age=0; Path=/'.format( + 'Max-Age=0; Path=/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) @@ -787,8 +788,9 @@ def test_session_delete_on_end_with_custom_domain_and_path(self): # Path=/example/ self.assertEqual( 'Set-Cookie: {}=""; Domain=.example.local; expires=Thu, ' - '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/'.format( + '01 Jan 1970 00:00:00 GMT; Max-Age=0; Path=/example/; SameSite={}'.format( settings.SESSION_COOKIE_NAME, + settings.SESSION_COOKIE_SAMESITE, ), str(response.cookies[settings.SESSION_COOKIE_NAME]) ) diff --git a/tests/staticfiles_tests/project/documents/nested/css/base.css b/tests/staticfiles_tests/project/documents/nested/css/base.css new file mode 100644 index 000000000000..06041ca25f1e --- /dev/null +++ b/tests/staticfiles_tests/project/documents/nested/css/base.css @@ -0,0 +1 @@ +html {height: 100%;} diff --git a/tests/staticfiles_tests/test_storage.py b/tests/staticfiles_tests/test_storage.py index 93bd60446a42..063fbb38e453 100644 --- a/tests/staticfiles_tests/test_storage.py +++ b/tests/staticfiles_tests/test_storage.py @@ -4,6 +4,7 @@ import tempfile import unittest from io import StringIO +from pathlib import Path from unittest import mock from django.conf import settings @@ -530,12 +531,19 @@ def run_collectstatic(self, **kwargs): ) def test_collect_static_files_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o655) - self.assertEqual(dir_mode, 0o765) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o765) @override_settings( FILE_UPLOAD_PERMISSIONS=None, @@ -543,12 +551,19 @@ def test_collect_static_files_permissions(self): ) def test_collect_static_files_default_permissions(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o666 & ~self.umask) - self.assertEqual(dir_mode, 0o777 & ~self.umask) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o777 & ~self.umask) @override_settings( FILE_UPLOAD_PERMISSIONS=0o655, @@ -557,12 +572,19 @@ def test_collect_static_files_default_permissions(self): ) def test_collect_static_files_subclass_of_static_storage(self): call_command('collectstatic', **self.command_params) - test_file = os.path.join(settings.STATIC_ROOT, "test.txt") - test_dir = os.path.join(settings.STATIC_ROOT, "subdir") - file_mode = os.stat(test_file)[0] & 0o777 - dir_mode = os.stat(test_dir)[0] & 0o777 + static_root = Path(settings.STATIC_ROOT) + test_file = static_root / 'test.txt' + file_mode = test_file.stat().st_mode & 0o777 self.assertEqual(file_mode, 0o640) - self.assertEqual(dir_mode, 0o740) + tests = [ + static_root / 'subdir', + static_root / 'nested', + static_root / 'nested' / 'css', + ] + for directory in tests: + with self.subTest(directory=directory): + dir_mode = directory.stat().st_mode & 0o777 + self.assertEqual(dir_mode, 0o740) @override_settings( diff --git a/tests/utils_tests/test_archive.py b/tests/utils_tests/test_archive.py index dfbef8ab184a..8fdf3ec4459b 100644 --- a/tests/utils_tests/test_archive.py +++ b/tests/utils_tests/test_archive.py @@ -4,6 +4,8 @@ import tempfile import unittest +from django.core.exceptions import SuspiciousOperation +from django.test import SimpleTestCase from django.utils import archive @@ -44,4 +46,23 @@ def test_extract_file_permissions(self): self.assertEqual(os.stat(filepath).st_mode & mask, 0o775) # A file is readable even if permission data is missing. filepath = os.path.join(tmpdir, 'no_permissions') - self.assertEqual(os.stat(filepath).st_mode & mask, 0o664 & ~umask) + self.assertEqual(os.stat(filepath).st_mode & mask, 0o666 & ~umask) + + +class TestArchiveInvalid(SimpleTestCase): + def test_extract_function_traversal(self): + archives_dir = os.path.join(os.path.dirname(__file__), 'traversal_archives') + tests = [ + ('traversal.tar', '..'), + ('traversal_absolute.tar', '/tmp/evil.py'), + ] + if sys.platform == 'win32': + tests += [ + ('traversal_disk_win.tar', 'd:evil.py'), + ('traversal_disk_win.zip', 'd:evil.py'), + ] + msg = "Archive contains invalid path: '%s'" + for entry, invalid_path in tests: + with self.subTest(entry), tempfile.TemporaryDirectory() as tmpdir: + with self.assertRaisesMessage(SuspiciousOperation, msg % invalid_path): + archive.extract(os.path.join(archives_dir, entry), tmpdir) diff --git a/tests/utils_tests/test_http.py b/tests/utils_tests/test_http.py index aa9f194a8a53..0ec14a681961 100644 --- a/tests/utils_tests/test_http.py +++ b/tests/utils_tests/test_http.py @@ -3,14 +3,16 @@ from datetime import datetime from unittest import mock +from django.core.exceptions import TooManyFieldsSent from django.test import SimpleTestCase, ignore_warnings from django.utils.datastructures import MultiValueDict from django.utils.deprecation import RemovedInDjango40Warning from django.utils.http import ( base36_to_int, escape_leading_slashes, http_date, int_to_base36, - is_safe_url, is_same_domain, parse_etags, parse_http_date, quote_etag, - url_has_allowed_host_and_scheme, urlencode, urlquote, urlquote_plus, - urlsafe_base64_decode, urlsafe_base64_encode, urlunquote, urlunquote_plus, + is_safe_url, is_same_domain, limited_parse_qsl, parse_etags, + parse_http_date, quote_etag, url_has_allowed_host_and_scheme, urlencode, + urlquote, urlquote_plus, urlsafe_base64_decode, urlsafe_base64_encode, + urlunquote, urlunquote_plus, ) @@ -359,3 +361,47 @@ def test(self): for url, expected in tests: with self.subTest(url=url): self.assertEqual(escape_leading_slashes(url), expected) + + +# Backport of unit tests for urllib.parse.parse_qsl() from Python 3.8.8. +# Copyright (C) 2021 Python Software Foundation (see LICENSE.python). +class ParseQSLBackportTests(unittest.TestCase): + def test_parse_qsl(self): + tests = [ + ('', []), + ('&', []), + ('&&', []), + ('=', [('', '')]), + ('=a', [('', 'a')]), + ('a', [('a', '')]), + ('a=', [('a', '')]), + ('&a=b', [('a', 'b')]), + ('a=a+b&b=b+c', [('a', 'a b'), ('b', 'b c')]), + ('a=1&a=2', [('a', '1'), ('a', '2')]), + (';a=b', [(';a', 'b')]), + ('a=a+b;b=b+c', [('a', 'a b;b=b c')]), + ] + for original, expected in tests: + with self.subTest(original): + result = limited_parse_qsl(original, keep_blank_values=True) + self.assertEqual(result, expected, 'Error parsing %r' % original) + expect_without_blanks = [v for v in expected if len(v[1])] + result = limited_parse_qsl(original, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks, 'Error parsing %r' % original) + + def test_parse_qsl_encoding(self): + result = limited_parse_qsl('key=\u0141%E9', encoding='latin-1') + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = limited_parse_qsl('key=\u0141%C3%A9', encoding='utf-8') + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = limited_parse_qsl('key=\u0141%C3%A9', encoding='ascii') + self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')]) + result = limited_parse_qsl('key=\u0141%E9-', encoding='ascii') + self.assertEqual(result, [('key', '\u0141\ufffd-')]) + result = limited_parse_qsl('key=\u0141%E9-', encoding='ascii', errors='ignore') + self.assertEqual(result, [('key', '\u0141-')]) + + def test_parse_qsl_field_limit(self): + with self.assertRaises(TooManyFieldsSent): + limited_parse_qsl('&'.join(['a=a'] * 11), fields_limit=10) + limited_parse_qsl('&'.join(['a=a'] * 10), fields_limit=10) diff --git a/tests/utils_tests/traversal_archives/traversal.tar b/tests/utils_tests/traversal_archives/traversal.tar new file mode 100644 index 000000000000..07eede517a56 Binary files /dev/null and b/tests/utils_tests/traversal_archives/traversal.tar differ diff --git a/tests/utils_tests/traversal_archives/traversal_absolute.tar b/tests/utils_tests/traversal_archives/traversal_absolute.tar new file mode 100644 index 000000000000..231566b0699d Binary files /dev/null and b/tests/utils_tests/traversal_archives/traversal_absolute.tar differ diff --git a/tests/utils_tests/traversal_archives/traversal_disk_win.tar b/tests/utils_tests/traversal_archives/traversal_disk_win.tar new file mode 100644 index 000000000000..97f0b95501c1 Binary files /dev/null and b/tests/utils_tests/traversal_archives/traversal_disk_win.tar differ diff --git a/tests/utils_tests/traversal_archives/traversal_disk_win.zip b/tests/utils_tests/traversal_archives/traversal_disk_win.zip new file mode 100644 index 000000000000..e5ab2083985e Binary files /dev/null and b/tests/utils_tests/traversal_archives/traversal_disk_win.zip differ diff --git a/tox.ini b/tox.ini index 9e0bdf7accd6..15617d39c515 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ passenv = DJANGO_SETTINGS_MODULE PYTHONPATH HOME DISPLAY setenv = PYTHONDONTWRITEBYTECODE=1 deps = - py{3,36,37,38}: -rtests/requirements/py3.txt + py{3,36,37,38,39}: -rtests/requirements/py3.txt postgres: -rtests/requirements/postgres.txt mysql: -rtests/requirements/mysql.txt oracle: -rtests/requirements/oracle.txt