From 4d1fce01251f9d1e52ecb3d7fd05c789854d5322 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:50:04 -0400 Subject: [PATCH 01/25] [pre-commit.ci] pre-commit autoupdate (#1973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.7 → v0.4.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.7...v0.4.8) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 488a6b6a8..8a0423407 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.4.8 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5c0738e0f55628095e0da47dd176502dd1b0c077 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:39:39 -0400 Subject: [PATCH 02/25] [pre-commit.ci] pre-commit autoupdate (#1974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.8 → v0.4.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.8...v0.4.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a0423407..2af25d37b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.4.9 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 161a09137d54efbfab03b6a281b911aac15bc250 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:01:47 +0200 Subject: [PATCH 03/25] [pre-commit.ci] pre-commit autoupdate (#1975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.9 → v0.4.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.9...v0.4.10) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2af25d37b..4972ed0ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 + rev: v0.4.10 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 2fd1360566f1c9c8c40f2e6020a9c4830001e601 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 29 Jun 2024 14:14:42 +0200 Subject: [PATCH 04/25] Update to modern ruff syntax --- .github/workflows/test.yml | 2 +- haystack/views.py | 1 - pyproject.toml | 15 ++++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 257b6a42b..4dec7412f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: pip install --user ruff - - run: ruff --output-format=github + - run: ruff check --output-format=github test: runs-on: ubuntu-latest diff --git a/haystack/views.py b/haystack/views.py index fed1808ea..f203f5e3a 100644 --- a/haystack/views.py +++ b/haystack/views.py @@ -11,7 +11,6 @@ class SearchView: template = "search/search.html" - extra_context = {} query = "" results = EmptySearchQuerySet() request = None diff --git a/pyproject.toml b/pyproject.toml index da82ce895..5962dae5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,20 +83,21 @@ profile = "black" multi_line_output = 3 [tool.ruff] -exclude = ["test_haystack"] -ignore = ["B018", "B028", "B904", "B905"] +extend-exclude = ["test_haystack/*"] line-length = 162 -select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] -show-source = true target-version = "py38" -[tool.ruff.isort] +[tool.ruff.lint] +ignore = ["B018", "B028", "B904", "B905"] +select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] + +[tool.ruff.lint.isort] known-first-party = ["haystack", "test_haystack"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 14 -[tool.ruff.pylint] +[tool.ruff.lint.pylint] max-args = 20 max-branches = 40 max-returns = 8 From 1189b8d4ee5b0dea37cca6c0fd0ce381e006244b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 29 Jun 2024 14:35:32 +0200 Subject: [PATCH 05/25] Remove obsolete ElasticSearch2 support and tests --- docs/installing_search_engines.rst | 2 +- docs/tutorial.rst | 20 - haystack/backends/elasticsearch2_backend.py | 384 ---- .../elasticsearch2_tests/__init__.py | 35 - .../elasticsearch2_tests/test_backend.py | 1820 ----------------- .../elasticsearch2_tests/test_inputs.py | 85 - .../elasticsearch2_tests/test_query.py | 247 --- test_haystack/settings.py | 8 +- 8 files changed, 2 insertions(+), 2599 deletions(-) delete mode 100644 haystack/backends/elasticsearch2_backend.py delete mode 100644 test_haystack/elasticsearch2_tests/__init__.py delete mode 100644 test_haystack/elasticsearch2_tests/test_backend.py delete mode 100644 test_haystack/elasticsearch2_tests/test_inputs.py delete mode 100644 test_haystack/elasticsearch2_tests/test_query.py diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 50bd6fb06..8b4157dcb 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -153,7 +153,7 @@ Elasticsearch is similar to Solr — another Java application using Lucene — b focused on ease of deployment and clustering. See https://www.elastic.co/products/elasticsearch for more information. -Haystack currently supports Elasticsearch 1.x, 2.x, 5.x, and 7.x. +Haystack currently supports Elasticsearch 5.x and 7.x. Follow the instructions on https://www.elastic.co/downloads/elasticsearch to download and install Elasticsearch and configure it for your environment. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index d3228beea..d8886c58a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -141,26 +141,6 @@ Example (Solr 6.X):: Elasticsearch ~~~~~~~~~~~~~ -Example (ElasticSearch 1.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - -Example (ElasticSearch 2.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - Example (ElasticSearch 5.x):: HAYSTACK_CONNECTIONS = { diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py deleted file mode 100644 index ce744107f..000000000 --- a/haystack/backends/elasticsearch2_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -import datetime -import warnings - -from django.conf import settings - -from haystack.backends import BaseEngine -from haystack.backends.elasticsearch_backend import ( - ElasticsearchSearchBackend, - ElasticsearchSearchQuery, -) -from haystack.constants import DJANGO_CT -from haystack.exceptions import MissingDependency -from haystack.utils import get_identifier, get_model_ct - -try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch.helpers import bulk, scan - - warnings.warn( - "ElasticSearch 2.x support deprecated, will be removed in 4.0", - DeprecationWarning, - ) -except ImportError: - raise MissingDependency( - "The 'elasticsearch2' backend requires the \ - installation of 'elasticsearch>=2.0.0,<3.0.0'. \ - Please refer to the documentation." - ) - - -class Elasticsearch2SearchBackend(ElasticsearchSearchBackend): - def __init__(self, connection_alias, **connection_options): - super().__init__(connection_alias, **connection_options) - self.content_field_name = None - - def clear(self, models=None, commit=True): - """ - Clears the backend of all documents/objects for a collection of models. - - :param models: List or tuple of models to clear. - :param commit: Not used. - """ - if models is not None: - assert isinstance(models, (list, tuple)) - - try: - if models is None: - self.conn.indices.delete(index=self.index_name, ignore=404) - self.setup_complete = False - self.existing_mapping = {} - self.content_field_name = None - else: - models_to_delete = [] - - for model in models: - models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - - # Delete using scroll API - query = { - "query": {"query_string": {"query": " OR ".join(models_to_delete)}} - } - generator = scan( - self.conn, - query=query, - index=self.index_name, - **self._get_doc_type_option(), - ) - actions = ( - {"_op_type": "delete", "_id": doc["_id"]} for doc in generator - ) - bulk( - self.conn, - actions=actions, - index=self.index_name, - **self._get_doc_type_option(), - ) - self.conn.indices.refresh(index=self.index_name) - - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - if models is not None: - self.log.exception( - "Failed to clear Elasticsearch index of models '%s'", - ",".join(models_to_delete), - ) - else: - self.log.exception("Failed to clear Elasticsearch index") - - def build_search_kwargs( - self, - query_string, - sort_by=None, - start_offset=0, - end_offset=None, - fields="", - highlight=False, - facets=None, - date_facets=None, - query_facets=None, - narrow_queries=None, - spelling_query=None, - within=None, - dwithin=None, - distance_point=None, - models=None, - limit_to_registered_models=None, - result_class=None, - ): - kwargs = super().build_search_kwargs( - query_string, - sort_by, - start_offset, - end_offset, - fields, - highlight, - spelling_query=spelling_query, - within=within, - dwithin=dwithin, - distance_point=distance_point, - models=models, - limit_to_registered_models=limit_to_registered_models, - result_class=result_class, - ) - - filters = [] - if start_offset is not None: - kwargs["from"] = start_offset - - if end_offset is not None: - kwargs["size"] = end_offset - start_offset - - if narrow_queries is None: - narrow_queries = set() - - if facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, extra_options in facets.items(): - facet_options = { - "meta": {"_type": "terms"}, - "terms": {"field": facet_fieldname}, - } - if "order" in extra_options: - facet_options["meta"]["order"] = extra_options.pop("order") - # Special cases for options applied at the facet level (not the terms level). - if extra_options.pop("global_scope", False): - # Renamed "global_scope" since "global" is a python keyword. - facet_options["global"] = True - if "facet_filter" in extra_options: - facet_options["facet_filter"] = extra_options.pop("facet_filter") - facet_options["terms"].update(extra_options) - kwargs["aggs"][facet_fieldname] = facet_options - - if date_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in date_facets.items(): - # Need to detect on gap_by & only add amount if it's more than one. - interval = value.get("gap_by").lower() - - # Need to detect on amount (can't be applied on months or years). - if value.get("gap_amount", 1) != 1 and interval not in ( - "month", - "year", - ): - # Just the first character is valid for use. - interval = "%s%s" % (value["gap_amount"], interval[:1]) - - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "date_histogram"}, - "date_histogram": {"field": facet_fieldname, "interval": interval}, - "aggs": { - facet_fieldname: { - "date_range": { - "field": facet_fieldname, - "ranges": [ - { - "from": self._from_python( - value.get("start_date") - ), - "to": self._from_python(value.get("end_date")), - } - ], - } - } - }, - } - - if query_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in query_facets: - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "query"}, - "filter": {"query_string": {"query": value}}, - } - - for q in narrow_queries: - filters.append({"query_string": {"query": q}}) - - # if we want to filter, change the query type to filteres - if filters: - kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}} - filtered = kwargs["query"]["filtered"] - if "filter" in filtered: - if "bool" in filtered["filter"].keys(): - another_filters = kwargs["query"]["filtered"]["filter"]["bool"][ - "must" - ] - else: - another_filters = [kwargs["query"]["filtered"]["filter"]] - else: - another_filters = filters - - if len(another_filters) == 1: - kwargs["query"]["filtered"]["filter"] = another_filters[0] - else: - kwargs["query"]["filtered"]["filter"] = { - "bool": {"must": another_filters} - } - - return kwargs - - def more_like_this( - self, - model_instance, - additional_query_string=None, - start_offset=0, - end_offset=None, - models=None, - limit_to_registered_models=None, - result_class=None, - **kwargs - ): - from haystack import connections - - if not self.setup_complete: - self.setup() - - # Deferred models will have a different class ("RealClass_Deferred_fieldname") - # which won't be in our registry: - model_klass = model_instance._meta.concrete_model - - index = ( - connections[self.connection_alias] - .get_unified_index() - .get_index(model_klass) - ) - field_name = index.get_content_field() - params = {} - - if start_offset is not None: - params["from_"] = start_offset - - if end_offset is not None: - params["size"] = end_offset - start_offset - - doc_id = get_identifier(model_instance) - - try: - # More like this Query - # https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html - mlt_query = { - "query": { - "more_like_this": { - "fields": [field_name], - "like": [{"_id": doc_id}], - } - } - } - - narrow_queries = [] - - if additional_query_string and additional_query_string != "*:*": - additional_filter = { - "query": {"query_string": {"query": additional_query_string}} - } - narrow_queries.append(additional_filter) - - if limit_to_registered_models is None: - limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - - if models and len(models): - model_choices = sorted(get_model_ct(model) for model in models) - elif limit_to_registered_models: - # Using narrow queries, limit the results to only models handled - # with the current routers. - model_choices = self.build_models_list() - else: - model_choices = [] - - if len(model_choices) > 0: - model_filter = {"terms": {DJANGO_CT: model_choices}} - narrow_queries.append(model_filter) - - if len(narrow_queries) > 0: - mlt_query = { - "query": { - "filtered": { - "query": mlt_query["query"], - "filter": {"bool": {"must": list(narrow_queries)}}, - } - } - } - - raw_results = self.conn.search( - body=mlt_query, - index=self.index_name, - _source=True, - **self._get_doc_type_option(), - **params, - ) - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - self.log.exception( - "Failed to fetch More Like This from Elasticsearch for document '%s'", - doc_id, - ) - raw_results = {} - - return self._process_results(raw_results, result_class=result_class) - - def _process_results( - self, - raw_results, - highlight=False, - result_class=None, - distance_point=None, - geo_sort=False, - ): - results = super()._process_results( - raw_results, highlight, result_class, distance_point, geo_sort - ) - facets = {} - if "aggregations" in raw_results: - facets = {"fields": {}, "dates": {}, "queries": {}} - - for facet_fieldname, facet_info in raw_results["aggregations"].items(): - facet_type = facet_info["meta"]["_type"] - if facet_type == "terms": - facets["fields"][facet_fieldname] = [ - (individual["key"], individual["doc_count"]) - for individual in facet_info["buckets"] - ] - if "order" in facet_info["meta"]: - if facet_info["meta"]["order"] == "reverse_count": - srt = sorted( - facets["fields"][facet_fieldname], key=lambda x: x[1] - ) - facets["fields"][facet_fieldname] = srt - elif facet_type == "date_histogram": - # Elasticsearch provides UTC timestamps with an extra three - # decimals of precision, which datetime barfs on. - facets["dates"][facet_fieldname] = [ - ( - datetime.datetime.utcfromtimestamp( - individual["key"] / 1000 - ), - individual["doc_count"], - ) - for individual in facet_info["buckets"] - ] - elif facet_type == "query": - facets["queries"][facet_fieldname] = facet_info["doc_count"] - results["facets"] = facets - return results - - -class Elasticsearch2SearchQuery(ElasticsearchSearchQuery): - pass - - -class Elasticsearch2SearchEngine(BaseEngine): - backend = Elasticsearch2SearchBackend - query = Elasticsearch2SearchQuery diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py deleted file mode 100644 index 38fa24fbc..000000000 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import unittest - -from django.conf import settings - -from haystack.utils import log as logging - - -def load_tests(loader, standard_tests, pattern): - log = logging.getLogger("haystack") - try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch import Elasticsearch, exceptions - except ImportError: - log.error( - "Skipping ElasticSearch 2 tests: 'elasticsearch>=2.0.0,<3.0.0' not installed." - ) - raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") - - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - es = Elasticsearch(url) - try: - es.info() - except exceptions.ConnectionError as e: - log.error("elasticsearch not running on %r" % url, exc_info=True) - raise unittest.SkipTest("elasticsearch not running on %r" % url, e) - - package_tests = loader.discover( - start_dir=os.path.dirname(__file__), pattern=pattern - ) - standard_tests.addTests(package_tests) - return standard_tests diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py deleted file mode 100644 index 0ec9608b0..000000000 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ /dev/null @@ -1,1820 +0,0 @@ -import datetime -import logging as std_logging -import operator -import pickle -import unittest -from decimal import Decimal - -import elasticsearch -from django.apps import apps -from django.conf import settings -from django.test import TestCase -from django.test.utils import override_settings - -from haystack import connections, indexes, reset_search_queries -from haystack.exceptions import SkipDocument -from haystack.inputs import AutoQuery -from haystack.models import SearchResult -from haystack.query import SQ, RelatedSearchQuerySet, SearchQuerySet -from haystack.utils import log as logging -from haystack.utils.loading import UnifiedIndex - -from ..core.models import AFourthMockModel, AnotherMockModel, ASixthMockModel, MockModel -from ..mocks import MockSearchResult - - -def clear_elasticsearch_index(): - # Wipe it clean. - raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - try: - raw_es.indices.delete( - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - raw_es.indices.refresh() - except elasticsearch.TransportError: - pass - - # Since we've just completely deleted the index, we'll reset setup_complete so the next access will - # correctly define the mappings: - connections["elasticsearch"].get_backend().setup_complete = False - - -class Elasticsearch2MockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2MockSearchIndexWithSkipDocument(Elasticsearch2MockSearchIndex): - def prepare_text(self, obj): - if obj.author == "daniel3": - raise SkipDocument - return "Indexed!\n%s" % obj.id - - -class Elasticsearch2MockSpellingIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - def prepare_text(self, obj): - return obj.foo - - -class Elasticsearch2MaintainTypeMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - month = indexes.CharField(indexed=False) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def prepare_month(self, obj): - return "%02d" % obj.pub_date.month - - def get_model(self): - return MockModel - - -class Elasticsearch2MockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2AnotherMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AnotherMockModel - - def prepare_text(self, obj): - return "You might be searching for the user %s" % obj.author - - -class Elasticsearch2BoostMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField( - document=True, - use_template=True, - template_name="search/indexes/core/mockmodel_template.txt", - ) - author = indexes.CharField(model_attr="author", weight=2.0) - editor = indexes.CharField(model_attr="editor") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AFourthMockModel - - def prepare(self, obj): - data = super().prepare(obj) - - if obj.pk == 4: - data["boost"] = 5.0 - - return data - - -class Elasticsearch2FacetingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - author = indexes.CharField(model_attr="author", faceted=True) - editor = indexes.CharField(model_attr="editor", faceted=True) - pub_date = indexes.DateField(model_attr="pub_date", faceted=True) - facet_field = indexes.FacetCharField(model_attr="author") - - def prepare_text(self, obj): - return "%s %s" % (obj.author, obj.editor) - - def get_model(self): - return AFourthMockModel - - -class Elasticsearch2RoundTripSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, default="") - name = indexes.CharField() - is_active = indexes.BooleanField() - post_count = indexes.IntegerField() - average_rating = indexes.FloatField() - price = indexes.DecimalField() - pub_date = indexes.DateField() - created = indexes.DateTimeField() - tags = indexes.MultiValueField() - sites = indexes.MultiValueField() - - def get_model(self): - return MockModel - - def prepare(self, obj): - prepped = super().prepare(obj) - prepped.update( - { - "text": "This is some example text.", - "name": "Mister Pants", - "is_active": True, - "post_count": 25, - "average_rating": 3.6, - "price": Decimal("24.99"), - "pub_date": datetime.date(2009, 11, 21), - "created": datetime.datetime(2009, 11, 21, 21, 31, 00), - "tags": ["staff", "outdoor", "activist", "scientist"], - "sites": [3, 5, 1], - } - ) - return prepped - - -class Elasticsearch2ComplexFacetsMockSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(document=True, default="") - name = indexes.CharField(faceted=True) - is_active = indexes.BooleanField(faceted=True) - post_count = indexes.IntegerField() - post_count_i = indexes.FacetIntegerField(facet_for="post_count") - average_rating = indexes.FloatField(faceted=True) - pub_date = indexes.DateField(faceted=True) - created = indexes.DateTimeField(faceted=True) - sites = indexes.MultiValueField(faceted=True) - - def get_model(self): - return MockModel - - -class Elasticsearch2AutocompleteMockModelSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - text_auto = indexes.EdgeNgramField(model_attr="foo") - name_auto = indexes.EdgeNgramField(model_attr="author") - - def get_model(self): - return MockModel - - -class Elasticsearch2SpatialSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="name", document=True) - location = indexes.LocationField() - - def prepare_location(self, obj): - return "%s,%s" % (obj.lat, obj.lon) - - def get_model(self): - return ASixthMockModel - - -class TestSettings(TestCase): - def test_kwargs_are_passed_on(self): - from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend - - backend = ElasticsearchSearchBackend( - "alias", - **{ - "URL": settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], - "INDEX_NAME": "testing", - "KWARGS": {"max_retries": 42}, - } - ) - - self.assertEqual(backend.conn.transport.max_retries, 42) - - -class Elasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.smmidni = Elasticsearch2MockSearchIndexWithSkipDocument() - self.smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - self.sb.silently_fail = True - - def raw_search(self, query): - try: - return self.raw_es.search( - q="*:*", - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"], - ) - except elasticsearch.TransportError: - return {} - - def test_non_silent(self): - bad_sb = connections["elasticsearch"].backend( - "bad", - URL="http://omg.wtf.bbq:1000/", - INDEX_NAME="whatver", - SILENTLY_FAIL=False, - TIMEOUT=1, - ) - - try: - bad_sb.update(self.smmi, self.sample_objs) - self.fail() - except: - pass - - try: - bad_sb.remove("core.mockmodel.1") - self.fail() - except: - pass - - try: - bad_sb.clear() - self.fail() - except: - pass - - try: - bad_sb.search("foo") - self.fail() - except: - pass - - def test_update_no_documents(self): - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - index_name = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True - ) - self.assertEqual(sb.update(self.smmi, []), None) - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=False - ) - try: - sb.update(self.smmi, []) - self.fail() - except: - pass - - def test_update(self): - self.sb.update(self.smmi, self.sample_objs) - - # Check what Elasticsearch thinks is there. - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=lambda x: x["id"], - ), - [ - { - "django_id": "1", - "django_ct": "core.mockmodel", - "name": "daniel1", - "name_exact": "daniel1", - "text": "Indexed!\n1\n", - "pub_date": "2009-02-24T00:00:00", - "id": "core.mockmodel.1", - }, - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_update_with_SkipDocument_raised(self): - self.sb.update(self.smmidni, self.sample_objs) - - # Check what Elasticsearch thinks is there. - res = self.raw_search("*:*")["hits"] - self.assertEqual(res["total"], 2) - self.assertListEqual( - sorted([x["_source"]["id"] for x in res["hits"]]), - ["core.mockmodel.1", "core.mockmodel.2"], - ) - - def test_remove(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 2) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=operator.itemgetter("django_id"), - ), - [ - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_remove_succeeds_on_404(self): - self.sb.silently_fail = False - self.sb.remove("core.mockmodel.421") - - def test_clear(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear() - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel, MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - def test_search(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - set([result.pk for result in self.sb.search("*:*")["results"]]), - {"2", "1", "3"}, - ) - - self.assertEqual(self.sb.search("", highlight=True), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("Index", highlight=True)["hits"], 3) - self.assertEqual( - sorted( - [ - result.highlighted[0] - for result in self.sb.search("Index", highlight=True)["results"] - ] - ), - [ - "Indexed!\n1\n", - "Indexed!\n2\n", - "Indexed!\n3\n", - ], - ) - - self.assertEqual(self.sb.search("Indx")["hits"], 0) - self.assertEqual(self.sb.search("indaxed")["spelling_suggestion"], "indexed") - self.assertEqual( - self.sb.search("arf", spelling_query="indexyd")["spelling_suggestion"], - "indexed", - ) - - self.assertEqual( - self.sb.search("", facets={"name": {}}), {"hits": 0, "results": []} - ) - results = self.sb.search("Index", facets={"name": {}}) - self.assertEqual(results["hits"], 3) - self.assertSetEqual( - set(results["facets"]["fields"]["name"]), - {("daniel3", 1), ("daniel2", 1), ("daniel1", 1)}, - ) - - self.assertEqual( - self.sb.search( - "", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ), - {"hits": 0, "results": []}, - ) - results = self.sb.search( - "Index", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ) - self.assertEqual(results["hits"], 3) - self.assertEqual( - results["facets"]["dates"]["pub_date"], - [(datetime.datetime(2009, 2, 1, 0, 0), 3)], - ) - - self.assertEqual( - self.sb.search("", query_facets=[("name", "[* TO e]")]), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", query_facets=[("name", "[* TO e]")]) - self.assertEqual(results["hits"], 3) - self.assertEqual(results["facets"]["queries"], {"name": 3}) - - self.assertEqual( - self.sb.search("", narrow_queries={"name:daniel1"}), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", narrow_queries={"name:daniel1"}) - self.assertEqual(results["hits"], 1) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sb.search("index", result_class=MockSearchResult)["results"][0], - MockSearchResult, - ) - ) - - # Check the use of ``limit_to_registered_models``. - self.assertEqual( - self.sb.search("", limit_to_registered_models=False), - {"hits": 0, "results": []}, - ) - self.assertEqual( - self.sb.search("*:*", limit_to_registered_models=False)["hits"], 3 - ) - self.assertEqual( - sorted( - [ - result.pk - for result in self.sb.search( - "*:*", limit_to_registered_models=False - )["results"] - ] - ), - ["1", "2", "3"], - ) - - # Stow. - old_limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = False - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - sorted([result.pk for result in self.sb.search("*:*")["results"]]), - ["1", "2", "3"], - ) - - # Restore. - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models - - def test_spatial_search_parameters(self): - from django.contrib.gis.geos import Point - - p1 = Point(1.23, 4.56) - kwargs = self.sb.build_search_kwargs( - "*:*", - distance_point={"field": "location", "point": p1}, - sort_by=(("distance", "desc"),), - ) - - self.assertIn("sort", kwargs) - self.assertEqual(1, len(kwargs["sort"])) - geo_d = kwargs["sort"][0]["_geo_distance"] - - # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be - # in the same order as we used to create the Point(): - # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4 - - self.assertDictEqual( - geo_d, {"location": [1.23, 4.56], "unit": "km", "order": "desc"} - ) - - def test_more_like_this(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - # A functional MLT example with enough data to work is below. Rely on - # this to ensure the API is correct enough. - self.assertEqual(self.sb.more_like_this(self.sample_objs[0])["hits"], 0) - self.assertEqual( - [ - result.pk - for result in self.sb.more_like_this(self.sample_objs[0])["results"] - ], - [], - ) - - def test_build_schema(self): - old_ui = connections["elasticsearch"].get_unified_index() - - (content_field_name, mapping) = self.sb.build_schema(old_ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 4 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - }, - ) - - ui = UnifiedIndex() - ui.build(indexes=[Elasticsearch2ComplexFacetsMockSearchIndex()]) - (content_field_name, mapping) = self.sb.build_schema(ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 15 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name": {"type": "string", "analyzer": "snowball"}, - "is_active_exact": {"type": "boolean"}, - "created": {"type": "date"}, - "post_count": {"type": "long"}, - "created_exact": {"type": "date"}, - "sites_exact": {"index": "not_analyzed", "type": "string"}, - "is_active": {"type": "boolean"}, - "sites": {"type": "string", "analyzer": "snowball"}, - "post_count_i": {"type": "long"}, - "average_rating": {"type": "float"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date_exact": {"type": "date"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - "pub_date": {"type": "date"}, - "average_rating_exact": {"type": "float"}, - }, - ) - - def test_verify_type(self): - old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - ui.build(indexes=[smtmmi]) - connections["elasticsearch"]._index = ui - sb = connections["elasticsearch"].get_backend() - sb.update(smtmmi, self.sample_objs) - - self.assertEqual(sb.search("*:*")["hits"], 3) - self.assertEqual( - [result.month for result in sb.search("*:*")["results"]], ["02", "02", "02"] - ) - connections["elasticsearch"]._index = old_ui - - -class CaptureHandler(std_logging.Handler): - logs_seen = [] - - def emit(self, record): - CaptureHandler.logs_seen.append(record) - - -class FailedElasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - # Stow. - # Point the backend at a URL that doesn't exist so we can watch the - # sparks fly. - self.old_es_url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = ( - "%s/foo/" % self.old_es_url - ) - self.cap = CaptureHandler() - logging.getLogger("haystack").addHandler(self.cap) - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(config.stream) - - # Setup the rest of the bits. - self.old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = ui - self.sb = connections["elasticsearch"].get_backend() - - def tearDown(self): - # Restore. - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = self.old_es_url - connections["elasticsearch"]._index = self.old_ui - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(self.cap) - logging.getLogger("haystack").addHandler(config.stream) - - @unittest.expectedFailure - def test_all_cases(self): - # Prior to the addition of the try/except bits, these would all fail miserably. - self.assertEqual(len(CaptureHandler.logs_seen), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(len(CaptureHandler.logs_seen), 1) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 2) - - self.sb.search("search") - self.assertEqual(len(CaptureHandler.logs_seen), 3) - - self.sb.more_like_this(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 4) - - self.sb.clear([MockModel]) - self.assertEqual(len(CaptureHandler.logs_seen), 5) - - self.sb.clear() - self.assertEqual(len(CaptureHandler.logs_seen), 6) - - -class LiveElasticsearch2SearchQueryTestCase(TestCase): - fixtures = ["base_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - self.sq = connections["elasticsearch"].get_query() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_log_query(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=False): - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=True): - # Redefine it to clear out the cached results. - self.sq = connections["elasticsearch"].query(using="elasticsearch") - self.sq.add_filter(SQ(name="bar")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - - # And again, for good measure. - self.sq = connections["elasticsearch"].query("elasticsearch") - self.sq.add_filter(SQ(name="bar")) - self.sq.add_filter(SQ(text="moof")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - self.assertEqual( - connections["elasticsearch"].queries[1]["query_string"], - "(name:(bar) AND text:(moof))", - ) - - -lssqstc_all_loaded = None - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SearchQuerySetTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - self.rsqs = RelatedSearchQuerySet("elasticsearch") - - # Ugly but not constantly reindexing saves us almost 50% runtime. - global lssqstc_all_loaded - - if lssqstc_all_loaded is None: - lssqstc_all_loaded = True - - # Wipe it clean. - clear_elasticsearch_index() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_load_all(self): - sqs = self.sqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - results = sorted([int(result.pk) for result in sqs]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_values_slicing(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends - - # The values will come back as strings because Hasytack doesn't assume PKs are integers. - # We'll prepare this set once since we're going to query the same results in multiple ways: - expected_pks = [str(i) for i in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]] - - results = self.sqs.all().order_by("pub_date").values("pk") - self.assertListEqual([i["pk"] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk") - self.assertListEqual([i[0] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk", flat=True) - self.assertListEqual(results[1:11], expected_pks) - - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_count(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - self.assertEqual(sqs.count(), 23) - self.assertEqual(sqs.count(), 23) - self.assertEqual(len(sqs), 23) - self.assertEqual(sqs.count(), 23) - # Should only execute one query to count the length of the result set. - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_manual_iter(self): - results = self.sqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = set([int(result.pk) for result in results._manual_iter()]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.sqs._cache_is_full(), False) - results = self.sqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test___and__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 & sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) AND (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar") - sqs = sqs3 & sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 3) - self.assertEqual( - sqs.query.build_query(), - "(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))", - ) - - def test___or__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 | sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) OR (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar").models(MockModel) - sqs = sqs3 | sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual( - sqs.query.build_query(), - "((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))", - ) - - def test_auto_query(self): - # Ensure bits in exact matches get escaped properly as well. - # This will break horrifically if escaping isn't working. - sqs = self.sqs.auto_query('"pants:rule"') - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual( - repr(sqs.query.query_filter), '' - ) - self.assertEqual(sqs.query.build_query(), '("pants\\:rule")') - self.assertEqual(len(sqs), 0) - - # Regressions - - def test_regression_proper_start_offsets(self): - sqs = self.sqs.filter(text="index") - self.assertNotEqual(sqs.count(), 0) - - id_counts = {} - - for item in sqs: - if item.id in id_counts: - id_counts[item.id] += 1 - else: - id_counts[item.id] = 1 - - for key, value in id_counts.items(): - if value > 1: - self.fail( - "Result with id '%s' seen more than once in the results." % key - ) - - def test_regression_raw_search_breaks_slicing(self): - sqs = self.sqs.raw_search("text:index") - page_1 = [result.pk for result in sqs[0:10]] - page_2 = [result.pk for result in sqs[10:20]] - - for pk in page_2: - if pk in page_1: - self.fail( - "Result with id '%s' seen more than once in the results." % pk - ) - - # RelatedSearchQuerySet Tests - - def test_related_load_all(self): - sqs = self.rsqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_related_load_all_queryset(self): - sqs = self.rsqs.load_all().order_by("pub_date") - self.assertEqual(len(sqs._load_all_querysets), 0) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24))) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual( - set([obj.object.id for obj in sqs]), - {12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20}, - ) - self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), {21, 22, 23}) - - def test_related_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.rsqs.all() - results = set([int(result.pk) for result in sqs]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - set([int(result.pk) for result in results[20:30]]), {21, 22, 23} - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_related_manual_iter(self): - results = self.rsqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = sorted([int(result.pk) for result in results._manual_iter()]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_related_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.rsqs._cache_is_full(), False) - results = self.rsqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_quotes_regression(self): - sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E") - # Should not have empty terms. - self.assertEqual(sqs.query.build_query(), "(44\xb048'40''N 20\xb028'32''E)") - # Should not cause Elasticsearch to 500. - self.assertEqual(sqs.count(), 0) - - sqs = self.sqs.auto_query("blazing") - self.assertEqual(sqs.query.build_query(), "(blazing)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("blazing saddles") - self.assertEqual(sqs.query.build_query(), "(blazing saddles)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles') - self.assertEqual(sqs.query.build_query(), '(\\"blazing saddles)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles"') - self.assertEqual(sqs.query.build_query(), '("blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing \'saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing \'saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" ')") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" '\\\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel brooks') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" \\"brooks)') - self.assertEqual(sqs.count(), 0) - - def test_query_generation(self): - sqs = self.sqs.filter( - SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")) - ) - self.assertEqual( - sqs.query.build_query(), "((hello world) OR title:(hello world))" - ) - - def test_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - sqs = self.sqs.all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - # Custom class. - sqs = self.sqs.result_class(MockSearchResult).all() - self.assertTrue(isinstance(sqs[0], MockSearchResult)) - - # Reset to default. - sqs = self.sqs.result_class(None).all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SpellingTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSpellingIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_spelling(self): - self.assertEqual( - self.sqs.auto_query("structurd").spelling_suggestion(), "structured" - ) - self.assertEqual(self.sqs.spelling_suggestion("structurd"), "structured") - self.assertEqual( - self.sqs.auto_query("srchindex instanc").spelling_suggestion(), - "searchindex instance", - ) - self.assertEqual( - self.sqs.spelling_suggestion("srchindex instanc"), "searchindex instance" - ) - - -class LiveElasticsearch2MoreLikeThisTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_more_like_this(self): - mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1)) - results = [result.pk for result in mlt] - self.assertEqual(mlt.count(), 11) - self.assertEqual( - set(results), {"10", "5", "2", "21", "4", "6", "23", "9", "14"} - ) - self.assertEqual(len(results), 10) - - alt_mlt = self.sqs.filter(name="daniel3").more_like_this( - MockModel.objects.get(pk=2) - ) - results = [result.pk for result in alt_mlt] - self.assertEqual(alt_mlt.count(), 9) - self.assertEqual( - set(results), {"2", "16", "3", "19", "4", "17", "10", "22", "23"} - ) - self.assertEqual(len(results), 9) - - alt_mlt_with_models = self.sqs.models(MockModel).more_like_this( - MockModel.objects.get(pk=1) - ) - results = [result.pk for result in alt_mlt_with_models] - self.assertEqual(alt_mlt_with_models.count(), 10) - self.assertEqual( - set(results), {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"} - ) - self.assertEqual(len(results), 10) - - if hasattr(MockModel.objects, "defer"): - # Make sure MLT works with deferred bits. - qs = MockModel.objects.defer("foo") - self.assertEqual(qs.query.deferred_loading[1], True) - deferred = self.sqs.models(MockModel).more_like_this(qs.get(pk=1)) - self.assertEqual(deferred.count(), 10) - self.assertEqual( - {result.pk for result in deferred}, - {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"}, - ) - self.assertEqual(len([result.pk for result in deferred]), 10) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sqs.result_class(MockSearchResult).more_like_this( - MockModel.objects.get(pk=1) - )[0], - MockSearchResult, - ) - ) - - -class LiveElasticsearch2AutocompleteTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2AutocompleteMockModelSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_build_schema(self): - self.sb = connections["elasticsearch"].get_backend() - content_name, mapping = self.sb.build_schema(self.ui.all_searchfields()) - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "text_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - }, - ) - - def test_autocomplete(self): - autocomplete = self.sqs.autocomplete(text_auto="mod") - self.assertEqual(autocomplete.count(), 16) - self.assertEqual( - set([result.pk for result in autocomplete]), - { - "1", - "12", - "6", - "14", - "7", - "4", - "23", - "17", - "13", - "18", - "20", - "22", - "19", - "15", - "10", - "2", - }, - ) - self.assertTrue("mod" in autocomplete[0].text.lower()) - self.assertTrue("mod" in autocomplete[1].text.lower()) - self.assertTrue("mod" in autocomplete[2].text.lower()) - self.assertTrue("mod" in autocomplete[3].text.lower()) - self.assertTrue("mod" in autocomplete[4].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete]), 16) - - # Test multiple words. - autocomplete_2 = self.sqs.autocomplete(text_auto="your mod") - self.assertEqual(autocomplete_2.count(), 13) - self.assertEqual( - set([result.pk for result in autocomplete_2]), - {"1", "6", "2", "14", "12", "13", "10", "19", "4", "20", "23", "22", "15"}, - ) - map_results = {result.pk: result for result in autocomplete_2} - self.assertTrue("your" in map_results["1"].text.lower()) - self.assertTrue("mod" in map_results["1"].text.lower()) - self.assertTrue("your" in map_results["6"].text.lower()) - self.assertTrue("mod" in map_results["6"].text.lower()) - self.assertTrue("your" in map_results["2"].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete_2]), 13) - - # Test multiple fields. - autocomplete_3 = self.sqs.autocomplete(text_auto="Django", name_auto="dan") - self.assertEqual(autocomplete_3.count(), 4) - self.assertEqual( - set([result.pk for result in autocomplete_3]), {"12", "1", "22", "14"} - ) - self.assertEqual(len([result.pk for result in autocomplete_3]), 4) - - # Test numbers in phrases - autocomplete_4 = self.sqs.autocomplete(text_auto="Jen 867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - # Test numbers alone - autocomplete_4 = self.sqs.autocomplete(text_auto="867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - -class LiveElasticsearch2RoundTripTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.srtsi = Elasticsearch2RoundTripSearchIndex() - self.ui.build(indexes=[self.srtsi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sqs = SearchQuerySet("elasticsearch") - - # Fake indexing. - mock = MockModel() - mock.id = 1 - self.sb.update(self.srtsi, [mock]) - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_round_trip(self): - results = self.sqs.filter(id="core.mockmodel.1") - - # Sanity check. - self.assertEqual(results.count(), 1) - - # Check the individual fields. - result = results[0] - self.assertEqual(result.id, "core.mockmodel.1") - self.assertEqual(result.text, "This is some example text.") - self.assertEqual(result.name, "Mister Pants") - self.assertEqual(result.is_active, True) - self.assertEqual(result.post_count, 25) - self.assertEqual(result.average_rating, 3.6) - self.assertEqual(result.price, "24.99") - self.assertEqual(result.pub_date, datetime.date(2009, 11, 21)) - self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00)) - self.assertEqual(result.tags, ["staff", "outdoor", "activist", "scientist"]) - self.assertEqual(result.sites, [3, 5, 1]) - - -class LiveElasticsearch2PickleTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_pickling(self): - results = self.sqs.all() - - for res in results: - # Make sure the cache is full. - pass - - in_a_pickle = pickle.dumps(results) - like_a_cuke = pickle.loads(in_a_pickle) - self.assertEqual(len(like_a_cuke), len(results)) - self.assertEqual(like_a_cuke[0].id, results[0].id) - - -class Elasticsearch2BoostBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2BoostMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sample_objs = [] - - for i in range(1, 5): - mock = AFourthMockModel() - mock.id = i - - if i % 2: - mock.author = "daniel" - mock.editor = "david" - else: - mock.author = "david" - mock.editor = "daniel" - - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def raw_search(self, query): - return self.raw_es.search( - q="*:*", index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - - def test_boost(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 4) - - results = SearchQuerySet(using="elasticsearch").filter( - SQ(author="daniel") | SQ(editor="daniel") - ) - - self.assertEqual( - set([result.id for result in results]), - { - "core.afourthmockmodel.4", - "core.afourthmockmodel.3", - "core.afourthmockmodel.1", - "core.afourthmockmodel.2", - }, - ) - - def test__to_python(self): - self.assertEqual(self.sb._to_python("abc"), "abc") - self.assertEqual(self.sb._to_python("1"), 1) - self.assertEqual(self.sb._to_python("2653"), 2653) - self.assertEqual(self.sb._to_python("25.5"), 25.5) - self.assertEqual(self.sb._to_python("[1, 2, 3]"), [1, 2, 3]) - self.assertEqual( - self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {"a": 1, "c": 3, "b": 2} - ) - self.assertEqual( - self.sb._to_python("2009-05-09T16:14:00"), - datetime.datetime(2009, 5, 9, 16, 14), - ) - self.assertEqual( - self.sb._to_python("2009-05-09T00:00:00"), - datetime.datetime(2009, 5, 9, 0, 0), - ) - self.assertEqual(self.sb._to_python(None), None) - - -class RecreateIndexTestCase(TestCase): - def setUp(self): - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - - def test_recreate_index(self): - clear_elasticsearch_index() - - sb = connections["elasticsearch"].get_backend() - sb.silently_fail = True - sb.setup() - - original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) - - sb.clear() - sb.setup() - - try: - updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) - except elasticsearch.NotFoundError: - self.fail("There is no mapping after recreating the index") - - self.assertEqual( - original_mapping, - updated_mapping, - "Mapping after recreating the index differs from the original one", - ) - - -class Elasticsearch2FacetingTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2FacetingMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 10): - mock = AFourthMockModel() - mock.id = i - if i > 5: - mock.editor = "George Taylor" - else: - mock.editor = "Perry White" - if i % 2: - mock.author = "Daniel Lindsley" - else: - mock.author = "Dan Watson" - mock.pub_date = datetime.date(2013, 9, (i % 4) + 1) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_facet(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 5), ("Dan Watson", 4)] - ) - self.assertEqual( - counts["fields"]["editor"], [("Perry White", 5), ("George Taylor", 4)] - ) - counts = ( - SearchQuerySet("elasticsearch") - .filter(content="white") - .facet("facet_field", order="reverse_count") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["facet_field"], [("Dan Watson", 2), ("Daniel Lindsley", 3)] - ) - - def test_multiple_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .narrow('editor_exact:"Perry White"') - .narrow('author_exact:"Daniel Lindsley"') - .facet("author") - .facet_counts() - ) - self.assertEqual(counts["fields"]["author"], [("Daniel Lindsley", 3)]) - - def test_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .narrow('editor_exact:"Perry White"') - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 3), ("Dan Watson", 2)] - ) - self.assertEqual(counts["fields"]["editor"], [("Perry White", 5)]) - - def test_date_facet(self): - self.sb.update(self.smmi, self.sample_objs) - start = datetime.date(2013, 9, 1) - end = datetime.date(2013, 9, 30) - # Facet by day - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="day") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], - [ - (datetime.datetime(2013, 9, 1), 2), - (datetime.datetime(2013, 9, 2), 3), - (datetime.datetime(2013, 9, 3), 2), - (datetime.datetime(2013, 9, 4), 2), - ], - ) - # By month - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="month") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], [(datetime.datetime(2013, 9, 1), 9)] - ) diff --git a/test_haystack/elasticsearch2_tests/test_inputs.py b/test_haystack/elasticsearch2_tests/test_inputs.py deleted file mode 100644 index af9f8f332..000000000 --- a/test_haystack/elasticsearch2_tests/test_inputs.py +++ /dev/null @@ -1,85 +0,0 @@ -from django.test import TestCase - -from haystack import connections, inputs - - -class Elasticsearch2InputTestCase(TestCase): - def setUp(self): - super().setUp() - self.query_obj = connections["elasticsearch"].get_query() - - def test_raw_init(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {}) - self.assertEqual(raw.post_process, False) - - raw = inputs.Raw("hello OR there, :you", test="really") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {"test": "really"}) - self.assertEqual(raw.post_process, False) - - def test_raw_prepare(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.prepare(self.query_obj), "hello OR there, :you") - - def test_clean_init(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.query_string, "hello OR there, :you") - self.assertEqual(clean.post_process, True) - - def test_clean_prepare(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.prepare(self.query_obj), "hello or there, \\:you") - - def test_exact_init(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.query_string, "hello OR there, :you") - self.assertEqual(exact.post_process, True) - - def test_exact_prepare(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.prepare(self.query_obj), '"hello OR there, :you"') - - exact = inputs.Exact("hello OR there, :you", clean=True) - self.assertEqual(exact.prepare(self.query_obj), '"hello or there, \\:you"') - - def test_not_init(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.query_string, "hello OR there, :you") - self.assertEqual(not_it.post_process, True) - - def test_not_prepare(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.prepare(self.query_obj), "NOT (hello or there, \\:you)") - - def test_autoquery_init(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual(autoquery.query_string, 'panic -don\'t "froody dude"') - self.assertEqual(autoquery.post_process, False) - - def test_autoquery_prepare(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual( - autoquery.prepare(self.query_obj), 'panic NOT don\'t "froody dude"' - ) - - def test_altparser_init(self): - altparser = inputs.AltParser("dismax") - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "") - self.assertEqual(altparser.kwargs, {}) - self.assertEqual(altparser.post_process, False) - - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "douglas adams") - self.assertEqual(altparser.kwargs, {"mm": 1, "qf": "author"}) - self.assertEqual(altparser.post_process, False) - - def test_altparser_prepare(self): - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual( - altparser.prepare(self.query_obj), - """{!dismax mm=1 qf=author v='douglas adams'}""", - ) diff --git a/test_haystack/elasticsearch2_tests/test_query.py b/test_haystack/elasticsearch2_tests/test_query.py deleted file mode 100644 index 5a0111d5b..000000000 --- a/test_haystack/elasticsearch2_tests/test_query.py +++ /dev/null @@ -1,247 +0,0 @@ -import datetime - -import elasticsearch -from django.contrib.gis.measure import D -from django.test import TestCase - -from haystack import connections -from haystack.inputs import Exact -from haystack.models import SearchResult -from haystack.query import SQ, SearchQuerySet - -from ..core.models import AnotherMockModel, MockModel - - -class Elasticsearch2SearchQueryTestCase(TestCase): - def setUp(self): - super().setUp() - self.sq = connections["elasticsearch"].get_query() - - def test_build_query_all(self): - self.assertEqual(self.sq.build_query(), "*:*") - - def test_build_query_single_word(self): - self.sq.add_filter(SQ(content="hello")) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_build_query_boolean(self): - self.sq.add_filter(SQ(content=True)) - self.assertEqual(self.sq.build_query(), "(True)") - - def test_regression_slash_search(self): - self.sq.add_filter(SQ(content="hello/")) - self.assertEqual(self.sq.build_query(), "(hello\\/)") - - def test_build_query_datetime(self): - self.sq.add_filter(SQ(content=datetime.datetime(2009, 5, 8, 11, 28))) - self.assertEqual(self.sq.build_query(), "(2009-05-08T11:28:00)") - - def test_build_query_multiple_words_and(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_filter(SQ(content="world")) - self.assertEqual(self.sq.build_query(), "((hello) AND (world))") - - def test_build_query_multiple_words_not(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) AND NOT ((world)))") - - def test_build_query_multiple_words_or(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) OR (hello))") - - def test_build_query_multiple_words_mixed(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual( - self.sq.build_query(), "(((why) OR (hello)) AND NOT ((world)))" - ) - - def test_build_query_phrase(self): - self.sq.add_filter(SQ(content="hello world")) - self.assertEqual(self.sq.build_query(), "(hello AND world)") - - self.sq.add_filter(SQ(content__exact="hello world")) - self.assertEqual( - self.sq.build_query(), '((hello AND world) AND ("hello world"))' - ) - - def test_build_query_boost(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_boost("world", 5) - self.assertEqual(self.sq.build_query(), "(hello) world^5") - - def test_build_query_multiple_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=Exact("2009-02-10 01:59:00"))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=Exact("2009-02-12 12:13:00"))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_multiple_filter_types_with_datetimes(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13, 0))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10T01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12T12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_in_filter_multiple_words(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=["A Famous Paper", "An Infamous Article"])) - self.assertEqual( - self.sq.build_query(), - '((why) AND title:("A Famous Paper" OR "An Infamous Article"))', - ) - - def test_build_query_in_filter_datetime(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__in=[datetime.datetime(2009, 7, 6, 1, 56, 21)])) - self.assertEqual( - self.sq.build_query(), '((why) AND pub_date:("2009-07-06T01:56:21"))' - ) - - def test_build_query_in_with_set(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in={"A Famous Paper", "An Infamous Article"})) - self.assertTrue("((why) AND title:(" in self.sq.build_query()) - self.assertTrue('"A Famous Paper"' in self.sq.build_query()) - self.assertTrue('"An Infamous Article"' in self.sq.build_query()) - - def test_build_query_wildcard_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__startswith="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack*))") - - def test_build_query_fuzzy_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__fuzzy="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack~))") - - def test_clean(self): - self.assertEqual(self.sq.clean("hello world"), "hello world") - self.assertEqual(self.sq.clean("hello AND world"), "hello and world") - self.assertEqual( - self.sq.clean( - r'hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world' - ), - 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world', - ) - self.assertEqual( - self.sq.clean("so please NOTe i am in a bAND and bORed"), - "so please NOTe i am in a bAND and bORed", - ) - - def test_build_query_with_models(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_model(MockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - self.sq.add_model(AnotherMockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_set_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - # Custom class. - class IttyBittyResult: - pass - - self.sq.set_result_class(IttyBittyResult) - self.assertTrue(issubclass(self.sq.result_class, IttyBittyResult)) - - # Reset to default. - self.sq.set_result_class(None) - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - def test_in_filter_values_list(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=[1, 2, 3])) - self.assertEqual(self.sq.build_query(), '((why) AND title:("1" OR "2" OR "3"))') - - def test_narrow_sq(self): - sqs = SearchQuerySet(using="elasticsearch").narrow(SQ(foo="moof")) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.narrow_queries), 1) - self.assertEqual(sqs.query.narrow_queries.pop(), "foo:(moof)") - - -class Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (0, 9, 9) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions < 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - {"distance": 0.5, "location_field": {"lat": 2.3456789, "lon": 1.2345678}}, - ) - - -class Elasticsearch2SearchQuerySpatialAfterReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (1, 0, 0) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions >= 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - { - "distance": "0.500000km", - "location_field": {"lat": 2.3456789, "lon": 1.2345678}, - }, - ) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 9a78bc5bc..7c658836a 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -95,13 +95,7 @@ try: import elasticsearch - if (2,) <= elasticsearch.__version__ <= (3,): - HAYSTACK_CONNECTIONS["elasticsearch"].update( - { - "ENGINE": "haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine" - } - ) - elif (5,) <= elasticsearch.__version__ <= (6,): + if (5,) <= elasticsearch.__version__ <= (6,): HAYSTACK_CONNECTIONS["elasticsearch"].update( { "ENGINE": "haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine" From a200e257f0f1c6c7bb7ba3ab08313ad6ff479f3e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 07:10:14 +0200 Subject: [PATCH 06/25] [pre-commit.ci] pre-commit autoupdate (#1979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.18.0 → 1.19.0](https://github.com/adamchainz/django-upgrade/compare/1.18.0...1.19.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.10 → v0.5.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.10...v0.5.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4972ed0ea..434f03a11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.18.0 + rev: 1.19.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.10 + rev: v0.5.0 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From aeefbd8bf9e48577716c6e3997bf4d846990cff9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 02:11:56 +0200 Subject: [PATCH 07/25] [pre-commit.ci] pre-commit autoupdate (#1981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.0 → v0.5.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.0...v0.5.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 434f03a11..6378eb2b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.5.1 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 4bb8bcf353159cb3fb5e2dcf2d1634d93fbb7048 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 9 Jul 2024 18:57:46 +0200 Subject: [PATCH 08/25] pre-commit no longer supports the prettier file formater https://github.com/pre-commit/mirrors-prettier --- .pre-commit-config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6378eb2b6..d314b465f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,9 +46,3 @@ repos: - id: pretty-format-json args: ["--autofix", "--no-sort-keys", "--indent=4"] - id: trailing-whitespace - - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 - hooks: - - id: prettier - types_or: [json, toml, xml, yaml] From 28a539e6a7587d87a92de80c4a287feb35f6ac85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 07:13:27 +0200 Subject: [PATCH 09/25] [pre-commit.ci] pre-commit autoupdate (#1985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.1 → v0.5.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.1...v0.5.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d314b465f..a7ff528ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.1 + rev: v0.5.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 98c8d737bfc5f6ce865abb89e15b9d0c5b3899cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:10:49 +0200 Subject: [PATCH 10/25] [pre-commit.ci] pre-commit autoupdate (#1987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.19.0 → 1.20.0](https://github.com/adamchainz/django-upgrade/compare/1.19.0...1.20.0) - [github.com/astral-sh/ruff-pre-commit: v0.5.2 → v0.5.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.2...v0.5.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7ff528ea..c828c0afa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.19.0 + rev: 1.20.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.2 + rev: v0.5.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 33f3e8fd9423b9facb270384720b2efd3ff710d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 01:39:01 +0200 Subject: [PATCH 11/25] [pre-commit.ci] pre-commit autoupdate (#1988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.4 → v0.5.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.4...v0.5.5) * test.yml: sudo apt update --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .github/workflows/test.yml | 4 +++- .pre-commit-config.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4dec7412f..95bc31008 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install system dependencies - run: sudo apt install --no-install-recommends -y gdal-bin + run: | + sudo apt update + sudo apt install --no-install-recommends -y gdal-bin - name: Setup solr test server in Docker run: bash test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c828c0afa..84fb6c0c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.5.5 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5993d986e55ce3a367114a8231c57a0a3661e618 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 06:29:54 +0200 Subject: [PATCH 12/25] [pre-commit.ci] pre-commit autoupdate (#1989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.5 → v0.5.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.5...v0.5.6) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) * .pre-commit-config.yaml: ci: autoupdate_schedule: monthly --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84fb6c0c6..7f04b2f94 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +ci: + autoupdate_schedule: monthly exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade @@ -7,7 +9,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.5.6 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -18,7 +20,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black From 7f492268d906de488cc1a72fb8ea89de4decd5c3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 7 Aug 2024 17:23:54 +0200 Subject: [PATCH 13/25] Add Django v5.1 to the testing --- .github/workflows/test.yml | 6 +++++- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 + tox.ini | 4 +++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95bc31008..a520d80d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - django-version: ["3.2", "4.2", "5.0"] + django-version: ["3.2", "4.2", "5.0", "5.1"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] elastic-version: ["7.17.9"] exclude: @@ -32,6 +32,10 @@ jobs: python-version: "3.8" - django-version: "5.0" python-version: "3.9" + - django-version: "5.1" + python-version: "3.8" + - django-version: "5.1" + python-version: "3.9" services: elastic: image: elasticsearch:${{ matrix.elastic-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f04b2f94..2ff0fcf4e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: rev: 1.20.0 hooks: - id: django-upgrade - args: [--target-version, "5.0"] # Replace with Django version + args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.6 diff --git a/pyproject.toml b/pyproject.toml index 5962dae5b..57c95a8e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Framework :: Django :: 3.2", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", diff --git a/tox.ini b/tox.ini index d5a436091..7868aec7d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs - py{38,39,310,311,312}-django{3.2,4.2,5.0}-es7.x + py{38,39,310,311,312}-django{3.2,4.2,5.0,5.1}-es7.x [gh-actions] python = @@ -16,6 +16,7 @@ DJANGO = 3.2: django3.2 4.2: django4.2 5.0: django5.0 + 5.1: django5.1 [testenv] commands = @@ -32,6 +33,7 @@ deps = django3.2: Django>=3.2,<3.3 django4.2: Django>=4.2,<4.3 django5.0: Django>=5.0,<5.1 + django5.1: Django>=5.1,<5.2 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6 From 068507e6627f96ebd4f3cbe1b789e7b35e590c77 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:00:49 +0200 Subject: [PATCH 14/25] [pre-commit.ci] pre-commit autoupdate (#1995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.6 → v0.6.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.6...v0.6.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ff0fcf4e..65d32e8c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.3 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 10c15ec12c6d71a815b8843c34ab9dd4d01fddb3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 14 Oct 2024 12:44:15 +0200 Subject: [PATCH 15/25] GitHub Actions: Add Python 3.13 to the testing (#1997) * GitHub Actions: Add Python 3.13 to the testing * elastic-version: ["7.17.12"] --- .github/workflows/test.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a520d80d4..7349dd71d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,19 +21,21 @@ jobs: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django django-version: ["3.2", "4.2", "5.0", "5.1"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - elastic-version: ["7.17.9"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + elastic-version: ["7.17.12"] exclude: - django-version: "3.2" python-version: "3.11" - django-version: "3.2" python-version: "3.12" - - django-version: "5.0" - python-version: "3.8" + - django-version: "3.2" + python-version: "3.13" + - django-version: "4.2" + python-version: "3.13" - django-version: "5.0" python-version: "3.9" - - django-version: "5.1" - python-version: "3.8" + - django-version: "5.0" + python-version: "3.13" - django-version: "5.1" python-version: "3.9" services: From 887836c5e20fdfcd29124beb697ed7dfcc079fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Martano?= Date: Mon, 28 Oct 2024 11:48:14 -0300 Subject: [PATCH 16/25] Fix typo. --- docs/installing_search_engines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 8b4157dcb..d2556298b 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -28,7 +28,7 @@ but not useful for haystack, and we'll need to configure solr to use a static (classic) schema. Haystack can generate a viable schema.xml and solrconfig.xml for you from your application and reload the core for you (once Haystack is installed and setup). To do this run: -``./manage.py build_solr_schema --configure-directory= +``./manage.py build_solr_schema --configure-directory= --reload-core``. In this example CoreConfigDir is something like ``../solr-6.5.0/server/solr/tester/conf``, and ``--reload-core`` is what triggers reloading of the core. Please refer to ``build_solr_schema`` From 9a7e091df60c8c2f9b4b24763e7fcee8d9a26a5e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:26:35 +0100 Subject: [PATCH 17/25] [pre-commit.ci] pre-commit autoupdate (#2001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/adamchainz/django-upgrade: 1.20.0 → 1.23.1](https://github.com/adamchainz/django-upgrade/compare/1.20.0...1.23.1) - [github.com/astral-sh/ruff-pre-commit: v0.6.3 → v0.9.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.3...v0.9.9) - [github.com/PyCQA/isort: 5.13.2 → 6.0.1](https://github.com/PyCQA/isort/compare/5.13.2...6.0.1) - [github.com/psf/black: 24.8.0 → 25.1.0](https://github.com/psf/black/compare/24.8.0...25.1.0) - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- haystack/exceptions.py | 1 + test_haystack/test_django_config_detection.py | 4 ---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65d32e8c1..d2c6368f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,29 +3,29 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.20.0 + rev: 1.23.1 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.9.9 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 6.0.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 25.1.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] diff --git a/haystack/exceptions.py b/haystack/exceptions.py index 5c2c4b9a3..95d0bb92a 100644 --- a/haystack/exceptions.py +++ b/haystack/exceptions.py @@ -48,6 +48,7 @@ class SpatialError(HaystackError): class StatsError(HaystackError): "Raised when incorrect arguments have been provided for stats" + pass diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py index 0c3827882..f4808f68c 100644 --- a/test_haystack/test_django_config_detection.py +++ b/test_haystack/test_django_config_detection.py @@ -16,10 +16,6 @@ def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertTrue(has_default_appconfig_attr) - @unittest.skipIf( - django.VERSION < (3, 2), - "default_app_config should be used in versions prior to django 3.2.", - ) def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertFalse(has_default_appconfig_attr) From 9f970a861d3e53af1f1c425da23e4e516674d442 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Apr 2025 23:01:28 +0200 Subject: [PATCH 18/25] [pre-commit.ci] pre-commit autoupdate (#2007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/adamchainz/django-upgrade: 1.23.1 → 1.24.0](https://github.com/adamchainz/django-upgrade/compare/1.23.1...1.24.0) - [github.com/astral-sh/ruff-pre-commit: v0.9.9 → v0.11.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.9...v0.11.4) * facet_types.update(dict.fromkeys(facets, "fields")) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 4 ++-- haystack/backends/whoosh_backend.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2c6368f8..f088cd191 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.23.1 + rev: 1.24.0 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.9 + rev: v0.11.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 13d68035c..f63ce100a 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -462,7 +462,7 @@ def search( group_by += [ FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets ] - facet_types.update({facet: "fields" for facet in facets}) + facet_types.update(dict.fromkeys(facets, "fields")) if date_facets is not None: From 7be88384752865756a3618c078473ad497e8f44a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 20:35:26 +0000 Subject: [PATCH 19/25] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.11.4 → v0.11.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.4...v0.11.8) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f088cd191..29df9666c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.4 + rev: v0.11.8 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 7d139b4937b821cd12e055d3cb0f20a69a62919b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:59:58 +0200 Subject: [PATCH 20/25] [pre-commit.ci] pre-commit autoupdate (#2010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.24.0 → 1.25.0](https://github.com/adamchainz/django-upgrade/compare/1.24.0...1.25.0) - [github.com/astral-sh/ruff-pre-commit: v0.11.8 → v0.11.12](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.8...v0.11.12) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29df9666c..9b3c2d6e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.24.0 + rev: 1.25.0 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.11.12 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From d04a5841d72228961ee84715b9a9562c80fabcb2 Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Wed, 4 Jun 2025 15:36:13 +1200 Subject: [PATCH 21/25] Fix RelatedSearchQueryset.load_all() truncating results Fixes #2011 --- haystack/query.py | 6 +++--- test_haystack/solr_tests/test_solr_backend.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/haystack/query.py b/haystack/query.py index a3cf9490c..0e49486dc 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -194,9 +194,9 @@ def post_process_results(self, results): # No objects were returned -- possible due to SQS nesting such as # XYZ.objects.filter(id__gt=10) where the amount ignored are # exactly equal to the ITERATOR_LOAD_PER_QUERY - del self._result_cache[: len(results)] - self._ignored_result_count += len(results) - break + del self._result_cache[:1] + self._ignored_result_count += 1 + continue to_cache.append(result) diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index cc0ad551a..cab7b88b1 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -1220,6 +1220,21 @@ def test_related_load_all_queryset(self): self.assertEqual([obj.object.id for obj in sqs], list(range(11, 24))) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) + def test_related_load_all_with_empty_model_results(self): + another_index = SolrAnotherMockModelSearchIndex() + another_index.update("solr") + self.ui.build(indexes=[self.smmi, another_index]) + + sqs = self.rsqs.order_by("id") + assert len(list(sqs)) == 25 + sqs = sqs.all().load_all_queryset( + AnotherMockModel, AnotherMockModel.objects.none() + ) + sqs = sqs.load_all() + # two AnotherMockModel objects are skipped, so only 23 results now + # (but those results are still present and weren't skipped) + assert len(list(sqs)) == 23 + def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections["solr"].queries), 0) From f3abe0edc57f0999b67ec43e63aa1b6dfd8835a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 23:19:05 +0200 Subject: [PATCH 22/25] [pre-commit.ci] pre-commit autoupdate (#2014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.11.12 → v0.12.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.12...v0.12.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b3c2d6e1..42930e402 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.12 + rev: v0.12.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 8330862da58abe03ae778f6fa112740442d16b92 Mon Sep 17 00:00:00 2001 From: Dhaval Gojiya <53856555+DhavalGojiya@users.noreply.github.com> Date: Thu, 10 Jul 2025 22:35:02 +0530 Subject: [PATCH 23/25] FIXED: Handle trailing slash in Solr index URL for core reload. (#1968) - When running `python manage.py build_solr_schema --reload_core=True`, it is crucial to correctly extract the Solr core name from the URL defined in the settings. - The existing implementation failed if the URL ended with a trailing slash, resulting in an empty core name due to the final slash being considered as a separator. Added test cases: - `test_build_solr_schema_reload_core_with_trailing_slash` - `test_build_solr_schema_reload_core_without_trailing_slash` These ensure that the core reload logic works correctly regardless of whether the Solr URL has a trailing slash. --- .../management/commands/build_solr_schema.py | 6 ++- .../test_solr_management_commands.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/haystack/management/commands/build_solr_schema.py b/haystack/management/commands/build_solr_schema.py index 21fd4c86b..0ff6215d1 100644 --- a/haystack/management/commands/build_solr_schema.py +++ b/haystack/management/commands/build_solr_schema.py @@ -111,7 +111,11 @@ def handle(self, **options): ) if reload_core: - core = settings.HAYSTACK_CONNECTIONS[using]["URL"].rsplit("/", 1)[-1] + core = ( + settings.HAYSTACK_CONNECTIONS[using]["URL"] + .rstrip("/") + .rsplit("/", 1)[-1] + ) if "ADMIN_URL" not in settings.HAYSTACK_CONNECTIONS[using]: raise ImproperlyConfigured( diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 419d21b6d..73ad57c74 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -290,6 +290,48 @@ def test_build_schema(self): settings.HAYSTACK_CONNECTIONS["solr"]["URL"] = oldurl shutil.rmtree(conf_dir, ignore_errors=True) + def test_build_solr_schema_reload_core_without_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL does not have a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Remove trailing slash if present + updated_url = ( + current_url.rstrip("/") if current_url.endswith("/") else current_url + ) + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + + def test_build_solr_schema_reload_core_with_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL has a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Add a trailing slash if not present + updated_url = current_url if current_url.endswith("/") else current_url + "/" + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + class AppModelManagementCommandTestCase(TestCase): fixtures = ["base_data", "bulk_data.json"] From 63f95058f4d17dacb3a84f60fadf8345e9722353 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 06:58:53 +0200 Subject: [PATCH 24/25] [pre-commit.ci] pre-commit autoupdate (#2017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.2 → v0.12.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.2...v0.12.7) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42930e402..27a1e0665 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.2 + rev: v0.12.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 04dcb8ad0bf494f5dd0a012af934f96d82e80f5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:46:23 +0200 Subject: [PATCH 25/25] Bump the github-actions group with 2 updates (#2018) Bumps the github-actions group with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/checkout` from 4 to 5 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) Updates `actions/download-artifact` from 4 to 5 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/download-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pypi-release.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 91fea6827..664a4dca9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index edbe9af1a..7fb7221a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 7a158c5be..e1bd2ac86 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,7 +7,7 @@ jobs: name: Build Python source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build sdist run: pipx run build --sdist @@ -28,7 +28,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: # unpacks default artifact into dist/ # if `name: artifact` is omitted, the action will create extra parent dir diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7349dd71d..43f75ecf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: ruff: # https://docs.astral.sh/ruff runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: pip install --user ruff - run: ruff check --output-format=github @@ -55,7 +55,7 @@ jobs: ports: - 9001:8983 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: