diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 91fea6827..664a4dca9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index edbe9af1a..7fb7221a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 7a158c5be..e1bd2ac86 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,7 +7,7 @@ jobs: name: Build Python source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build sdist run: pipx run build --sdist @@ -28,7 +28,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: # unpacks default artifact into dist/ # if `name: artifact` is omitted, the action will create extra parent dir diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 257b6a42b..43f75ecf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,9 +10,9 @@ jobs: ruff: # https://docs.astral.sh/ruff runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: pip install --user ruff - - run: ruff --output-format=github + - run: ruff check --output-format=github test: runs-on: ubuntu-latest @@ -20,17 +20,23 @@ jobs: strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - django-version: ["3.2", "4.2", "5.0"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - elastic-version: ["7.17.9"] + django-version: ["3.2", "4.2", "5.0", "5.1"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + elastic-version: ["7.17.12"] exclude: - django-version: "3.2" python-version: "3.11" - django-version: "3.2" python-version: "3.12" + - django-version: "3.2" + python-version: "3.13" + - django-version: "4.2" + python-version: "3.13" - django-version: "5.0" - python-version: "3.8" + python-version: "3.9" - django-version: "5.0" + python-version: "3.13" + - django-version: "5.1" python-version: "3.9" services: elastic: @@ -49,13 +55,15 @@ jobs: ports: - 9001:8983 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install system dependencies - run: sudo apt install --no-install-recommends -y gdal-bin + run: | + sudo apt update + sudo apt install --no-install-recommends -y gdal-bin - name: Setup solr test server in Docker run: bash test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 488a6b6a8..27a1e0665 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,29 +1,31 @@ +ci: + autoupdate_schedule: monthly exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.18.0 + rev: 1.25.0 hooks: - id: django-upgrade - args: [--target-version, "5.0"] # Replace with Django version + args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.12.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 6.0.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 25.1.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] @@ -46,9 +48,3 @@ repos: - id: pretty-format-json args: ["--autofix", "--no-sort-keys", "--indent=4"] - id: trailing-whitespace - - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 - hooks: - - id: prettier - types_or: [json, toml, xml, yaml] diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 50bd6fb06..d2556298b 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -28,7 +28,7 @@ but not useful for haystack, and we'll need to configure solr to use a static (classic) schema. Haystack can generate a viable schema.xml and solrconfig.xml for you from your application and reload the core for you (once Haystack is installed and setup). To do this run: -``./manage.py build_solr_schema --configure-directory= +``./manage.py build_solr_schema --configure-directory= --reload-core``. In this example CoreConfigDir is something like ``../solr-6.5.0/server/solr/tester/conf``, and ``--reload-core`` is what triggers reloading of the core. Please refer to ``build_solr_schema`` @@ -153,7 +153,7 @@ Elasticsearch is similar to Solr — another Java application using Lucene — b focused on ease of deployment and clustering. See https://www.elastic.co/products/elasticsearch for more information. -Haystack currently supports Elasticsearch 1.x, 2.x, 5.x, and 7.x. +Haystack currently supports Elasticsearch 5.x and 7.x. Follow the instructions on https://www.elastic.co/downloads/elasticsearch to download and install Elasticsearch and configure it for your environment. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index d3228beea..d8886c58a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -141,26 +141,6 @@ Example (Solr 6.X):: Elasticsearch ~~~~~~~~~~~~~ -Example (ElasticSearch 1.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - -Example (ElasticSearch 2.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - Example (ElasticSearch 5.x):: HAYSTACK_CONNECTIONS = { diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py deleted file mode 100644 index ce744107f..000000000 --- a/haystack/backends/elasticsearch2_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -import datetime -import warnings - -from django.conf import settings - -from haystack.backends import BaseEngine -from haystack.backends.elasticsearch_backend import ( - ElasticsearchSearchBackend, - ElasticsearchSearchQuery, -) -from haystack.constants import DJANGO_CT -from haystack.exceptions import MissingDependency -from haystack.utils import get_identifier, get_model_ct - -try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch.helpers import bulk, scan - - warnings.warn( - "ElasticSearch 2.x support deprecated, will be removed in 4.0", - DeprecationWarning, - ) -except ImportError: - raise MissingDependency( - "The 'elasticsearch2' backend requires the \ - installation of 'elasticsearch>=2.0.0,<3.0.0'. \ - Please refer to the documentation." - ) - - -class Elasticsearch2SearchBackend(ElasticsearchSearchBackend): - def __init__(self, connection_alias, **connection_options): - super().__init__(connection_alias, **connection_options) - self.content_field_name = None - - def clear(self, models=None, commit=True): - """ - Clears the backend of all documents/objects for a collection of models. - - :param models: List or tuple of models to clear. - :param commit: Not used. - """ - if models is not None: - assert isinstance(models, (list, tuple)) - - try: - if models is None: - self.conn.indices.delete(index=self.index_name, ignore=404) - self.setup_complete = False - self.existing_mapping = {} - self.content_field_name = None - else: - models_to_delete = [] - - for model in models: - models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - - # Delete using scroll API - query = { - "query": {"query_string": {"query": " OR ".join(models_to_delete)}} - } - generator = scan( - self.conn, - query=query, - index=self.index_name, - **self._get_doc_type_option(), - ) - actions = ( - {"_op_type": "delete", "_id": doc["_id"]} for doc in generator - ) - bulk( - self.conn, - actions=actions, - index=self.index_name, - **self._get_doc_type_option(), - ) - self.conn.indices.refresh(index=self.index_name) - - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - if models is not None: - self.log.exception( - "Failed to clear Elasticsearch index of models '%s'", - ",".join(models_to_delete), - ) - else: - self.log.exception("Failed to clear Elasticsearch index") - - def build_search_kwargs( - self, - query_string, - sort_by=None, - start_offset=0, - end_offset=None, - fields="", - highlight=False, - facets=None, - date_facets=None, - query_facets=None, - narrow_queries=None, - spelling_query=None, - within=None, - dwithin=None, - distance_point=None, - models=None, - limit_to_registered_models=None, - result_class=None, - ): - kwargs = super().build_search_kwargs( - query_string, - sort_by, - start_offset, - end_offset, - fields, - highlight, - spelling_query=spelling_query, - within=within, - dwithin=dwithin, - distance_point=distance_point, - models=models, - limit_to_registered_models=limit_to_registered_models, - result_class=result_class, - ) - - filters = [] - if start_offset is not None: - kwargs["from"] = start_offset - - if end_offset is not None: - kwargs["size"] = end_offset - start_offset - - if narrow_queries is None: - narrow_queries = set() - - if facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, extra_options in facets.items(): - facet_options = { - "meta": {"_type": "terms"}, - "terms": {"field": facet_fieldname}, - } - if "order" in extra_options: - facet_options["meta"]["order"] = extra_options.pop("order") - # Special cases for options applied at the facet level (not the terms level). - if extra_options.pop("global_scope", False): - # Renamed "global_scope" since "global" is a python keyword. - facet_options["global"] = True - if "facet_filter" in extra_options: - facet_options["facet_filter"] = extra_options.pop("facet_filter") - facet_options["terms"].update(extra_options) - kwargs["aggs"][facet_fieldname] = facet_options - - if date_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in date_facets.items(): - # Need to detect on gap_by & only add amount if it's more than one. - interval = value.get("gap_by").lower() - - # Need to detect on amount (can't be applied on months or years). - if value.get("gap_amount", 1) != 1 and interval not in ( - "month", - "year", - ): - # Just the first character is valid for use. - interval = "%s%s" % (value["gap_amount"], interval[:1]) - - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "date_histogram"}, - "date_histogram": {"field": facet_fieldname, "interval": interval}, - "aggs": { - facet_fieldname: { - "date_range": { - "field": facet_fieldname, - "ranges": [ - { - "from": self._from_python( - value.get("start_date") - ), - "to": self._from_python(value.get("end_date")), - } - ], - } - } - }, - } - - if query_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in query_facets: - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "query"}, - "filter": {"query_string": {"query": value}}, - } - - for q in narrow_queries: - filters.append({"query_string": {"query": q}}) - - # if we want to filter, change the query type to filteres - if filters: - kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}} - filtered = kwargs["query"]["filtered"] - if "filter" in filtered: - if "bool" in filtered["filter"].keys(): - another_filters = kwargs["query"]["filtered"]["filter"]["bool"][ - "must" - ] - else: - another_filters = [kwargs["query"]["filtered"]["filter"]] - else: - another_filters = filters - - if len(another_filters) == 1: - kwargs["query"]["filtered"]["filter"] = another_filters[0] - else: - kwargs["query"]["filtered"]["filter"] = { - "bool": {"must": another_filters} - } - - return kwargs - - def more_like_this( - self, - model_instance, - additional_query_string=None, - start_offset=0, - end_offset=None, - models=None, - limit_to_registered_models=None, - result_class=None, - **kwargs - ): - from haystack import connections - - if not self.setup_complete: - self.setup() - - # Deferred models will have a different class ("RealClass_Deferred_fieldname") - # which won't be in our registry: - model_klass = model_instance._meta.concrete_model - - index = ( - connections[self.connection_alias] - .get_unified_index() - .get_index(model_klass) - ) - field_name = index.get_content_field() - params = {} - - if start_offset is not None: - params["from_"] = start_offset - - if end_offset is not None: - params["size"] = end_offset - start_offset - - doc_id = get_identifier(model_instance) - - try: - # More like this Query - # https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html - mlt_query = { - "query": { - "more_like_this": { - "fields": [field_name], - "like": [{"_id": doc_id}], - } - } - } - - narrow_queries = [] - - if additional_query_string and additional_query_string != "*:*": - additional_filter = { - "query": {"query_string": {"query": additional_query_string}} - } - narrow_queries.append(additional_filter) - - if limit_to_registered_models is None: - limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - - if models and len(models): - model_choices = sorted(get_model_ct(model) for model in models) - elif limit_to_registered_models: - # Using narrow queries, limit the results to only models handled - # with the current routers. - model_choices = self.build_models_list() - else: - model_choices = [] - - if len(model_choices) > 0: - model_filter = {"terms": {DJANGO_CT: model_choices}} - narrow_queries.append(model_filter) - - if len(narrow_queries) > 0: - mlt_query = { - "query": { - "filtered": { - "query": mlt_query["query"], - "filter": {"bool": {"must": list(narrow_queries)}}, - } - } - } - - raw_results = self.conn.search( - body=mlt_query, - index=self.index_name, - _source=True, - **self._get_doc_type_option(), - **params, - ) - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - self.log.exception( - "Failed to fetch More Like This from Elasticsearch for document '%s'", - doc_id, - ) - raw_results = {} - - return self._process_results(raw_results, result_class=result_class) - - def _process_results( - self, - raw_results, - highlight=False, - result_class=None, - distance_point=None, - geo_sort=False, - ): - results = super()._process_results( - raw_results, highlight, result_class, distance_point, geo_sort - ) - facets = {} - if "aggregations" in raw_results: - facets = {"fields": {}, "dates": {}, "queries": {}} - - for facet_fieldname, facet_info in raw_results["aggregations"].items(): - facet_type = facet_info["meta"]["_type"] - if facet_type == "terms": - facets["fields"][facet_fieldname] = [ - (individual["key"], individual["doc_count"]) - for individual in facet_info["buckets"] - ] - if "order" in facet_info["meta"]: - if facet_info["meta"]["order"] == "reverse_count": - srt = sorted( - facets["fields"][facet_fieldname], key=lambda x: x[1] - ) - facets["fields"][facet_fieldname] = srt - elif facet_type == "date_histogram": - # Elasticsearch provides UTC timestamps with an extra three - # decimals of precision, which datetime barfs on. - facets["dates"][facet_fieldname] = [ - ( - datetime.datetime.utcfromtimestamp( - individual["key"] / 1000 - ), - individual["doc_count"], - ) - for individual in facet_info["buckets"] - ] - elif facet_type == "query": - facets["queries"][facet_fieldname] = facet_info["doc_count"] - results["facets"] = facets - return results - - -class Elasticsearch2SearchQuery(ElasticsearchSearchQuery): - pass - - -class Elasticsearch2SearchEngine(BaseEngine): - backend = Elasticsearch2SearchBackend - query = Elasticsearch2SearchQuery diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 13d68035c..f63ce100a 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -462,7 +462,7 @@ def search( group_by += [ FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets ] - facet_types.update({facet: "fields" for facet in facets}) + facet_types.update(dict.fromkeys(facets, "fields")) if date_facets is not None: diff --git a/haystack/exceptions.py b/haystack/exceptions.py index 5c2c4b9a3..95d0bb92a 100644 --- a/haystack/exceptions.py +++ b/haystack/exceptions.py @@ -48,6 +48,7 @@ class SpatialError(HaystackError): class StatsError(HaystackError): "Raised when incorrect arguments have been provided for stats" + pass diff --git a/haystack/management/commands/build_solr_schema.py b/haystack/management/commands/build_solr_schema.py index 21fd4c86b..0ff6215d1 100644 --- a/haystack/management/commands/build_solr_schema.py +++ b/haystack/management/commands/build_solr_schema.py @@ -111,7 +111,11 @@ def handle(self, **options): ) if reload_core: - core = settings.HAYSTACK_CONNECTIONS[using]["URL"].rsplit("/", 1)[-1] + core = ( + settings.HAYSTACK_CONNECTIONS[using]["URL"] + .rstrip("/") + .rsplit("/", 1)[-1] + ) if "ADMIN_URL" not in settings.HAYSTACK_CONNECTIONS[using]: raise ImproperlyConfigured( diff --git a/haystack/query.py b/haystack/query.py index a3cf9490c..0e49486dc 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -194,9 +194,9 @@ def post_process_results(self, results): # No objects were returned -- possible due to SQS nesting such as # XYZ.objects.filter(id__gt=10) where the amount ignored are # exactly equal to the ITERATOR_LOAD_PER_QUERY - del self._result_cache[: len(results)] - self._ignored_result_count += len(results) - break + del self._result_cache[:1] + self._ignored_result_count += 1 + continue to_cache.append(result) diff --git a/haystack/views.py b/haystack/views.py index fed1808ea..f203f5e3a 100644 --- a/haystack/views.py +++ b/haystack/views.py @@ -11,7 +11,6 @@ class SearchView: template = "search/search.html" - extra_context = {} query = "" results = EmptySearchQuerySet() request = None diff --git a/pyproject.toml b/pyproject.toml index da82ce895..57c95a8e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Framework :: Django :: 3.2", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", @@ -83,20 +84,21 @@ profile = "black" multi_line_output = 3 [tool.ruff] -exclude = ["test_haystack"] -ignore = ["B018", "B028", "B904", "B905"] +extend-exclude = ["test_haystack/*"] line-length = 162 -select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] -show-source = true target-version = "py38" -[tool.ruff.isort] +[tool.ruff.lint] +ignore = ["B018", "B028", "B904", "B905"] +select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] + +[tool.ruff.lint.isort] known-first-party = ["haystack", "test_haystack"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 14 -[tool.ruff.pylint] +[tool.ruff.lint.pylint] max-args = 20 max-branches = 40 max-returns = 8 diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py deleted file mode 100644 index 38fa24fbc..000000000 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import unittest - -from django.conf import settings - -from haystack.utils import log as logging - - -def load_tests(loader, standard_tests, pattern): - log = logging.getLogger("haystack") - try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch import Elasticsearch, exceptions - except ImportError: - log.error( - "Skipping ElasticSearch 2 tests: 'elasticsearch>=2.0.0,<3.0.0' not installed." - ) - raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") - - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - es = Elasticsearch(url) - try: - es.info() - except exceptions.ConnectionError as e: - log.error("elasticsearch not running on %r" % url, exc_info=True) - raise unittest.SkipTest("elasticsearch not running on %r" % url, e) - - package_tests = loader.discover( - start_dir=os.path.dirname(__file__), pattern=pattern - ) - standard_tests.addTests(package_tests) - return standard_tests diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py deleted file mode 100644 index 0ec9608b0..000000000 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ /dev/null @@ -1,1820 +0,0 @@ -import datetime -import logging as std_logging -import operator -import pickle -import unittest -from decimal import Decimal - -import elasticsearch -from django.apps import apps -from django.conf import settings -from django.test import TestCase -from django.test.utils import override_settings - -from haystack import connections, indexes, reset_search_queries -from haystack.exceptions import SkipDocument -from haystack.inputs import AutoQuery -from haystack.models import SearchResult -from haystack.query import SQ, RelatedSearchQuerySet, SearchQuerySet -from haystack.utils import log as logging -from haystack.utils.loading import UnifiedIndex - -from ..core.models import AFourthMockModel, AnotherMockModel, ASixthMockModel, MockModel -from ..mocks import MockSearchResult - - -def clear_elasticsearch_index(): - # Wipe it clean. - raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - try: - raw_es.indices.delete( - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - raw_es.indices.refresh() - except elasticsearch.TransportError: - pass - - # Since we've just completely deleted the index, we'll reset setup_complete so the next access will - # correctly define the mappings: - connections["elasticsearch"].get_backend().setup_complete = False - - -class Elasticsearch2MockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2MockSearchIndexWithSkipDocument(Elasticsearch2MockSearchIndex): - def prepare_text(self, obj): - if obj.author == "daniel3": - raise SkipDocument - return "Indexed!\n%s" % obj.id - - -class Elasticsearch2MockSpellingIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - def prepare_text(self, obj): - return obj.foo - - -class Elasticsearch2MaintainTypeMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - month = indexes.CharField(indexed=False) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def prepare_month(self, obj): - return "%02d" % obj.pub_date.month - - def get_model(self): - return MockModel - - -class Elasticsearch2MockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2AnotherMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AnotherMockModel - - def prepare_text(self, obj): - return "You might be searching for the user %s" % obj.author - - -class Elasticsearch2BoostMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField( - document=True, - use_template=True, - template_name="search/indexes/core/mockmodel_template.txt", - ) - author = indexes.CharField(model_attr="author", weight=2.0) - editor = indexes.CharField(model_attr="editor") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AFourthMockModel - - def prepare(self, obj): - data = super().prepare(obj) - - if obj.pk == 4: - data["boost"] = 5.0 - - return data - - -class Elasticsearch2FacetingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - author = indexes.CharField(model_attr="author", faceted=True) - editor = indexes.CharField(model_attr="editor", faceted=True) - pub_date = indexes.DateField(model_attr="pub_date", faceted=True) - facet_field = indexes.FacetCharField(model_attr="author") - - def prepare_text(self, obj): - return "%s %s" % (obj.author, obj.editor) - - def get_model(self): - return AFourthMockModel - - -class Elasticsearch2RoundTripSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, default="") - name = indexes.CharField() - is_active = indexes.BooleanField() - post_count = indexes.IntegerField() - average_rating = indexes.FloatField() - price = indexes.DecimalField() - pub_date = indexes.DateField() - created = indexes.DateTimeField() - tags = indexes.MultiValueField() - sites = indexes.MultiValueField() - - def get_model(self): - return MockModel - - def prepare(self, obj): - prepped = super().prepare(obj) - prepped.update( - { - "text": "This is some example text.", - "name": "Mister Pants", - "is_active": True, - "post_count": 25, - "average_rating": 3.6, - "price": Decimal("24.99"), - "pub_date": datetime.date(2009, 11, 21), - "created": datetime.datetime(2009, 11, 21, 21, 31, 00), - "tags": ["staff", "outdoor", "activist", "scientist"], - "sites": [3, 5, 1], - } - ) - return prepped - - -class Elasticsearch2ComplexFacetsMockSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(document=True, default="") - name = indexes.CharField(faceted=True) - is_active = indexes.BooleanField(faceted=True) - post_count = indexes.IntegerField() - post_count_i = indexes.FacetIntegerField(facet_for="post_count") - average_rating = indexes.FloatField(faceted=True) - pub_date = indexes.DateField(faceted=True) - created = indexes.DateTimeField(faceted=True) - sites = indexes.MultiValueField(faceted=True) - - def get_model(self): - return MockModel - - -class Elasticsearch2AutocompleteMockModelSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - text_auto = indexes.EdgeNgramField(model_attr="foo") - name_auto = indexes.EdgeNgramField(model_attr="author") - - def get_model(self): - return MockModel - - -class Elasticsearch2SpatialSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="name", document=True) - location = indexes.LocationField() - - def prepare_location(self, obj): - return "%s,%s" % (obj.lat, obj.lon) - - def get_model(self): - return ASixthMockModel - - -class TestSettings(TestCase): - def test_kwargs_are_passed_on(self): - from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend - - backend = ElasticsearchSearchBackend( - "alias", - **{ - "URL": settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], - "INDEX_NAME": "testing", - "KWARGS": {"max_retries": 42}, - } - ) - - self.assertEqual(backend.conn.transport.max_retries, 42) - - -class Elasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.smmidni = Elasticsearch2MockSearchIndexWithSkipDocument() - self.smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - self.sb.silently_fail = True - - def raw_search(self, query): - try: - return self.raw_es.search( - q="*:*", - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"], - ) - except elasticsearch.TransportError: - return {} - - def test_non_silent(self): - bad_sb = connections["elasticsearch"].backend( - "bad", - URL="http://omg.wtf.bbq:1000/", - INDEX_NAME="whatver", - SILENTLY_FAIL=False, - TIMEOUT=1, - ) - - try: - bad_sb.update(self.smmi, self.sample_objs) - self.fail() - except: - pass - - try: - bad_sb.remove("core.mockmodel.1") - self.fail() - except: - pass - - try: - bad_sb.clear() - self.fail() - except: - pass - - try: - bad_sb.search("foo") - self.fail() - except: - pass - - def test_update_no_documents(self): - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - index_name = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True - ) - self.assertEqual(sb.update(self.smmi, []), None) - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=False - ) - try: - sb.update(self.smmi, []) - self.fail() - except: - pass - - def test_update(self): - self.sb.update(self.smmi, self.sample_objs) - - # Check what Elasticsearch thinks is there. - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=lambda x: x["id"], - ), - [ - { - "django_id": "1", - "django_ct": "core.mockmodel", - "name": "daniel1", - "name_exact": "daniel1", - "text": "Indexed!\n1\n", - "pub_date": "2009-02-24T00:00:00", - "id": "core.mockmodel.1", - }, - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_update_with_SkipDocument_raised(self): - self.sb.update(self.smmidni, self.sample_objs) - - # Check what Elasticsearch thinks is there. - res = self.raw_search("*:*")["hits"] - self.assertEqual(res["total"], 2) - self.assertListEqual( - sorted([x["_source"]["id"] for x in res["hits"]]), - ["core.mockmodel.1", "core.mockmodel.2"], - ) - - def test_remove(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 2) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=operator.itemgetter("django_id"), - ), - [ - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_remove_succeeds_on_404(self): - self.sb.silently_fail = False - self.sb.remove("core.mockmodel.421") - - def test_clear(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear() - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel, MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - def test_search(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - set([result.pk for result in self.sb.search("*:*")["results"]]), - {"2", "1", "3"}, - ) - - self.assertEqual(self.sb.search("", highlight=True), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("Index", highlight=True)["hits"], 3) - self.assertEqual( - sorted( - [ - result.highlighted[0] - for result in self.sb.search("Index", highlight=True)["results"] - ] - ), - [ - "Indexed!\n1\n", - "Indexed!\n2\n", - "Indexed!\n3\n", - ], - ) - - self.assertEqual(self.sb.search("Indx")["hits"], 0) - self.assertEqual(self.sb.search("indaxed")["spelling_suggestion"], "indexed") - self.assertEqual( - self.sb.search("arf", spelling_query="indexyd")["spelling_suggestion"], - "indexed", - ) - - self.assertEqual( - self.sb.search("", facets={"name": {}}), {"hits": 0, "results": []} - ) - results = self.sb.search("Index", facets={"name": {}}) - self.assertEqual(results["hits"], 3) - self.assertSetEqual( - set(results["facets"]["fields"]["name"]), - {("daniel3", 1), ("daniel2", 1), ("daniel1", 1)}, - ) - - self.assertEqual( - self.sb.search( - "", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ), - {"hits": 0, "results": []}, - ) - results = self.sb.search( - "Index", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ) - self.assertEqual(results["hits"], 3) - self.assertEqual( - results["facets"]["dates"]["pub_date"], - [(datetime.datetime(2009, 2, 1, 0, 0), 3)], - ) - - self.assertEqual( - self.sb.search("", query_facets=[("name", "[* TO e]")]), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", query_facets=[("name", "[* TO e]")]) - self.assertEqual(results["hits"], 3) - self.assertEqual(results["facets"]["queries"], {"name": 3}) - - self.assertEqual( - self.sb.search("", narrow_queries={"name:daniel1"}), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", narrow_queries={"name:daniel1"}) - self.assertEqual(results["hits"], 1) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sb.search("index", result_class=MockSearchResult)["results"][0], - MockSearchResult, - ) - ) - - # Check the use of ``limit_to_registered_models``. - self.assertEqual( - self.sb.search("", limit_to_registered_models=False), - {"hits": 0, "results": []}, - ) - self.assertEqual( - self.sb.search("*:*", limit_to_registered_models=False)["hits"], 3 - ) - self.assertEqual( - sorted( - [ - result.pk - for result in self.sb.search( - "*:*", limit_to_registered_models=False - )["results"] - ] - ), - ["1", "2", "3"], - ) - - # Stow. - old_limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = False - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - sorted([result.pk for result in self.sb.search("*:*")["results"]]), - ["1", "2", "3"], - ) - - # Restore. - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models - - def test_spatial_search_parameters(self): - from django.contrib.gis.geos import Point - - p1 = Point(1.23, 4.56) - kwargs = self.sb.build_search_kwargs( - "*:*", - distance_point={"field": "location", "point": p1}, - sort_by=(("distance", "desc"),), - ) - - self.assertIn("sort", kwargs) - self.assertEqual(1, len(kwargs["sort"])) - geo_d = kwargs["sort"][0]["_geo_distance"] - - # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be - # in the same order as we used to create the Point(): - # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4 - - self.assertDictEqual( - geo_d, {"location": [1.23, 4.56], "unit": "km", "order": "desc"} - ) - - def test_more_like_this(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - # A functional MLT example with enough data to work is below. Rely on - # this to ensure the API is correct enough. - self.assertEqual(self.sb.more_like_this(self.sample_objs[0])["hits"], 0) - self.assertEqual( - [ - result.pk - for result in self.sb.more_like_this(self.sample_objs[0])["results"] - ], - [], - ) - - def test_build_schema(self): - old_ui = connections["elasticsearch"].get_unified_index() - - (content_field_name, mapping) = self.sb.build_schema(old_ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 4 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - }, - ) - - ui = UnifiedIndex() - ui.build(indexes=[Elasticsearch2ComplexFacetsMockSearchIndex()]) - (content_field_name, mapping) = self.sb.build_schema(ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 15 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name": {"type": "string", "analyzer": "snowball"}, - "is_active_exact": {"type": "boolean"}, - "created": {"type": "date"}, - "post_count": {"type": "long"}, - "created_exact": {"type": "date"}, - "sites_exact": {"index": "not_analyzed", "type": "string"}, - "is_active": {"type": "boolean"}, - "sites": {"type": "string", "analyzer": "snowball"}, - "post_count_i": {"type": "long"}, - "average_rating": {"type": "float"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date_exact": {"type": "date"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - "pub_date": {"type": "date"}, - "average_rating_exact": {"type": "float"}, - }, - ) - - def test_verify_type(self): - old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - ui.build(indexes=[smtmmi]) - connections["elasticsearch"]._index = ui - sb = connections["elasticsearch"].get_backend() - sb.update(smtmmi, self.sample_objs) - - self.assertEqual(sb.search("*:*")["hits"], 3) - self.assertEqual( - [result.month for result in sb.search("*:*")["results"]], ["02", "02", "02"] - ) - connections["elasticsearch"]._index = old_ui - - -class CaptureHandler(std_logging.Handler): - logs_seen = [] - - def emit(self, record): - CaptureHandler.logs_seen.append(record) - - -class FailedElasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - # Stow. - # Point the backend at a URL that doesn't exist so we can watch the - # sparks fly. - self.old_es_url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = ( - "%s/foo/" % self.old_es_url - ) - self.cap = CaptureHandler() - logging.getLogger("haystack").addHandler(self.cap) - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(config.stream) - - # Setup the rest of the bits. - self.old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = ui - self.sb = connections["elasticsearch"].get_backend() - - def tearDown(self): - # Restore. - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = self.old_es_url - connections["elasticsearch"]._index = self.old_ui - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(self.cap) - logging.getLogger("haystack").addHandler(config.stream) - - @unittest.expectedFailure - def test_all_cases(self): - # Prior to the addition of the try/except bits, these would all fail miserably. - self.assertEqual(len(CaptureHandler.logs_seen), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(len(CaptureHandler.logs_seen), 1) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 2) - - self.sb.search("search") - self.assertEqual(len(CaptureHandler.logs_seen), 3) - - self.sb.more_like_this(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 4) - - self.sb.clear([MockModel]) - self.assertEqual(len(CaptureHandler.logs_seen), 5) - - self.sb.clear() - self.assertEqual(len(CaptureHandler.logs_seen), 6) - - -class LiveElasticsearch2SearchQueryTestCase(TestCase): - fixtures = ["base_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - self.sq = connections["elasticsearch"].get_query() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_log_query(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=False): - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=True): - # Redefine it to clear out the cached results. - self.sq = connections["elasticsearch"].query(using="elasticsearch") - self.sq.add_filter(SQ(name="bar")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - - # And again, for good measure. - self.sq = connections["elasticsearch"].query("elasticsearch") - self.sq.add_filter(SQ(name="bar")) - self.sq.add_filter(SQ(text="moof")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - self.assertEqual( - connections["elasticsearch"].queries[1]["query_string"], - "(name:(bar) AND text:(moof))", - ) - - -lssqstc_all_loaded = None - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SearchQuerySetTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - self.rsqs = RelatedSearchQuerySet("elasticsearch") - - # Ugly but not constantly reindexing saves us almost 50% runtime. - global lssqstc_all_loaded - - if lssqstc_all_loaded is None: - lssqstc_all_loaded = True - - # Wipe it clean. - clear_elasticsearch_index() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_load_all(self): - sqs = self.sqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - results = sorted([int(result.pk) for result in sqs]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_values_slicing(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends - - # The values will come back as strings because Hasytack doesn't assume PKs are integers. - # We'll prepare this set once since we're going to query the same results in multiple ways: - expected_pks = [str(i) for i in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]] - - results = self.sqs.all().order_by("pub_date").values("pk") - self.assertListEqual([i["pk"] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk") - self.assertListEqual([i[0] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk", flat=True) - self.assertListEqual(results[1:11], expected_pks) - - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_count(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - self.assertEqual(sqs.count(), 23) - self.assertEqual(sqs.count(), 23) - self.assertEqual(len(sqs), 23) - self.assertEqual(sqs.count(), 23) - # Should only execute one query to count the length of the result set. - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_manual_iter(self): - results = self.sqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = set([int(result.pk) for result in results._manual_iter()]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.sqs._cache_is_full(), False) - results = self.sqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test___and__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 & sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) AND (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar") - sqs = sqs3 & sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 3) - self.assertEqual( - sqs.query.build_query(), - "(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))", - ) - - def test___or__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 | sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) OR (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar").models(MockModel) - sqs = sqs3 | sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual( - sqs.query.build_query(), - "((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))", - ) - - def test_auto_query(self): - # Ensure bits in exact matches get escaped properly as well. - # This will break horrifically if escaping isn't working. - sqs = self.sqs.auto_query('"pants:rule"') - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual( - repr(sqs.query.query_filter), '' - ) - self.assertEqual(sqs.query.build_query(), '("pants\\:rule")') - self.assertEqual(len(sqs), 0) - - # Regressions - - def test_regression_proper_start_offsets(self): - sqs = self.sqs.filter(text="index") - self.assertNotEqual(sqs.count(), 0) - - id_counts = {} - - for item in sqs: - if item.id in id_counts: - id_counts[item.id] += 1 - else: - id_counts[item.id] = 1 - - for key, value in id_counts.items(): - if value > 1: - self.fail( - "Result with id '%s' seen more than once in the results." % key - ) - - def test_regression_raw_search_breaks_slicing(self): - sqs = self.sqs.raw_search("text:index") - page_1 = [result.pk for result in sqs[0:10]] - page_2 = [result.pk for result in sqs[10:20]] - - for pk in page_2: - if pk in page_1: - self.fail( - "Result with id '%s' seen more than once in the results." % pk - ) - - # RelatedSearchQuerySet Tests - - def test_related_load_all(self): - sqs = self.rsqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_related_load_all_queryset(self): - sqs = self.rsqs.load_all().order_by("pub_date") - self.assertEqual(len(sqs._load_all_querysets), 0) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24))) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual( - set([obj.object.id for obj in sqs]), - {12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20}, - ) - self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), {21, 22, 23}) - - def test_related_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.rsqs.all() - results = set([int(result.pk) for result in sqs]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - set([int(result.pk) for result in results[20:30]]), {21, 22, 23} - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_related_manual_iter(self): - results = self.rsqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = sorted([int(result.pk) for result in results._manual_iter()]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_related_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.rsqs._cache_is_full(), False) - results = self.rsqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_quotes_regression(self): - sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E") - # Should not have empty terms. - self.assertEqual(sqs.query.build_query(), "(44\xb048'40''N 20\xb028'32''E)") - # Should not cause Elasticsearch to 500. - self.assertEqual(sqs.count(), 0) - - sqs = self.sqs.auto_query("blazing") - self.assertEqual(sqs.query.build_query(), "(blazing)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("blazing saddles") - self.assertEqual(sqs.query.build_query(), "(blazing saddles)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles') - self.assertEqual(sqs.query.build_query(), '(\\"blazing saddles)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles"') - self.assertEqual(sqs.query.build_query(), '("blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing \'saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing \'saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" ')") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" '\\\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel brooks') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" \\"brooks)') - self.assertEqual(sqs.count(), 0) - - def test_query_generation(self): - sqs = self.sqs.filter( - SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")) - ) - self.assertEqual( - sqs.query.build_query(), "((hello world) OR title:(hello world))" - ) - - def test_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - sqs = self.sqs.all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - # Custom class. - sqs = self.sqs.result_class(MockSearchResult).all() - self.assertTrue(isinstance(sqs[0], MockSearchResult)) - - # Reset to default. - sqs = self.sqs.result_class(None).all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SpellingTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSpellingIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_spelling(self): - self.assertEqual( - self.sqs.auto_query("structurd").spelling_suggestion(), "structured" - ) - self.assertEqual(self.sqs.spelling_suggestion("structurd"), "structured") - self.assertEqual( - self.sqs.auto_query("srchindex instanc").spelling_suggestion(), - "searchindex instance", - ) - self.assertEqual( - self.sqs.spelling_suggestion("srchindex instanc"), "searchindex instance" - ) - - -class LiveElasticsearch2MoreLikeThisTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_more_like_this(self): - mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1)) - results = [result.pk for result in mlt] - self.assertEqual(mlt.count(), 11) - self.assertEqual( - set(results), {"10", "5", "2", "21", "4", "6", "23", "9", "14"} - ) - self.assertEqual(len(results), 10) - - alt_mlt = self.sqs.filter(name="daniel3").more_like_this( - MockModel.objects.get(pk=2) - ) - results = [result.pk for result in alt_mlt] - self.assertEqual(alt_mlt.count(), 9) - self.assertEqual( - set(results), {"2", "16", "3", "19", "4", "17", "10", "22", "23"} - ) - self.assertEqual(len(results), 9) - - alt_mlt_with_models = self.sqs.models(MockModel).more_like_this( - MockModel.objects.get(pk=1) - ) - results = [result.pk for result in alt_mlt_with_models] - self.assertEqual(alt_mlt_with_models.count(), 10) - self.assertEqual( - set(results), {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"} - ) - self.assertEqual(len(results), 10) - - if hasattr(MockModel.objects, "defer"): - # Make sure MLT works with deferred bits. - qs = MockModel.objects.defer("foo") - self.assertEqual(qs.query.deferred_loading[1], True) - deferred = self.sqs.models(MockModel).more_like_this(qs.get(pk=1)) - self.assertEqual(deferred.count(), 10) - self.assertEqual( - {result.pk for result in deferred}, - {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"}, - ) - self.assertEqual(len([result.pk for result in deferred]), 10) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sqs.result_class(MockSearchResult).more_like_this( - MockModel.objects.get(pk=1) - )[0], - MockSearchResult, - ) - ) - - -class LiveElasticsearch2AutocompleteTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2AutocompleteMockModelSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_build_schema(self): - self.sb = connections["elasticsearch"].get_backend() - content_name, mapping = self.sb.build_schema(self.ui.all_searchfields()) - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "text_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - }, - ) - - def test_autocomplete(self): - autocomplete = self.sqs.autocomplete(text_auto="mod") - self.assertEqual(autocomplete.count(), 16) - self.assertEqual( - set([result.pk for result in autocomplete]), - { - "1", - "12", - "6", - "14", - "7", - "4", - "23", - "17", - "13", - "18", - "20", - "22", - "19", - "15", - "10", - "2", - }, - ) - self.assertTrue("mod" in autocomplete[0].text.lower()) - self.assertTrue("mod" in autocomplete[1].text.lower()) - self.assertTrue("mod" in autocomplete[2].text.lower()) - self.assertTrue("mod" in autocomplete[3].text.lower()) - self.assertTrue("mod" in autocomplete[4].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete]), 16) - - # Test multiple words. - autocomplete_2 = self.sqs.autocomplete(text_auto="your mod") - self.assertEqual(autocomplete_2.count(), 13) - self.assertEqual( - set([result.pk for result in autocomplete_2]), - {"1", "6", "2", "14", "12", "13", "10", "19", "4", "20", "23", "22", "15"}, - ) - map_results = {result.pk: result for result in autocomplete_2} - self.assertTrue("your" in map_results["1"].text.lower()) - self.assertTrue("mod" in map_results["1"].text.lower()) - self.assertTrue("your" in map_results["6"].text.lower()) - self.assertTrue("mod" in map_results["6"].text.lower()) - self.assertTrue("your" in map_results["2"].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete_2]), 13) - - # Test multiple fields. - autocomplete_3 = self.sqs.autocomplete(text_auto="Django", name_auto="dan") - self.assertEqual(autocomplete_3.count(), 4) - self.assertEqual( - set([result.pk for result in autocomplete_3]), {"12", "1", "22", "14"} - ) - self.assertEqual(len([result.pk for result in autocomplete_3]), 4) - - # Test numbers in phrases - autocomplete_4 = self.sqs.autocomplete(text_auto="Jen 867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - # Test numbers alone - autocomplete_4 = self.sqs.autocomplete(text_auto="867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - -class LiveElasticsearch2RoundTripTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.srtsi = Elasticsearch2RoundTripSearchIndex() - self.ui.build(indexes=[self.srtsi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sqs = SearchQuerySet("elasticsearch") - - # Fake indexing. - mock = MockModel() - mock.id = 1 - self.sb.update(self.srtsi, [mock]) - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_round_trip(self): - results = self.sqs.filter(id="core.mockmodel.1") - - # Sanity check. - self.assertEqual(results.count(), 1) - - # Check the individual fields. - result = results[0] - self.assertEqual(result.id, "core.mockmodel.1") - self.assertEqual(result.text, "This is some example text.") - self.assertEqual(result.name, "Mister Pants") - self.assertEqual(result.is_active, True) - self.assertEqual(result.post_count, 25) - self.assertEqual(result.average_rating, 3.6) - self.assertEqual(result.price, "24.99") - self.assertEqual(result.pub_date, datetime.date(2009, 11, 21)) - self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00)) - self.assertEqual(result.tags, ["staff", "outdoor", "activist", "scientist"]) - self.assertEqual(result.sites, [3, 5, 1]) - - -class LiveElasticsearch2PickleTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_pickling(self): - results = self.sqs.all() - - for res in results: - # Make sure the cache is full. - pass - - in_a_pickle = pickle.dumps(results) - like_a_cuke = pickle.loads(in_a_pickle) - self.assertEqual(len(like_a_cuke), len(results)) - self.assertEqual(like_a_cuke[0].id, results[0].id) - - -class Elasticsearch2BoostBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2BoostMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sample_objs = [] - - for i in range(1, 5): - mock = AFourthMockModel() - mock.id = i - - if i % 2: - mock.author = "daniel" - mock.editor = "david" - else: - mock.author = "david" - mock.editor = "daniel" - - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def raw_search(self, query): - return self.raw_es.search( - q="*:*", index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - - def test_boost(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 4) - - results = SearchQuerySet(using="elasticsearch").filter( - SQ(author="daniel") | SQ(editor="daniel") - ) - - self.assertEqual( - set([result.id for result in results]), - { - "core.afourthmockmodel.4", - "core.afourthmockmodel.3", - "core.afourthmockmodel.1", - "core.afourthmockmodel.2", - }, - ) - - def test__to_python(self): - self.assertEqual(self.sb._to_python("abc"), "abc") - self.assertEqual(self.sb._to_python("1"), 1) - self.assertEqual(self.sb._to_python("2653"), 2653) - self.assertEqual(self.sb._to_python("25.5"), 25.5) - self.assertEqual(self.sb._to_python("[1, 2, 3]"), [1, 2, 3]) - self.assertEqual( - self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {"a": 1, "c": 3, "b": 2} - ) - self.assertEqual( - self.sb._to_python("2009-05-09T16:14:00"), - datetime.datetime(2009, 5, 9, 16, 14), - ) - self.assertEqual( - self.sb._to_python("2009-05-09T00:00:00"), - datetime.datetime(2009, 5, 9, 0, 0), - ) - self.assertEqual(self.sb._to_python(None), None) - - -class RecreateIndexTestCase(TestCase): - def setUp(self): - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - - def test_recreate_index(self): - clear_elasticsearch_index() - - sb = connections["elasticsearch"].get_backend() - sb.silently_fail = True - sb.setup() - - original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) - - sb.clear() - sb.setup() - - try: - updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) - except elasticsearch.NotFoundError: - self.fail("There is no mapping after recreating the index") - - self.assertEqual( - original_mapping, - updated_mapping, - "Mapping after recreating the index differs from the original one", - ) - - -class Elasticsearch2FacetingTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2FacetingMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 10): - mock = AFourthMockModel() - mock.id = i - if i > 5: - mock.editor = "George Taylor" - else: - mock.editor = "Perry White" - if i % 2: - mock.author = "Daniel Lindsley" - else: - mock.author = "Dan Watson" - mock.pub_date = datetime.date(2013, 9, (i % 4) + 1) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_facet(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 5), ("Dan Watson", 4)] - ) - self.assertEqual( - counts["fields"]["editor"], [("Perry White", 5), ("George Taylor", 4)] - ) - counts = ( - SearchQuerySet("elasticsearch") - .filter(content="white") - .facet("facet_field", order="reverse_count") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["facet_field"], [("Dan Watson", 2), ("Daniel Lindsley", 3)] - ) - - def test_multiple_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .narrow('editor_exact:"Perry White"') - .narrow('author_exact:"Daniel Lindsley"') - .facet("author") - .facet_counts() - ) - self.assertEqual(counts["fields"]["author"], [("Daniel Lindsley", 3)]) - - def test_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .narrow('editor_exact:"Perry White"') - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 3), ("Dan Watson", 2)] - ) - self.assertEqual(counts["fields"]["editor"], [("Perry White", 5)]) - - def test_date_facet(self): - self.sb.update(self.smmi, self.sample_objs) - start = datetime.date(2013, 9, 1) - end = datetime.date(2013, 9, 30) - # Facet by day - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="day") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], - [ - (datetime.datetime(2013, 9, 1), 2), - (datetime.datetime(2013, 9, 2), 3), - (datetime.datetime(2013, 9, 3), 2), - (datetime.datetime(2013, 9, 4), 2), - ], - ) - # By month - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="month") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], [(datetime.datetime(2013, 9, 1), 9)] - ) diff --git a/test_haystack/elasticsearch2_tests/test_inputs.py b/test_haystack/elasticsearch2_tests/test_inputs.py deleted file mode 100644 index af9f8f332..000000000 --- a/test_haystack/elasticsearch2_tests/test_inputs.py +++ /dev/null @@ -1,85 +0,0 @@ -from django.test import TestCase - -from haystack import connections, inputs - - -class Elasticsearch2InputTestCase(TestCase): - def setUp(self): - super().setUp() - self.query_obj = connections["elasticsearch"].get_query() - - def test_raw_init(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {}) - self.assertEqual(raw.post_process, False) - - raw = inputs.Raw("hello OR there, :you", test="really") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {"test": "really"}) - self.assertEqual(raw.post_process, False) - - def test_raw_prepare(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.prepare(self.query_obj), "hello OR there, :you") - - def test_clean_init(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.query_string, "hello OR there, :you") - self.assertEqual(clean.post_process, True) - - def test_clean_prepare(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.prepare(self.query_obj), "hello or there, \\:you") - - def test_exact_init(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.query_string, "hello OR there, :you") - self.assertEqual(exact.post_process, True) - - def test_exact_prepare(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.prepare(self.query_obj), '"hello OR there, :you"') - - exact = inputs.Exact("hello OR there, :you", clean=True) - self.assertEqual(exact.prepare(self.query_obj), '"hello or there, \\:you"') - - def test_not_init(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.query_string, "hello OR there, :you") - self.assertEqual(not_it.post_process, True) - - def test_not_prepare(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.prepare(self.query_obj), "NOT (hello or there, \\:you)") - - def test_autoquery_init(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual(autoquery.query_string, 'panic -don\'t "froody dude"') - self.assertEqual(autoquery.post_process, False) - - def test_autoquery_prepare(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual( - autoquery.prepare(self.query_obj), 'panic NOT don\'t "froody dude"' - ) - - def test_altparser_init(self): - altparser = inputs.AltParser("dismax") - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "") - self.assertEqual(altparser.kwargs, {}) - self.assertEqual(altparser.post_process, False) - - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "douglas adams") - self.assertEqual(altparser.kwargs, {"mm": 1, "qf": "author"}) - self.assertEqual(altparser.post_process, False) - - def test_altparser_prepare(self): - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual( - altparser.prepare(self.query_obj), - """{!dismax mm=1 qf=author v='douglas adams'}""", - ) diff --git a/test_haystack/elasticsearch2_tests/test_query.py b/test_haystack/elasticsearch2_tests/test_query.py deleted file mode 100644 index 5a0111d5b..000000000 --- a/test_haystack/elasticsearch2_tests/test_query.py +++ /dev/null @@ -1,247 +0,0 @@ -import datetime - -import elasticsearch -from django.contrib.gis.measure import D -from django.test import TestCase - -from haystack import connections -from haystack.inputs import Exact -from haystack.models import SearchResult -from haystack.query import SQ, SearchQuerySet - -from ..core.models import AnotherMockModel, MockModel - - -class Elasticsearch2SearchQueryTestCase(TestCase): - def setUp(self): - super().setUp() - self.sq = connections["elasticsearch"].get_query() - - def test_build_query_all(self): - self.assertEqual(self.sq.build_query(), "*:*") - - def test_build_query_single_word(self): - self.sq.add_filter(SQ(content="hello")) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_build_query_boolean(self): - self.sq.add_filter(SQ(content=True)) - self.assertEqual(self.sq.build_query(), "(True)") - - def test_regression_slash_search(self): - self.sq.add_filter(SQ(content="hello/")) - self.assertEqual(self.sq.build_query(), "(hello\\/)") - - def test_build_query_datetime(self): - self.sq.add_filter(SQ(content=datetime.datetime(2009, 5, 8, 11, 28))) - self.assertEqual(self.sq.build_query(), "(2009-05-08T11:28:00)") - - def test_build_query_multiple_words_and(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_filter(SQ(content="world")) - self.assertEqual(self.sq.build_query(), "((hello) AND (world))") - - def test_build_query_multiple_words_not(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) AND NOT ((world)))") - - def test_build_query_multiple_words_or(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) OR (hello))") - - def test_build_query_multiple_words_mixed(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual( - self.sq.build_query(), "(((why) OR (hello)) AND NOT ((world)))" - ) - - def test_build_query_phrase(self): - self.sq.add_filter(SQ(content="hello world")) - self.assertEqual(self.sq.build_query(), "(hello AND world)") - - self.sq.add_filter(SQ(content__exact="hello world")) - self.assertEqual( - self.sq.build_query(), '((hello AND world) AND ("hello world"))' - ) - - def test_build_query_boost(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_boost("world", 5) - self.assertEqual(self.sq.build_query(), "(hello) world^5") - - def test_build_query_multiple_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=Exact("2009-02-10 01:59:00"))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=Exact("2009-02-12 12:13:00"))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_multiple_filter_types_with_datetimes(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13, 0))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10T01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12T12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_in_filter_multiple_words(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=["A Famous Paper", "An Infamous Article"])) - self.assertEqual( - self.sq.build_query(), - '((why) AND title:("A Famous Paper" OR "An Infamous Article"))', - ) - - def test_build_query_in_filter_datetime(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__in=[datetime.datetime(2009, 7, 6, 1, 56, 21)])) - self.assertEqual( - self.sq.build_query(), '((why) AND pub_date:("2009-07-06T01:56:21"))' - ) - - def test_build_query_in_with_set(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in={"A Famous Paper", "An Infamous Article"})) - self.assertTrue("((why) AND title:(" in self.sq.build_query()) - self.assertTrue('"A Famous Paper"' in self.sq.build_query()) - self.assertTrue('"An Infamous Article"' in self.sq.build_query()) - - def test_build_query_wildcard_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__startswith="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack*))") - - def test_build_query_fuzzy_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__fuzzy="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack~))") - - def test_clean(self): - self.assertEqual(self.sq.clean("hello world"), "hello world") - self.assertEqual(self.sq.clean("hello AND world"), "hello and world") - self.assertEqual( - self.sq.clean( - r'hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world' - ), - 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world', - ) - self.assertEqual( - self.sq.clean("so please NOTe i am in a bAND and bORed"), - "so please NOTe i am in a bAND and bORed", - ) - - def test_build_query_with_models(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_model(MockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - self.sq.add_model(AnotherMockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_set_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - # Custom class. - class IttyBittyResult: - pass - - self.sq.set_result_class(IttyBittyResult) - self.assertTrue(issubclass(self.sq.result_class, IttyBittyResult)) - - # Reset to default. - self.sq.set_result_class(None) - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - def test_in_filter_values_list(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=[1, 2, 3])) - self.assertEqual(self.sq.build_query(), '((why) AND title:("1" OR "2" OR "3"))') - - def test_narrow_sq(self): - sqs = SearchQuerySet(using="elasticsearch").narrow(SQ(foo="moof")) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.narrow_queries), 1) - self.assertEqual(sqs.query.narrow_queries.pop(), "foo:(moof)") - - -class Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (0, 9, 9) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions < 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - {"distance": 0.5, "location_field": {"lat": 2.3456789, "lon": 1.2345678}}, - ) - - -class Elasticsearch2SearchQuerySpatialAfterReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (1, 0, 0) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions >= 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - { - "distance": "0.500000km", - "location_field": {"lat": 2.3456789, "lon": 1.2345678}, - }, - ) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 9a78bc5bc..7c658836a 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -95,13 +95,7 @@ try: import elasticsearch - if (2,) <= elasticsearch.__version__ <= (3,): - HAYSTACK_CONNECTIONS["elasticsearch"].update( - { - "ENGINE": "haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine" - } - ) - elif (5,) <= elasticsearch.__version__ <= (6,): + if (5,) <= elasticsearch.__version__ <= (6,): HAYSTACK_CONNECTIONS["elasticsearch"].update( { "ENGINE": "haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine" diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index cc0ad551a..cab7b88b1 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -1220,6 +1220,21 @@ def test_related_load_all_queryset(self): self.assertEqual([obj.object.id for obj in sqs], list(range(11, 24))) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) + def test_related_load_all_with_empty_model_results(self): + another_index = SolrAnotherMockModelSearchIndex() + another_index.update("solr") + self.ui.build(indexes=[self.smmi, another_index]) + + sqs = self.rsqs.order_by("id") + assert len(list(sqs)) == 25 + sqs = sqs.all().load_all_queryset( + AnotherMockModel, AnotherMockModel.objects.none() + ) + sqs = sqs.load_all() + # two AnotherMockModel objects are skipped, so only 23 results now + # (but those results are still present and weren't skipped) + assert len(list(sqs)) == 23 + def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections["solr"].queries), 0) diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 419d21b6d..73ad57c74 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -290,6 +290,48 @@ def test_build_schema(self): settings.HAYSTACK_CONNECTIONS["solr"]["URL"] = oldurl shutil.rmtree(conf_dir, ignore_errors=True) + def test_build_solr_schema_reload_core_without_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL does not have a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Remove trailing slash if present + updated_url = ( + current_url.rstrip("/") if current_url.endswith("/") else current_url + ) + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + + def test_build_solr_schema_reload_core_with_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL has a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Add a trailing slash if not present + updated_url = current_url if current_url.endswith("/") else current_url + "/" + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + class AppModelManagementCommandTestCase(TestCase): fixtures = ["base_data", "bulk_data.json"] diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py index 0c3827882..f4808f68c 100644 --- a/test_haystack/test_django_config_detection.py +++ b/test_haystack/test_django_config_detection.py @@ -16,10 +16,6 @@ def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertTrue(has_default_appconfig_attr) - @unittest.skipIf( - django.VERSION < (3, 2), - "default_app_config should be used in versions prior to django 3.2.", - ) def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertFalse(has_default_appconfig_attr) diff --git a/tox.ini b/tox.ini index d5a436091..7868aec7d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs - py{38,39,310,311,312}-django{3.2,4.2,5.0}-es7.x + py{38,39,310,311,312}-django{3.2,4.2,5.0,5.1}-es7.x [gh-actions] python = @@ -16,6 +16,7 @@ DJANGO = 3.2: django3.2 4.2: django4.2 5.0: django5.0 + 5.1: django5.1 [testenv] commands = @@ -32,6 +33,7 @@ deps = django3.2: Django>=3.2,<3.3 django4.2: Django>=4.2,<4.3 django5.0: Django>=5.0,<5.1 + django5.1: Django>=5.1,<5.2 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6