diff --git a/.editorconfig b/.editorconfig index 87fb28e32..d4649a5fa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,6 +15,9 @@ charset = utf-8 [Makefile] indent_style = tab +[*.{yaml,yml}] +indent_size = 2 + # We don't want to apply our defaults to third-party code or minified bundles: [**/{external,vendor}/**,**.min.{js,css}] indent_style = ignore diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4b5e1c762 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly diff --git a/.github/issue_template.md b/.github/issue_template.md index 88490948a..cdef8e6bd 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -15,4 +15,4 @@ * Search engine version: * Python version: * Django version: -* Haystack version: \ No newline at end of file +* Haystack version: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 35d92349f..620b1ff84 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,3 @@ # Hey, thanks for contributing to Haystack. Please review [the contributor guidelines](https://django-haystack.readthedocs.io/en/latest/contributing.html) and confirm that [the tests pass](https://django-haystack.readthedocs.io/en/latest/running_tests.html) with at least one search engine. -# Once your pull request has been submitted, the full test suite will be executed on https://github.com/django-haystack/django-haystack/actions/workflows/test.yml. Pull requests with passing tests are far more likely to be reviewed and merged. \ No newline at end of file +# Once your pull request has been submitted, the full test suite will be executed on https://github.com/django-haystack/django-haystack/actions/workflows/test.yml. Pull requests with passing tests are far more likely to be reviewed and merged. diff --git a/.github/workflows/black+isort.yml b/.github/workflows/black+isort.yml deleted file mode 100644 index 20ea48d90..000000000 --- a/.github/workflows/black+isort.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: black+isort - -on: [pull_request, push] - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install tools - run: pip install black isort - - name: Run black+isort - run: | - black --check --diff . - isort --check . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b8a15d08b..664a4dca9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,12 +2,12 @@ name: "CodeQL" on: push: - branches: [master, ] + branches: [master] pull_request: # The branches below must be a subset of the branches above branches: [master] schedule: - - cron: '0 6 * * 5' + - cron: "0 6 * * 5" jobs: analyze: @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v5 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: python + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 532eaea9a..7fb7221a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,12 +6,12 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install dependencies - run: pip install sphinx - - name: Build docs - run: cd docs && make html + - uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Install dependencies + run: pip install sphinx + - name: Build docs + run: cd docs && make html diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index 6889aa5a4..000000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: flake8 - -on: [pull_request, push] - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install tools - run: pip install flake8 flake8-assertive flake8-bugbear flake8-builtins flake8-comprehensions flake8-logging-format - - name: Run flake8 - run: flake8 example_project haystack diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 13ae34cee..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Publish - -on: - release: - types: [published] - -jobs: - publish: - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install dependencies - run: python -m pip install --upgrade pip setuptools twine wheel - - name: Build package - run: python setup.py sdist bdist_wheel - - name: Publish to PyPI - run: twine upload --non-interactive dist/* diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 000000000..e1bd2ac86 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,38 @@ +name: "PyPI releases" + +on: release + +jobs: + build_sdist: + name: Build Python source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + path: dist/*.tar.gz + + pypi-publish: + name: Upload release to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + needs: + - build_sdist + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/django-haystack + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v5 + with: + # unpacks default artifact into dist/ + # if `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 066e5d6ff..43f75ecf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,29 +1,43 @@ name: Test -on: [pull_request, push] +on: + push: + branches: [master] + pull_request: + branches: [master] jobs: - test: + ruff: # https://docs.astral.sh/ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - run: pip install --user ruff + - run: ruff check --output-format=github + test: runs-on: ubuntu-latest + needs: ruff # Do not run the tests if linting fails. strategy: - matrix: - django-version: [2.2, 3.1, 3.2] - python-version: [3.6, 3.7, 3.8, 3.9] - elastic-version: [1.7, 2.4, 5.5, '7.13.1'] - include: - - django-version: 2.2 - python-version: 3.5 - elastic-version: 1.7 - - django-version: 2.2 - python-version: 3.5 - elastic-version: 2.4 - - django-version: 2.2 - python-version: 3.5 - elastic-version: 5.5 - - django-version: 2.2 - python-version: 3.5 - elastic-version: '7.13.1' + fail-fast: false + matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django + django-version: ["3.2", "4.2", "5.0", "5.1"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + elastic-version: ["7.17.12"] + exclude: + - django-version: "3.2" + python-version: "3.11" + - django-version: "3.2" + python-version: "3.12" + - django-version: "3.2" + python-version: "3.13" + - django-version: "4.2" + python-version: "3.13" + - django-version: "5.0" + python-version: "3.9" + - django-version: "5.0" + python-version: "3.13" + - django-version: "5.1" + python-version: "3.9" services: elastic: image: elasticsearch:${{ matrix.elastic-version }} @@ -39,21 +53,26 @@ jobs: solr: image: solr:6 ports: - - 9001:9001 + - 9001:8983 steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: sudo apt install --no-install-recommends -y gdal-bin - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - pip install coverage requests - pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} - python setup.py clean build install - - name: Run test - run: coverage run setup.py test - + - uses: actions/checkout@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: | + sudo apt update + sudo apt install --no-install-recommends -y gdal-bin + - name: Setup solr test server in Docker + run: bash test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install coverage requests tox tox-gh-actions + pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} + pip install --editable . + - name: Run test + run: tox -v + env: + DJANGO: ${{ matrix.django-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..27a1e0665 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +ci: + autoupdate_schedule: monthly +exclude: ".*/vendor/.*" +repos: + - repo: https://github.com/adamchainz/django-upgrade + rev: 1.25.0 + hooks: + - id: django-upgrade + args: [--target-version, "5.1"] # Replace with Django version + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.7 + hooks: + - id: ruff + # args: [ --fix, --exit-non-zero-on-fix ] + + - repo: https://github.com/PyCQA/isort + rev: 6.0.1 + hooks: + - id: isort + + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + args: ["--maxkb=128"] + - id: check-ast + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + - id: pretty-format-json + args: ["--autofix", "--no-sort-keys", "--indent=4"] + - id: trailing-whitespace diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..134784f59 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py diff --git a/AUTHORS b/AUTHORS index 411eaf69e..1842e588d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,3 +124,4 @@ Thanks to * parruc for basic Whoosh faceting support * Jens Kadenbach (audax) for updating and testing Whoosh faceting support * Alejandro Sedeño (asedeno) trying the Whoosh faceting thing again + * Fábio Piovam (fabiopiovam) for date_facet on Solr 6.6+ diff --git a/README.rst b/README.rst index ae447fa08..f8d07d548 100644 --- a/README.rst +++ b/README.rst @@ -48,9 +48,8 @@ Documentation ============= * Development version: http://docs.haystacksearch.org/ -* v2.8.X: https://django-haystack.readthedocs.io/en/v2.8.1/ -* v2.7.X: https://django-haystack.readthedocs.io/en/v2.7.0/ -* v2.6.X: https://django-haystack.readthedocs.io/en/v2.6.0/ +* v3.3.0: https://django-haystack.readthedocs.io/en/v3.3.0/ +* v2.8.1: https://django-haystack.readthedocs.io/en/v2.8.1/ See the `changelog `_ @@ -59,8 +58,8 @@ Requirements Haystack has a relatively easily-met set of requirements. -* Python 3.5+ -* A supported version of Django: https://www.djangoproject.com/download/#supported-versions +* Python 3.8+ +* Django 3-5 Additionally, each backend has its own requirements. You should refer to https://django-haystack.readthedocs.io/en/latest/installing_search_engines.html for more diff --git a/docs/backend_support.rst b/docs/backend_support.rst index e32d99f44..59614a83b 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -15,7 +15,7 @@ Supported Backends .. _Solr: http://lucene.apache.org/solr/ .. _ElasticSearch: http://elasticsearch.org/ -.. _Whoosh: https://github.com/mchaput/whoosh/ +.. _Whoosh: https://github.com/whoosh-community/whoosh/ .. _Xapian: http://xapian.org/ @@ -50,7 +50,7 @@ ElasticSearch * Stored (non-indexed) fields * Highlighting * Spatial search -* Requires: `elasticsearch-py `_ 1.x, 2.x, or 5.X. +* Requires: `elasticsearch-py `_ 1.x, 2.x, 5.X, or 7.X. Whoosh ------ diff --git a/docs/changelog.rst b/docs/changelog.rst index 00a749710..132326683 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -900,7 +900,7 @@ Other Add python 3.5 to tests - Add python 3.5 to tests. [Marco Badan] - ref: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django + ref: https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - SearchQuerySet: don’t trigger backend access in __repr__ [Chris Adams] This can lead to confusing errors or performance issues by diff --git a/docs/conf.py b/docs/conf.py index 3b46fa208..d8239e5a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,8 +10,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os -import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/docs/contributing.rst b/docs/contributing.rst index c1ca45c26..7d8f0934f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -115,7 +115,7 @@ If you've been granted the commit bit, here's how to shepherd the changes in: * ``git merge --squash`` is a good tool for performing this, as is ``git rebase -i HEAD~N``. - * This is done to prevent anyone using the git repo from accidently pulling + * This is done to prevent anyone using the git repo from accidentally pulling work-in-progress commits. * Commit messages should use past tense, describe what changed & thank anyone diff --git a/docs/faceting.rst b/docs/faceting.rst index a9e39f29f..c5cd20d5b 100644 --- a/docs/faceting.rst +++ b/docs/faceting.rst @@ -26,12 +26,12 @@ capabilities. The general workflow in this regard is: Faceting can be difficult, especially in providing the user with the right number of options and/or the right areas to be able to drill into. This is unique to every situation and demands following what real users need. - + You may want to consider logging queries and looking at popular terms to help you narrow down how you can help your users. Haystack provides functionality so that all of the above steps are possible. -From the ground up, let's build a faceted search setup. This assumes that you +From the ground up, let's build a faceted search setup. This assumes that you have been to work through the :doc:`tutorial` and have a working Haystack installation. The same setup from the :doc:`tutorial` applies here. @@ -214,8 +214,8 @@ URLconf should resemble:: from django.urls import path from haystack.forms import FacetedSearchForm from haystack.views import FacetedSearchView - - + + urlpatterns = [ path('', FacetedSearchView(form_class=FacetedSearchForm, facet_fields=['author']), name='haystack_search'), ] @@ -243,11 +243,11 @@ might look like this:: - + {% if query %}

By Author

- +
{% if facets.fields.author %} @@ -262,12 +262,12 @@ might look like this::
- + {% for result in page.object_list %}

{{ result.object.title }}

- +

{{ result.object.body|truncatewords:80 }}

{% empty %} diff --git a/docs/faq.rst b/docs/faq.rst index 94fd0d343..05481f107 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -32,7 +32,7 @@ When should I not be using Haystack? ==================================== * Non-Model-based data. If you just want to index random data (flat files, - alternate sources, etc.), Haystack isn't a good solution. Haystack is very + alternate sources, etc.), Haystack isn't a good solution. Haystack is very ``Model``-based and doesn't work well outside of that use case. * Ultra-high volume. Because of the very nature of Haystack (abstraction layer), there's more overhead involved. This makes it portable, but as with all @@ -111,7 +111,7 @@ Several possibilities on this. #. We're not aware of the engine If you think we may not be aware of the engine you'd like, please tell us - about it (preferably via the group - + about it (preferably via the group - http://groups.google.com/group/django-haystack/). Be sure to check through the backends (in case it wasn't documented) and search the history on the group to minimize duplicates. diff --git a/docs/haystack_theme/layout.html b/docs/haystack_theme/layout.html index e1d4ab39e..2cf423bf3 100644 --- a/docs/haystack_theme/layout.html +++ b/docs/haystack_theme/layout.html @@ -1,7 +1,7 @@ {% extends "basic/layout.html" %} {%- block extrahead %} - + {% endblock %} @@ -19,4 +19,4 @@

Haystack

  • Spelling Suggestions
    • -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/docs/haystack_theme/static/documentation.css b/docs/haystack_theme/static/documentation.css index 3e9492cd0..6fa063311 100644 --- a/docs/haystack_theme/static/documentation.css +++ b/docs/haystack_theme/static/documentation.css @@ -26,4 +26,4 @@ div.sphinxsidebar ul ul { padding-left: 10px; margin-left: 10px; } div.bodywrapper { margin: 0px; } div.highlight-python, div.highlight { background-color: #262511; margin-bottom: 10px; padding: 10px; } div.footer { background-color:#262511; font-size: 90%; padding: 10px; } -table thead { background-color: #053211; border-bottom: 1px solid #262511; } \ No newline at end of file +table thead { background-color: #053211; border-bottom: 1px solid #262511; } diff --git a/docs/haystack_theme/theme.conf b/docs/haystack_theme/theme.conf index 3161b4d41..89e03bbda 100644 --- a/docs/haystack_theme/theme.conf +++ b/docs/haystack_theme/theme.conf @@ -1,2 +1,2 @@ [theme] -inherit = basic \ No newline at end of file +inherit = basic diff --git a/docs/index.rst b/docs/index.rst index 747fdf733..7c3a96e10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,7 +43,6 @@ you up and running: glossary management_commands faq - who_uses other_apps installing_search_engines debugging @@ -117,8 +116,8 @@ Requirements Haystack has a relatively easily-met set of requirements. -* Python 2.7+ or Python 3.3+ -* A supported version of Django: https://www.djangoproject.com/download/#supported-versions +* Python 3.8+ +* Django 3-5 Additionally, each backend has its own requirements. You should refer to :doc:`installing_search_engines` for more details. diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 3f8a1e1a2..d2556298b 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -28,7 +28,7 @@ but not useful for haystack, and we'll need to configure solr to use a static (classic) schema. Haystack can generate a viable schema.xml and solrconfig.xml for you from your application and reload the core for you (once Haystack is installed and setup). To do this run: -``./manage.py build_solr_schema --configure-directory= +``./manage.py build_solr_schema --configure-directory= --reload-core``. In this example CoreConfigDir is something like ``../solr-6.5.0/server/solr/tester/conf``, and ``--reload-core`` is what triggers reloading of the core. Please refer to ``build_solr_schema`` @@ -61,7 +61,7 @@ You'll also need to install the ``pysolr`` client library from PyPI:: More Like This -------------- -On Solr 6.X+ "More Like This" functionality is enabled by default. To enable +On Solr 6.X+ "More Like This" functionality is enabled by default. To enable the "More Like This" functionality on earlier versions of Solr, you'll need to enable the ``MoreLikeThisHandler``. Add the following line to your ``solrconfig.xml`` file within the ``config`` tag:: @@ -93,7 +93,7 @@ Then, you enable it in Solr by adding the following line to your ``solrconfig.xml`` file within the ``config`` tag:: - + text_general default @@ -117,14 +117,14 @@ Then change your default handler from:: 10 - + ... to ...:: explicit 10 - + default on true @@ -153,7 +153,7 @@ Elasticsearch is similar to Solr — another Java application using Lucene — b focused on ease of deployment and clustering. See https://www.elastic.co/products/elasticsearch for more information. -Haystack currently supports Elasticsearch 1.x, 2.x, and 5.x. +Haystack currently supports Elasticsearch 5.x and 7.x. Follow the instructions on https://www.elastic.co/downloads/elasticsearch to download and install Elasticsearch and configure it for your environment. @@ -161,7 +161,7 @@ download and install Elasticsearch and configure it for your environment. You'll also need to install the Elasticsearch binding: elasticsearch_ for the appropriate backend version — for example:: - $ pip install "elasticsearch>=5,<6" + $ pip install "elasticsearch>=7,<8" .. _elasticsearch: https://pypi.python.org/pypi/elasticsearch/ diff --git a/docs/multiple_index.rst b/docs/multiple_index.rst index f295db207..3fde249d1 100644 --- a/docs/multiple_index.rst +++ b/docs/multiple_index.rst @@ -84,7 +84,7 @@ Haystack ships with a ``DefaultRouter`` enabled. It looks like:: class DefaultRouter(BaseRouter): def for_read(self, **hints): return DEFAULT_ALIAS - + def for_write(self, **hints): return DEFAULT_ALIAS @@ -118,20 +118,20 @@ Master-Slave Example The ``MasterRouter`` & ``SlaveRouter`` might look like:: from haystack import routers - - + + class MasterRouter(routers.BaseRouter): def for_write(self, **hints): return 'master' - + def for_read(self, **hints): return None - - + + class SlaveRouter(routers.BaseRouter): def for_write(self, **hints): return None - + def for_read(self, **hints): return 'slave' @@ -139,12 +139,12 @@ The observant might notice that since the methods don't overlap, this could be combined into one ``Router`` like so:: from haystack import routers - - + + class MasterSlaveRouter(routers.BaseRouter): def for_write(self, **hints): return 'master' - + def for_read(self, **hints): return 'slave' @@ -160,13 +160,13 @@ For this, the ``SearchQuerySet`` class allows for manually selecting the index via the ``SearchQuerySet.using`` method:: from haystack.query import SearchQuerySet - + # Uses the routers' opinion. sqs = SearchQuerySet().auto_query('banana') - + # Forces the default. sqs = SearchQuerySet().using('default').auto_query('banana') - + # Forces the slave connection (presuming it was setup). sqs = SearchQuerySet().using('slave').auto_query('banana') diff --git a/docs/python3.rst b/docs/python3.rst index 310ced294..ec5e8874e 100644 --- a/docs/python3.rst +++ b/docs/python3.rst @@ -15,7 +15,7 @@ Virtually all tests pass under both Python 2 & 3, with a small number of expected failures under Python (typically related to ordering, see below). .. _`six`: http://pythonhosted.org/six/ -.. _`Django`: https://docs.djangoproject.com/en/1.5/topics/python3/#str-and-unicode-methods +.. _`Django`: https://docs.djangoproject.com/en/stable/topics/python3/#str-and-unicode-methods Supported Backends diff --git a/docs/rich_content_extraction.rst b/docs/rich_content_extraction.rst index 19d672bbb..ef6f03bec 100644 --- a/docs/rich_content_extraction.rst +++ b/docs/rich_content_extraction.rst @@ -65,4 +65,4 @@ template, modified or intermixed with database content as appropriate: {% endfor %} {% endfor %} - {{ extracted.contents|striptags|safe }} \ No newline at end of file + {{ extracted.contents|striptags|safe }} diff --git a/docs/running_tests.rst b/docs/running_tests.rst index 76d4daea8..0f88ba2e1 100644 --- a/docs/running_tests.rst +++ b/docs/running_tests.rst @@ -29,17 +29,13 @@ the errors persist. To run just a portion of the tests you can use the script ``run_tests.py`` and just specify the files or directories you wish to run, for example:: - cd test_haystack - ./run_tests.py whoosh_tests test_loading.py + python test_haystack/run_tests.py whoosh_tests test_loading.py -The ``run_tests.py`` script is just a tiny wrapper around the nose_ library and -any options you pass to it will be passed on; including ``--help`` to get a -list of possible options:: +The ``run_tests.py`` script is just a tiny wrapper around the Django test +command and any options you pass to it will be passed on; including ``--help`` +to get a list of possible options:: - cd test_haystack - ./run_tests.py --help - -.. _nose: https://nose.readthedocs.io/en/latest/ + python test_haystack/run_tests.py --help Configuring Solr ================ @@ -67,4 +63,4 @@ If you want to run the geo-django tests you may need to review the cd test_haystack ./run_tests.py elasticsearch_tests -.. _GeoDjango GEOS and GDAL settings: https://docs.djangoproject.com/en/1.7/ref/contrib/gis/install/geolibs/#geos-library-path +.. _GeoDjango GEOS and GDAL settings: https://docs.djangoproject.com/en/stable/ref/contrib/gis/install/geolibs/#geos-library-path diff --git a/docs/searchindex_api.rst b/docs/searchindex_api.rst index 3f32c1b24..a537e1cda 100644 --- a/docs/searchindex_api.rst +++ b/docs/searchindex_api.rst @@ -352,7 +352,7 @@ non-existent), merely an example of how to extend existing fields. .. note:: - This method is analagous to Django's ``Field.clean`` methods. + This method is analogous to Django's ``Field.clean`` methods. Adding New Fields diff --git a/docs/searchquery_api.rst b/docs/searchquery_api.rst index 305557e06..8704f6fab 100644 --- a/docs/searchquery_api.rst +++ b/docs/searchquery_api.rst @@ -236,7 +236,7 @@ Adds a boosted term and the amount to boost it to the query. Runs a raw query (no parsing) against the backend. -This method causes the ``SearchQuery`` to ignore the standard query-generating +This method causes the ``SearchQuery`` to ignore the standard query-generating facilities, running only what was provided instead. Note that any kwargs passed along will override anything provided diff --git a/docs/settings.rst b/docs/settings.rst index b1a2eb5ae..2208e6e37 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -107,6 +107,8 @@ The following options are optional: don't want indexed or for when you want to replace an index. * ``KWARGS`` - (Solr and ElasticSearch) Any additional keyword arguments that should be passed on to the underlying client library. +* ``DATE_FACET_FIELD`` - (Solr-only) Support to ``date_facet`` on Solr >= 6.6. + Olders set ``date``. Default is ``range``. ``HAYSTACK_ROUTERS`` diff --git a/docs/spatial.rst b/docs/spatial.rst index 34227fa85..3f4b8e028 100644 --- a/docs/spatial.rst +++ b/docs/spatial.rst @@ -14,7 +14,7 @@ close to GeoDjango_ as possible. There are some differences, which we'll highlight throughout this guide. Additionally, while the support isn't as comprehensive as PostGIS (for example), it is still quite useful. -.. _GeoDjango: https://docs.djangoproject.com/en/1.11/ref/contrib/gis/ +.. _GeoDjango: https://docs.djangoproject.com/en/stable/ref/contrib/gis/ Additional Requirements @@ -261,7 +261,8 @@ calculations on your part. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) # Within a two miles. @@ -304,7 +305,8 @@ include these calculated distances on results. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) @@ -322,7 +324,8 @@ key, well-cached hotspots in town but want distances from the user's current position:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) user_loc = Point(-95.23455619812012, 38.97240128290697) @@ -363,7 +366,8 @@ distance information on the results & nothing to sort by. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) downtown_bottom_left = Point(-95.23947, 38.9637903) diff --git a/docs/templatetags.rst b/docs/templatetags.rst index 71d6e0842..f76e7edf7 100644 --- a/docs/templatetags.rst +++ b/docs/templatetags.rst @@ -12,7 +12,7 @@ special features available to templates. ============= Takes a block of text and highlights words from a provided query within that -block of text. Optionally accepts arguments to provide the HTML tag to wrap +block of text. Optionally accepts arguments to provide the HTML tag to wrap highlighted word in, a CSS class to use with the tag and a maximum length of the blurb in characters. @@ -27,11 +27,11 @@ Example:: # Highlight summary with default behavior. {% highlight result.summary with query %} - + # Highlight summary but wrap highlighted words with a div and the # following CSS class. {% highlight result.summary with query html_tag "div" css_class "highlight_me_please" %} - + # Highlight summary but only show 40 words. {% highlight result.summary with query max_length 40 %} @@ -57,10 +57,10 @@ Example:: # Pull a full SearchQuerySet (lazy loaded) of similar content. {% more_like_this entry as related_content %} - + # Pull just the top 5 similar pieces of content. {% more_like_this entry as related_content limit 5 %} - + # Pull just the top 5 similar entries or comments. {% more_like_this entry as related_content for "blog.entry,comments.comment" limit 5 %} diff --git a/docs/toc.rst b/docs/toc.rst index 46ed9bba8..4ae1c846a 100644 --- a/docs/toc.rst +++ b/docs/toc.rst @@ -15,7 +15,6 @@ Table Of Contents installing_search_engines settings faq - who_uses other_apps debugging @@ -50,4 +49,3 @@ Indices and tables ================== * :ref:`search` - diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 89bbe7587..d8886c58a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -4,7 +4,7 @@ Getting Started with Haystack ============================= -Search is a topic of ever increasing importance. Users increasing rely on search +Search is a topic of ever increasing importance. Users increasingly rely on search to separate signal from noise and find what they're looking for quickly. In addition, search can provide insight into what things are popular (many searches), what things are difficult to find on the site and ways you can @@ -54,7 +54,7 @@ note-taking application. Here is ``myapp/models.py``:: title = models.CharField(max_length=200) body = models.TextField() - def __unicode__(self): + def __str__(self): return self.title Finally, before starting with Haystack, you will want to choose a search @@ -141,26 +141,6 @@ Example (Solr 6.X):: Elasticsearch ~~~~~~~~~~~~~ -Example (ElasticSearch 1.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - -Example (ElasticSearch 2.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - Example (ElasticSearch 5.x):: HAYSTACK_CONNECTIONS = { diff --git a/docs/views_and_forms.rst b/docs/views_and_forms.rst index 0edeeeb54..7f518e79b 100644 --- a/docs/views_and_forms.rst +++ b/docs/views_and_forms.rst @@ -11,7 +11,7 @@ Views & Forms which use the standard Django `class-based views`_ which are available in every version of Django which is supported by Haystack. -.. _class-based views: https://docs.djangoproject.com/en/1.7/topics/class-based-views/ +.. _class-based views: https://docs.djangoproject.com/en/stable/topics/class-based-views/ Haystack comes with some default, simple views & forms as well as some django-style views to help you get started and to cover the common cases. @@ -137,7 +137,7 @@ Views which use the standard Django `class-based views`_ which are available in every version of Django which is supported by Haystack. -.. _class-based views: https://docs.djangoproject.com/en/1.7/topics/class-based-views/ +.. _class-based views: https://docs.djangoproject.com/en/stable/topics/class-based-views/ New Django Class Based Views ---------------------------- @@ -145,7 +145,7 @@ New Django Class Based Views .. versionadded:: 2.4.0 The views in ``haystack.generic_views.SearchView`` inherit from Django’s standard -`FormView `_. +`FormView `_. The example views can be customized like any other Django class-based view as demonstrated in this example which filters the search results in ``get_queryset``:: @@ -232,9 +232,9 @@ preprocess the values returned by Haystack, that code would move to ``get_contex | ``get_query()`` | `get_queryset()`_ | +-----------------------+-------------------------------------------+ -.. _get_context_data(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/mixins-simple/#django.views.generic.base.ContextMixin.get_context_data -.. _dispatch(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/base/#django.views.generic.base.View.dispatch -.. _get_queryset(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/mixins-multiple-object/#django.views.generic.list.MultipleObjectMixin.get_queryset +.. _get_context_data(): https://docs.djangoproject.com/en/stable/ref/class-based-views/mixins-simple/#django.views.generic.base.ContextMixin.get_context_data +.. _dispatch(): https://docs.djangoproject.com/en/stable/ref/class-based-views/base/#django.views.generic.base.View.dispatch +.. _get_queryset(): https://docs.djangoproject.com/en/stable/ref/class-based-views/mixins-multiple-object/#django.views.generic.list.MultipleObjectMixin.get_queryset Old-Style Views diff --git a/docs/who_uses.rst b/docs/who_uses.rst deleted file mode 100644 index dc8a13c1a..000000000 --- a/docs/who_uses.rst +++ /dev/null @@ -1,367 +0,0 @@ -.. _ref-who-uses: - -Sites Using Haystack -==================== - -The following sites are a partial list of people using Haystack. I'm always -interested in adding more sites, so please find me (``daniellindsley``) via -IRC or the mailing list thread. - - -LJWorld/Lawrence.com/KUSports ------------------------------ - -For all things search-related. - -Using: Solr - -* http://www2.ljworld.com/search/ -* http://www2.ljworld.com/search/vertical/news.story/ -* http://www2.ljworld.com/marketplace/ -* http://www.lawrence.com/search/ -* http://www.kusports.com/search/ - - -AltWeeklies ------------ - -Providing an API to story aggregation. - -Using: Whoosh - -* http://www.northcoastjournal.com/altweeklies/documentation/ - - -Teachoo ------------ - -Teachoo uses Haystack for its site search. - -Using: Elasticsearch - -* https://www.teachoo.com/ - - -Trapeze -------- - -Various projects. - -Using: Xapian - -* http://www.trapeze.com/ -* http://www.windmobile.ca/ -* http://www.bonefishgrill.com/ -* http://www.canadiantire.ca/ (Portions of) - - -Vickerey.com ------------- - -For (really well done) search & faceting. - -Using: Solr - -* http://store.vickerey.com/products/search/ - - -Eldarion --------- - -Various projects. - -Using: Solr - -* http://eldarion.com/ - - -Sunlight Labs -------------- - -For general search. - -Using: Whoosh & Solr - -* http://sunlightlabs.com/ -* http://subsidyscope.com/ - - -NASA ----- - -For general search. - -Using: Solr - -* An internal site called SMD Spacebook 1.1. -* http://science.nasa.gov/ - - -AllForLocal ------------ - -For general search. - -* http://www.allforlocal.com/ - - -HUGE ----- - -Various projects. - -Using: Solr - -* http://hugeinc.com/ -* http://houselogic.com/ - - -Brick Design ------------- - -For search on Explore. - -Using: Solr - -* http://bricksf.com/ -* http://explore.org/ - - -Winding Road ------------- - -For general search. - -Using: Solr - -* http://www.windingroad.com/ - - -Reddit ------- - -For Reddit Gifts. - -Using: Whoosh - -* http://redditgifts.com/ - - -Pegasus News ------------- - -For general search. - -Using: Xapian - -* http://www.pegasusnews.com/ - - -Rampframe ---------- - -For general search. - -Using: Xapian - -* http://www.rampframe.com/ - - -Forkinit --------- - -For general search, model-specific search and suggestions via MLT. - -Using: Solr - -* http://forkinit.com/ - - -Structured Abstraction ----------------------- - -For general search. - -Using: Xapian - -* http://www.structuredabstraction.com/ -* http://www.delivergood.org/ - - -CustomMade ----------- - -For general search. - -Using: Solr - -* http://www.custommade.com/ - - -University of the Andes, Dept. of Political Science ---------------------------------------------------- - -For general search & section-specific search. Developed by Monoku. - -Using: Solr - -* http://www.congresovisible.org/ -* http://www.monoku.com/ - - -Christchurch Art Gallery ------------------------- - -For general search & section-specific search. - -Using: Solr - -* http://christchurchartgallery.org.nz/search/ -* http://christchurchartgallery.org.nz/collection/browse/ - - -DevCheatSheet.com ------------------ - -For general search. - -Using: Xapian - -* http://devcheatsheet.com/ - - -TodasLasRecetas ---------------- - -For search, faceting & More Like This. - -Using: Solr - -* http://www.todaslasrecetas.es/receta/s/?q=langostinos -* http://www.todaslasrecetas.es/receta/9526/brochetas-de-langostinos - - -AstroBin --------- - -For general search. - -Using: Solr - -* http://www.astrobin.com/ - - -European Paper Company ----------------------- - -For general search. - -Using: ??? - -* http://europeanpaper.com/ - - -mtn-op ------- - -For general search. - -Using: ??? - -* http://mountain-op.com/ - - -Crate ------ - -Crate is a PyPI mirror/replacement. It's using Haystack to power all search & -faceted navigation on the site. - -Using: Elasticsearch - -* https://crate.io/ - - -Pix Populi ----------- - -Pix Populi is a popular French photo sharing site. - -Using: Solr - -* http://www.pix-populi.fr/ - - -LocalWiki ----------- - -LocalWiki is a tool for collaborating in local, geographic communities. -It's using Haystack to power search on every LocalWiki instance. - -Using: Solr - -* http://localwiki.org/ - - -Pitchup -------- - -For faceting, geo and autocomplete. - -Using: ??? - -* http://www.pitchup.com/search/ - - -Gidsy ------ - -Gidsy makes it easy for anyone to organize and find exciting things -to do everywhere in the world. - -For activity search, area pages, forums and private messages. - -Using: Elasticsearch - -* https://gidsy.com/ -* https://gidsy.com/search/ -* https://gidsy.com/forum/ - - -GroundCity ----------- - -Groundcity is a Romanian dynamic real estate site. - -For real estate, forums and comments. - -Using: Whoosh - -* http://groundcity.ro/cautare/ - - -Docket Alarm ------------- - -Docket Alarm allows people to search court dockets across -the country. With it, you can search court dockets in the International Trade -Commission (ITC), the Patent Trial and Appeal Board (PTAB) and All Federal -Courts. - -Using: Elasticsearch - -* https://www.docketalarm.com/search/ITC -* https://www.docketalarm.com/search/PTAB -* https://www.docketalarm.com/search/dockets - - -Educreations -------------- - -Educreations makes it easy for anyone to teach what they know and learn -what they don't with a recordable whiteboard. Haystack is used to -provide search across users and lessons. - -Using: Solr - -* http://www.educreations.com/browse/ diff --git a/example_project/regular_app/models.py b/example_project/regular_app/models.py index e1a075e69..854ab2c26 100644 --- a/example_project/regular_app/models.py +++ b/example_project/regular_app/models.py @@ -36,7 +36,7 @@ def full_name(self): class Toy(models.Model): - dog = models.ForeignKey(Dog, related_name="toys") + dog = models.ForeignKey(Dog, on_delete=models.CASCADE, related_name="toys") name = models.CharField(max_length=60) def __str__(self): diff --git a/example_project/templates/search/indexes/bare_bones_app/cat_text.txt b/example_project/templates/search/indexes/bare_bones_app/cat_text.txt index db8321ea5..10f379a16 100644 --- a/example_project/templates/search/indexes/bare_bones_app/cat_text.txt +++ b/example_project/templates/search/indexes/bare_bones_app/cat_text.txt @@ -1,2 +1,2 @@ {{ object.name }} -{{ object.bio }} \ No newline at end of file +{{ object.bio }} diff --git a/example_project/templates/search/indexes/regular_app/dog_text.txt b/example_project/templates/search/indexes/regular_app/dog_text.txt index 232490549..c796c047c 100644 --- a/example_project/templates/search/indexes/regular_app/dog_text.txt +++ b/example_project/templates/search/indexes/regular_app/dog_text.txt @@ -4,4 +4,4 @@ {% for toy in object.toys.all %} {{ toy.name }} -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/haystack/__init__.py b/haystack/__init__.py index 6282f322f..4f427573d 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -1,6 +1,8 @@ +from importlib.metadata import PackageNotFoundError, version + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from pkg_resources import DistributionNotFound, get_distribution, parse_version +from packaging.version import Version from haystack.constants import DEFAULT_ALIAS from haystack.utils import loading @@ -8,15 +10,11 @@ __author__ = "Daniel Lindsley" try: - pkg_distribution = get_distribution("django-haystack") - __version__ = pkg_distribution.version - version_info = pkg_distribution.parsed_version -except DistributionNotFound: + __version__ = version("django-haystack") + version_info = Version(__version__) +except PackageNotFoundError: __version__ = "0.0.dev0" - version_info = parse_version(__version__) - -default_app_config = "haystack.apps.HaystackConfig" - + version_info = Version(__version__) # Help people clean up from 1.X. if hasattr(settings, "HAYSTACK_SITECONF"): diff --git a/haystack/admin.py b/haystack/admin.py index 83ebe398e..3f2fd0c19 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -1,10 +1,11 @@ +from django import VERSION as django_version from django.contrib.admin.options import ModelAdmin, csrf_protect_m from django.contrib.admin.views.main import SEARCH_VAR, ChangeList from django.core.exceptions import PermissionDenied from django.core.paginator import InvalidPage, Paginator from django.shortcuts import render from django.utils.encoding import force_str -from django.utils.translation import ungettext +from django.utils.translation import ngettext from haystack import connections from haystack.constants import DEFAULT_ALIAS @@ -15,7 +16,10 @@ class SearchChangeList(ChangeList): def __init__(self, **kwargs): self.haystack_connection = kwargs.pop("haystack_connection", DEFAULT_ALIAS) - super().__init__(**kwargs) + super_kwargs = kwargs + if django_version[0] >= 4: + super_kwargs["search_help_text"] = "Search..." + super().__init__(**super_kwargs) def get_results(self, request): if SEARCH_VAR not in request.GET: @@ -41,7 +45,7 @@ def get_results(self, request): # Get the list of objects to display on this page. try: - result_list = paginator.page(self.page_num + 1).object_list + result_list = paginator.page(self.page_num).object_list # Grab just the Django models, since that's what everything else is # expecting. result_list = [result.object for result in result_list] @@ -116,12 +120,12 @@ def changelist_view(self, request, extra_context=None): else: action_form = None - selection_note = ungettext( + selection_note = ngettext( "0 of %(count)d selected", "of %(count)d selected", len(changelist.result_list), ) - selection_note_all = ungettext( + selection_note_all = ngettext( "%(total_count)s selected", "All %(total_count)s selected", changelist.result_count, diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py deleted file mode 100644 index 97c8cca15..000000000 --- a/haystack/backends/elasticsearch2_backend.py +++ /dev/null @@ -1,390 +0,0 @@ -import datetime -import warnings - -from django.conf import settings - -from haystack.backends import BaseEngine -from haystack.backends.elasticsearch_backend import ( - ElasticsearchSearchBackend, - ElasticsearchSearchQuery, -) -from haystack.constants import DJANGO_CT -from haystack.exceptions import MissingDependency -from haystack.utils import get_identifier, get_model_ct - -try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch.helpers import bulk, scan - - warnings.warn( - "ElasticSearch 2.x support deprecated, will be removed in 4.0", - DeprecationWarning, - ) -except ImportError: - raise MissingDependency( - "The 'elasticsearch2' backend requires the \ - installation of 'elasticsearch>=2.0.0,<3.0.0'. \ - Please refer to the documentation." - ) - - -class Elasticsearch2SearchBackend(ElasticsearchSearchBackend): - def __init__(self, connection_alias, **connection_options): - super().__init__(connection_alias, **connection_options) - self.content_field_name = None - - def clear(self, models=None, commit=True): - """ - Clears the backend of all documents/objects for a collection of models. - - :param models: List or tuple of models to clear. - :param commit: Not used. - """ - if models is not None: - assert isinstance(models, (list, tuple)) - - try: - if models is None: - self.conn.indices.delete(index=self.index_name, ignore=404) - self.setup_complete = False - self.existing_mapping = {} - self.content_field_name = None - else: - models_to_delete = [] - - for model in models: - models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - - # Delete using scroll API - query = { - "query": {"query_string": {"query": " OR ".join(models_to_delete)}} - } - generator = scan( - self.conn, - query=query, - index=self.index_name, - **self._get_doc_type_option(), - ) - actions = ( - {"_op_type": "delete", "_id": doc["_id"]} for doc in generator - ) - bulk( - self.conn, - actions=actions, - index=self.index_name, - **self._get_doc_type_option(), - ) - self.conn.indices.refresh(index=self.index_name) - - except elasticsearch.TransportError as e: - if not self.silently_fail: - raise - - if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", - ",".join(models_to_delete), - e, - exc_info=True, - ) - else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) - - def build_search_kwargs( - self, - query_string, - sort_by=None, - start_offset=0, - end_offset=None, - fields="", - highlight=False, - facets=None, - date_facets=None, - query_facets=None, - narrow_queries=None, - spelling_query=None, - within=None, - dwithin=None, - distance_point=None, - models=None, - limit_to_registered_models=None, - result_class=None, - ): - kwargs = super().build_search_kwargs( - query_string, - sort_by, - start_offset, - end_offset, - fields, - highlight, - spelling_query=spelling_query, - within=within, - dwithin=dwithin, - distance_point=distance_point, - models=models, - limit_to_registered_models=limit_to_registered_models, - result_class=result_class, - ) - - filters = [] - if start_offset is not None: - kwargs["from"] = start_offset - - if end_offset is not None: - kwargs["size"] = end_offset - start_offset - - if narrow_queries is None: - narrow_queries = set() - - if facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, extra_options in facets.items(): - facet_options = { - "meta": {"_type": "terms"}, - "terms": {"field": facet_fieldname}, - } - if "order" in extra_options: - facet_options["meta"]["order"] = extra_options.pop("order") - # Special cases for options applied at the facet level (not the terms level). - if extra_options.pop("global_scope", False): - # Renamed "global_scope" since "global" is a python keyword. - facet_options["global"] = True - if "facet_filter" in extra_options: - facet_options["facet_filter"] = extra_options.pop("facet_filter") - facet_options["terms"].update(extra_options) - kwargs["aggs"][facet_fieldname] = facet_options - - if date_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in date_facets.items(): - # Need to detect on gap_by & only add amount if it's more than one. - interval = value.get("gap_by").lower() - - # Need to detect on amount (can't be applied on months or years). - if value.get("gap_amount", 1) != 1 and interval not in ( - "month", - "year", - ): - # Just the first character is valid for use. - interval = "%s%s" % (value["gap_amount"], interval[:1]) - - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "date_histogram"}, - "date_histogram": {"field": facet_fieldname, "interval": interval}, - "aggs": { - facet_fieldname: { - "date_range": { - "field": facet_fieldname, - "ranges": [ - { - "from": self._from_python( - value.get("start_date") - ), - "to": self._from_python(value.get("end_date")), - } - ], - } - } - }, - } - - if query_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in query_facets: - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "query"}, - "filter": {"query_string": {"query": value}}, - } - - for q in narrow_queries: - filters.append({"query_string": {"query": q}}) - - # if we want to filter, change the query type to filteres - if filters: - kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}} - filtered = kwargs["query"]["filtered"] - if "filter" in filtered: - if "bool" in filtered["filter"].keys(): - another_filters = kwargs["query"]["filtered"]["filter"]["bool"][ - "must" - ] - else: - another_filters = [kwargs["query"]["filtered"]["filter"]] - else: - another_filters = filters - - if len(another_filters) == 1: - kwargs["query"]["filtered"]["filter"] = another_filters[0] - else: - kwargs["query"]["filtered"]["filter"] = { - "bool": {"must": another_filters} - } - - return kwargs - - def more_like_this( - self, - model_instance, - additional_query_string=None, - start_offset=0, - end_offset=None, - models=None, - limit_to_registered_models=None, - result_class=None, - **kwargs - ): - from haystack import connections - - if not self.setup_complete: - self.setup() - - # Deferred models will have a different class ("RealClass_Deferred_fieldname") - # which won't be in our registry: - model_klass = model_instance._meta.concrete_model - - index = ( - connections[self.connection_alias] - .get_unified_index() - .get_index(model_klass) - ) - field_name = index.get_content_field() - params = {} - - if start_offset is not None: - params["from_"] = start_offset - - if end_offset is not None: - params["size"] = end_offset - start_offset - - doc_id = get_identifier(model_instance) - - try: - # More like this Query - # https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html - mlt_query = { - "query": { - "more_like_this": { - "fields": [field_name], - "like": [{"_id": doc_id}], - } - } - } - - narrow_queries = [] - - if additional_query_string and additional_query_string != "*:*": - additional_filter = { - "query": {"query_string": {"query": additional_query_string}} - } - narrow_queries.append(additional_filter) - - if limit_to_registered_models is None: - limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - - if models and len(models): - model_choices = sorted(get_model_ct(model) for model in models) - elif limit_to_registered_models: - # Using narrow queries, limit the results to only models handled - # with the current routers. - model_choices = self.build_models_list() - else: - model_choices = [] - - if len(model_choices) > 0: - model_filter = {"terms": {DJANGO_CT: model_choices}} - narrow_queries.append(model_filter) - - if len(narrow_queries) > 0: - mlt_query = { - "query": { - "filtered": { - "query": mlt_query["query"], - "filter": {"bool": {"must": list(narrow_queries)}}, - } - } - } - - raw_results = self.conn.search( - body=mlt_query, - index=self.index_name, - _source=True, - **self._get_doc_type_option(), - **params, - ) - except elasticsearch.TransportError as e: - if not self.silently_fail: - raise - - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", - doc_id, - e, - exc_info=True, - ) - raw_results = {} - - return self._process_results(raw_results, result_class=result_class) - - def _process_results( - self, - raw_results, - highlight=False, - result_class=None, - distance_point=None, - geo_sort=False, - ): - results = super()._process_results( - raw_results, highlight, result_class, distance_point, geo_sort - ) - facets = {} - if "aggregations" in raw_results: - facets = {"fields": {}, "dates": {}, "queries": {}} - - for facet_fieldname, facet_info in raw_results["aggregations"].items(): - facet_type = facet_info["meta"]["_type"] - if facet_type == "terms": - facets["fields"][facet_fieldname] = [ - (individual["key"], individual["doc_count"]) - for individual in facet_info["buckets"] - ] - if "order" in facet_info["meta"]: - if facet_info["meta"]["order"] == "reverse_count": - srt = sorted( - facets["fields"][facet_fieldname], key=lambda x: x[1] - ) - facets["fields"][facet_fieldname] = srt - elif facet_type == "date_histogram": - # Elasticsearch provides UTC timestamps with an extra three - # decimals of precision, which datetime barfs on. - facets["dates"][facet_fieldname] = [ - ( - datetime.datetime.utcfromtimestamp( - individual["key"] / 1000 - ), - individual["doc_count"], - ) - for individual in facet_info["buckets"] - ] - elif facet_type == "query": - facets["queries"][facet_fieldname] = facet_info["doc_count"] - results["facets"] = facets - return results - - -class Elasticsearch2SearchQuery(ElasticsearchSearchQuery): - pass - - -class Elasticsearch2SearchEngine(BaseEngine): - backend = Elasticsearch2SearchBackend - query = Elasticsearch2SearchQuery diff --git a/haystack/backends/elasticsearch5_backend.py b/haystack/backends/elasticsearch5_backend.py index 2eedc1ad3..3afe11347 100644 --- a/haystack/backends/elasticsearch5_backend.py +++ b/haystack/backends/elasticsearch5_backend.py @@ -75,21 +75,17 @@ def clear(self, models=None, commit=True): ) self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -411,15 +407,13 @@ def more_like_this( **self._get_doc_type_option(), **params, ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} diff --git a/haystack/backends/elasticsearch7_backend.py b/haystack/backends/elasticsearch7_backend.py index dd9c9933d..f8d7d767e 100644 --- a/haystack/backends/elasticsearch7_backend.py +++ b/haystack/backends/elasticsearch7_backend.py @@ -27,29 +27,6 @@ ) -DEFAULT_FIELD_MAPPING = { - "type": "text", - "analyzer": "snowball", -} -FIELD_MAPPINGS = { - "edge_ngram": { - "type": "text", - "analyzer": "edgengram_analyzer", - }, - "ngram": { - "type": "text", - "analyzer": "ngram_analyzer", - }, - "date": {"type": "date"}, - "datetime": {"type": "date"}, - "location": {"type": "geo_point"}, - "boolean": {"type": "boolean"}, - "float": {"type": "float"}, - "long": {"type": "long"}, - "integer": {"type": "long"}, -} - - class Elasticsearch7SearchBackend(ElasticsearchSearchBackend): # Settings to add an n-gram & edge n-gram analyzer. DEFAULT_SETTINGS = { @@ -90,6 +67,29 @@ class Elasticsearch7SearchBackend(ElasticsearchSearchBackend): }, } + DEFAULT_FIELD_MAPPING = { + "type": "text", + "analyzer": "snowball", + } + + FIELD_MAPPINGS = { + "edge_ngram": { + "type": "text", + "analyzer": "edgengram_analyzer", + }, + "ngram": { + "type": "text", + "analyzer": "ngram_analyzer", + }, + "date": {"type": "date"}, + "datetime": {"type": "date"}, + "location": {"type": "geo_point"}, + "boolean": {"type": "boolean"}, + "float": {"type": "float"}, + "long": {"type": "long"}, + "integer": {"type": "long"}, + } + def __init__(self, connection_alias, **connection_options): super().__init__(connection_alias, **connection_options) self.content_field_name = None @@ -143,21 +143,17 @@ def clear(self, models=None, commit=True): ) self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -479,15 +475,13 @@ def more_like_this( raw_results = self.conn.search( body=mlt_query, index=self.index_name, _source=True, **params ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} @@ -556,8 +550,8 @@ def build_schema(self, fields): mapping = self._get_common_mapping() for _, field_class in fields.items(): - field_mapping = FIELD_MAPPINGS.get( - field_class.field_type, DEFAULT_FIELD_MAPPING + field_mapping = self.FIELD_MAPPINGS.get( + field_class.field_type, self.DEFAULT_FIELD_MAPPING ).copy() if field_class.boost != 1.0: field_mapping["boost"] = field_class.boost diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index c2fb47f5f..e8febf9d3 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -199,13 +199,11 @@ def update(self, index, iterable, commit=True): if not self.setup_complete: try: self.setup() - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to add documents to Elasticsearch: %s", e, exc_info=True - ) + self.log.exception("Failed to add documents to Elasticsearch") return prepped_docs = [] @@ -223,16 +221,15 @@ def update(self, index, iterable, commit=True): prepped_docs.append(final_data) except SkipDocument: self.log.debug("Indexing for object `%s` skipped", obj) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( - "%s while preparing object for update" % e.__class__.__name__, - exc_info=True, + self.log.exception( + "Preparing object for update", extra={"data": {"index": index, "object": get_identifier(obj)}}, ) @@ -252,15 +249,13 @@ def remove(self, obj_or_string, commit=True): if not self.setup_complete: try: self.setup() - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Elasticsearch: %s", + self.log.exception( + "Failed to remove document '%s' from Elasticsearch", doc_id, - e, - exc_info=True, ) return @@ -274,15 +269,13 @@ def remove(self, obj_or_string, commit=True): if commit: self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Elasticsearch: %s", + self.log.exception( + "Failed to remove document '%s' from Elasticsearch", doc_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -305,7 +298,7 @@ def clear(self, models=None, commit=True): for model in models: models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - # Delete by query in Elasticsearch asssumes you're dealing with + # Delete by query in Elasticsearch assumes you're dealing with # a ``query`` root object. :/ query = { "query": {"query_string": {"query": " OR ".join(models_to_delete)}} @@ -315,21 +308,17 @@ def clear(self, models=None, commit=True): body=query, **self._get_doc_type_option(), ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -588,15 +577,13 @@ def search(self, query_string, **kwargs): _source=True, **self._get_doc_type_option(), ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to query Elasticsearch using '%s': %s", + self.log.exception( + "Failed to query Elasticsearch using '%s'", query_string, - e, - exc_info=True, ) raw_results = {} @@ -652,15 +639,13 @@ def more_like_this( **self._get_doc_type_option(), **params, ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} @@ -692,9 +677,11 @@ def _process_results( if raw_suggest: spelling_suggestion = " ".join( [ - word["text"] - if len(word["options"]) == 0 - else word["options"][0]["text"] + ( + word["text"] + if len(word["options"]) == 0 + else word["options"][0]["text"] + ) for word in raw_suggest ] ) @@ -971,7 +958,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] if isinstance(prepared_value, str): diff --git a/haystack/backends/simple_backend.py b/haystack/backends/simple_backend.py index a3bb59400..bfef88cb2 100644 --- a/haystack/backends/simple_backend.py +++ b/haystack/backends/simple_backend.py @@ -1,6 +1,7 @@ """ A very basic, ORM-based backend for simple search during tests. """ + from functools import reduce from warnings import warn @@ -56,7 +57,7 @@ def search(self, query_string, **kwargs): if hasattr(field, "related"): continue - if not field.get_internal_type() in ( + if field.get_internal_type() not in ( "TextField", "CharField", "SlugField", diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index f94492585..e077aa302 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -66,6 +66,9 @@ def __init__(self, connection_alias, **connection_options): self.collate = connection_options.get("COLLATE_SPELLING", True) + # Support to `date_facet` on Solr >= 6.6. Olders set `date` + self.date_facet_field = connection_options.get("DATE_FACET_FIELD", "range") + self.conn = Solr( connection_options["URL"], timeout=self.timeout, @@ -88,20 +91,19 @@ def update(self, index, iterable, commit=True): # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( + self.log.exception( "UnicodeDecodeError while preparing object for update", - exc_info=True, extra={"data": {"index": index, "object": get_identifier(obj)}}, ) if len(docs) > 0: try: self.conn.add(docs, commit=commit, boost=index.get_field_weights()) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error("Failed to add documents to Solr: %s", e, exc_info=True) + self.log.exception("Failed to add documents to Solr") def remove(self, obj_or_string, commit=True): solr_id = get_identifier(obj_or_string) @@ -109,15 +111,13 @@ def remove(self, obj_or_string, commit=True): try: kwargs = {"commit": commit, "id": solr_id} self.conn.delete(**kwargs) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Solr: %s", + self.log.exception( + "Failed to remove document '%s' from Solr", solr_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -139,19 +139,17 @@ def clear(self, models=None, commit=True): if commit: # Run an optimize post-clear. http://wiki.apache.org/solr/FAQ#head-9aafb5d8dff5308e8ea4fcf4b71f19f029c4bb99 self.conn.optimize() - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Solr index of models '%s': %s", + self.log.exception( + "Failed to clear Solr index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error("Failed to clear Solr index: %s", e, exc_info=True) + self.log.exception("Failed to clear Solr index") @log_query def search(self, query_string, **kwargs): @@ -162,13 +160,11 @@ def search(self, query_string, **kwargs): try: raw_results = self.conn.search(query_string, **search_kwargs) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to query Solr using '%s': %s", query_string, e, exc_info=True - ) + self.log.exception("Failed to query Solr using '%s'", query_string) raw_results = EmptyResults() return self._process_results( @@ -201,7 +197,6 @@ def build_search_kwargs( collate=None, **extra_kwargs ): - index = haystack.connections[self.connection_alias].get_unified_index() kwargs = {"fl": "* score", "df": index.document_field} @@ -272,21 +267,21 @@ def build_search_kwargs( for facet_field, options in facets.items(): for key, value in options.items(): - kwargs[ - "f.%s.facet.%s" % (facet_field, key) - ] = self.conn._from_python(value) + kwargs["f.%s.facet.%s" % (facet_field, key)] = ( + self.conn._from_python(value) + ) if date_facets is not None: kwargs["facet"] = "on" - kwargs["facet.date"] = date_facets.keys() - kwargs["facet.date.other"] = "none" + kwargs["facet.%s" % self.date_facet_field] = date_facets.keys() + kwargs["facet.%s.other" % self.date_facet_field] = "none" for key, value in date_facets.items(): - kwargs["f.%s.facet.date.start" % key] = self.conn._from_python( - value.get("start_date") + kwargs["f.%s.facet.%s.start" % (key, self.date_facet_field)] = ( + self.conn._from_python(value.get("start_date")) ) - kwargs["f.%s.facet.date.end" % key] = self.conn._from_python( - value.get("end_date") + kwargs["f.%s.facet.%s.end" % (key, self.date_facet_field)] = ( + self.conn._from_python(value.get("end_date")) ) gap_by_string = value.get("gap_by").upper() gap_string = "%d%s" % (value.get("gap_amount"), gap_by_string) @@ -294,9 +289,12 @@ def build_search_kwargs( if value.get("gap_amount") != 1: gap_string += "S" - kwargs["f.%s.facet.date.gap" % key] = "+%s/%s" % ( - gap_string, - gap_by_string, + kwargs["f.%s.facet.%s.gap" % (key, self.date_facet_field)] = ( + "+%s/%s" + % ( + gap_string, + gap_by_string, + ) ) if query_facets is not None: @@ -445,15 +443,12 @@ def more_like_this( try: raw_results = self.conn.more_like_this(query, field_name, **params) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Solr for document '%s': %s", - query, - e, - exc_info=True, + self.log.exception( + "Failed to fetch More Like This from Solr for document '%s'", query ) raw_results = EmptyResults() @@ -481,6 +476,7 @@ def _process_results( "fields": raw_results.facets.get("facet_fields", {}), "dates": raw_results.facets.get("facet_dates", {}), "queries": raw_results.facets.get("facet_queries", {}), + "ranges": raw_results.facets.get("facet_ranges", {}), } for key in ["fields"]: @@ -494,14 +490,23 @@ def _process_results( ) ) + for key in ["ranges"]: + for facet_field in facets[key]: + # Convert to a two-tuple, as Solr's json format returns a list of + # pairs. + facets[key][facet_field] = list( + zip( + facets[key][facet_field]["counts"][::2], + facets[key][facet_field]["counts"][1::2], + ) + ) + if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) - except Exception as exc: - self.log.error( + except Exception: + self.log.exception( "Error extracting spelling suggestions: %s", - exc, - exc_info=True, extra={"data": {"spellcheck": raw_results.spellcheck}}, ) @@ -730,11 +735,9 @@ def extract_file_contents(self, file_obj, **kwargs): try: return self.conn.extract(file_obj, **kwargs) - except Exception as e: + except Exception: self.log.warning( - "Unable to extract file contents: %s", - e, - exc_info=True, + "Unable to extract file contents", extra={"data": {"file": file_obj}}, ) return None @@ -802,7 +805,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] for possible_value in prepared_value.split(" "): diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 5c06e8750..f63ce100a 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -4,10 +4,10 @@ import shutil import threading import warnings +from datetime import date, datetime from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.utils.datetime_safe import date, datetime from django.utils.encoding import force_str from haystack.backends import ( @@ -130,7 +130,13 @@ def setup(self): # Make sure the index is there. if self.use_file_storage and not os.path.exists(self.path): - os.makedirs(self.path) + try: + os.makedirs(self.path) + except Exception: + raise IOError( + "The directory of your Whoosh index '%s' (cwd='%s') cannot be created for the current user/group." + % (self.path, os.getcwd()) + ) new_index = True if self.use_file_storage and not os.access(self.path, os.W_OK): @@ -270,16 +276,15 @@ def update(self, index, iterable, commit=True): try: writer.update_document(**doc) - except Exception as e: + except Exception: if not self.silently_fail: raise # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( - "%s while preparing object for update" % e.__class__.__name__, - exc_info=True, + self.log.exception( + "Preparing object for update", extra={"data": {"index": index, "object": get_identifier(obj)}}, ) @@ -298,15 +303,13 @@ def remove(self, obj_or_string, commit=True): try: self.index.delete_by_query(q=self.parser.parse('%s:"%s"' % (ID, whoosh_id))) - except Exception as e: + except Exception: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Whoosh: %s", + self.log.exception( + "Failed to remove document '%s' from Whoosh", whoosh_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -330,19 +333,17 @@ def clear(self, models=None, commit=True): self.index.delete_by_query( q=self.parser.parse(" OR ".join(models_to_delete)) ) - except Exception as e: + except Exception: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Whoosh index of models '%s': %s", + self.log.exception( + "Failed to clear Whoosh index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error("Failed to clear Whoosh index: %s", e, exc_info=True) + self.log.exception("Failed to clear Whoosh index") def delete_index(self): # Per the Whoosh mailing list, if wiping out everything from the index, @@ -461,7 +462,7 @@ def search( group_by += [ FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets ] - facet_types.update({facet: "fields" for facet in facets}) + facet_types.update(dict.fromkeys(facets, "fields")) if date_facets is not None: @@ -929,8 +930,7 @@ class WhooshSearchQuery(BaseSearchQuery): def _convert_datetime(self, date): if hasattr(date, "hour"): return force_str(date.strftime("%Y%m%d%H%M%S")) - else: - return force_str(date.strftime("%Y%m%d000000")) + return force_str(date.strftime("%Y%m%d000000")) def clean(self, query_fragment): """ @@ -1019,7 +1019,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] if isinstance(prepared_value, str): diff --git a/haystack/exceptions.py b/haystack/exceptions.py index 5c2c4b9a3..95d0bb92a 100644 --- a/haystack/exceptions.py +++ b/haystack/exceptions.py @@ -48,6 +48,7 @@ class SpatialError(HaystackError): class StatsError(HaystackError): "Raised when incorrect arguments have been provided for stats" + pass diff --git a/haystack/fields.py b/haystack/fields.py index 0965377ea..3531bf31b 100644 --- a/haystack/fields.py +++ b/haystack/fields.py @@ -1,8 +1,8 @@ +import datetime import re from inspect import ismethod from django.template import loader -from django.utils import datetime_safe from haystack.exceptions import SearchFieldError from haystack.utils import get_model_ct_tuple @@ -395,7 +395,7 @@ def convert(self, value): if match: data = match.groupdict() - return datetime_safe.date( + return datetime.date( int(data["year"]), int(data["month"]), int(data["day"]) ) else: @@ -428,7 +428,7 @@ def convert(self, value): if match: data = match.groupdict() - return datetime_safe.datetime( + return datetime.datetime( int(data["year"]), int(data["month"]), int(data["day"]), diff --git a/haystack/forms.py b/haystack/forms.py index af9dd6964..b7dd1e28c 100644 --- a/haystack/forms.py +++ b/haystack/forms.py @@ -1,5 +1,5 @@ from django import forms -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from django.utils.text import capfirst from django.utils.translation import gettext_lazy as _ @@ -12,7 +12,7 @@ def model_choices(using=DEFAULT_ALIAS): choices = [ - (get_model_ct(m), capfirst(smart_text(m._meta.verbose_name_plural))) + (get_model_ct(m), capfirst(smart_str(m._meta.verbose_name_plural))) for m in connections[using].get_unified_index().get_indexed_models() ] return sorted(choices, key=lambda x: x[1]) diff --git a/haystack/generic_views.py b/haystack/generic_views.py index ac3182a6e..655ea4f74 100644 --- a/haystack/generic_views.py +++ b/haystack/generic_views.py @@ -92,6 +92,7 @@ class FacetedSearchMixin(SearchMixin): form_class = FacetedSearchForm facet_fields = None + date_facet_fields = None def get_form_kwargs(self): kwargs = super().get_form_kwargs() @@ -107,6 +108,11 @@ def get_queryset(self): qs = super().get_queryset() for field in self.facet_fields: qs = qs.facet(field) + + if self.date_facet_fields: + for field in self.date_facet_fields: + qs = qs.date_facet(**field) + return qs @@ -122,8 +128,7 @@ def get(self, request, *args, **kwargs): if form.is_valid(): return self.form_valid(form) - else: - return self.form_invalid(form) + return self.form_invalid(form) class FacetedSearchView(FacetedSearchMixin, SearchView): diff --git a/haystack/management/commands/build_solr_schema.py b/haystack/management/commands/build_solr_schema.py index 21fd4c86b..0ff6215d1 100644 --- a/haystack/management/commands/build_solr_schema.py +++ b/haystack/management/commands/build_solr_schema.py @@ -111,7 +111,11 @@ def handle(self, **options): ) if reload_core: - core = settings.HAYSTACK_CONNECTIONS[using]["URL"].rsplit("/", 1)[-1] + core = ( + settings.HAYSTACK_CONNECTIONS[using]["URL"] + .rstrip("/") + .rsplit("/", 1)[-1] + ) if "ADMIN_URL" not in settings.HAYSTACK_CONNECTIONS[using]: raise ImproperlyConfigured( diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py old mode 100755 new mode 100644 index 6dc9155f5..070332ff8 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -81,7 +81,6 @@ def do_update( max_retries=DEFAULT_MAX_RETRIES, last_max_pk=None, ): - # Get a clone of the QuerySet so that the cache doesn't bloat up # in memory. Useful when reindexing large amounts of data. # the query must be ordered by PK in order to get the max PK in each batch @@ -144,13 +143,13 @@ def do_update( error_msg += " (pid %(pid)s): %(exc)s" if retries >= max_retries: - LOG.error(error_msg, error_context, exc_info=True) + LOG.exception(error_msg, error_context) raise elif verbosity >= 2: LOG.warning(error_msg, error_context, exc_info=True) # If going to try again, sleep a bit before - time.sleep(2 ** retries) + time.sleep(2**retries) # Clear out the DB connections queries because it bloats up RAM. reset_queries() diff --git a/haystack/query.py b/haystack/query.py index 1be64658f..0e49486dc 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -172,7 +172,6 @@ def post_process_results(self, results): for result in results: if self._load_all: - model_objects = loaded_objects.get(result.model, {}) # Try to coerce a primary key object that matches the models pk # We have to deal with semi-arbitrary keys being cast from strings (UUID, int, etc) @@ -195,9 +194,9 @@ def post_process_results(self, results): # No objects were returned -- possible due to SQS nesting such as # XYZ.objects.filter(id__gt=10) where the amount ignored are # exactly equal to the ITERATOR_LOAD_PER_QUERY - del self._result_cache[: len(results)] - self._ignored_result_count += len(results) - break + del self._result_cache[:1] + self._ignored_result_count += 1 + continue to_cache.append(result) @@ -314,8 +313,7 @@ def __getitem__(self, k): # Cache should be full enough for our needs. if is_slice: return self._result_cache[start:bound] - else: - return self._result_cache[start] + return self._result_cache[start] # Methods that return a SearchQuerySet. def all(self): # noqa A003 @@ -330,8 +328,7 @@ def filter(self, *args, **kwargs): # noqa A003 """Narrows the search based on certain attributes and the default operator.""" if DEFAULT_OPERATOR == "OR": return self.filter_or(*args, **kwargs) - else: - return self.filter_and(*args, **kwargs) + return self.filter_and(*args, **kwargs) def exclude(self, *args, **kwargs): """Narrows the search by ensuring certain attributes are not included.""" diff --git a/haystack/templatetags/more_like_this.py b/haystack/templatetags/more_like_this.py index 2cc22751d..3f710e9a0 100644 --- a/haystack/templatetags/more_like_this.py +++ b/haystack/templatetags/more_like_this.py @@ -42,9 +42,11 @@ def render(self, context): sqs = sqs[: self.limit] context[self.varname] = sqs - except Exception as exc: - logging.warning( - "Unhandled exception rendering %r: %s", self, exc, exc_info=True + except Exception: + logging.exception( + "Unhandled exception rendering %r", + self, + level=logging.WARNING, ) return "" @@ -73,7 +75,7 @@ def more_like_this(parser, token): """ bits = token.split_contents() - if not len(bits) in (4, 6, 8): + if len(bits) not in (4, 6, 8): raise template.TemplateSyntaxError( "'%s' tag requires either 3, 5 or 7 arguments." % bits[0] ) diff --git a/haystack/utils/__init__.py b/haystack/utils/__init__.py index 6e335352c..18d939c41 100644 --- a/haystack/utils/__init__.py +++ b/haystack/utils/__init__.py @@ -4,6 +4,7 @@ from django.conf import settings from haystack.constants import DJANGO_CT, DJANGO_ID, ID +from haystack.utils.highlighting import Highlighter # noqa: F401 IDENTIFIER_REGEX = re.compile(r"^[\w\d_]+\.[\w\d_]+\.[\w\d-]+$") diff --git a/haystack/utils/app_loading.py b/haystack/utils/app_loading.py old mode 100755 new mode 100644 diff --git a/haystack/utils/loading.py b/haystack/utils/loading.py index 216e485a1..d96af7125 100644 --- a/haystack/utils/loading.py +++ b/haystack/utils/loading.py @@ -338,7 +338,6 @@ def get_index_fieldname(self, field): return self._fieldnames.get(field) or field def get_index(self, model_klass): - indexes = self.get_indexes() if model_klass not in indexes: diff --git a/haystack/views.py b/haystack/views.py index fed1808ea..f203f5e3a 100644 --- a/haystack/views.py +++ b/haystack/views.py @@ -11,7 +11,6 @@ class SearchView: template = "search/search.html" - extra_context = {} query = "" results = EmptySearchQuerySet() request = None diff --git a/pyproject.toml b/pyproject.toml index 403009f96..d112a5b00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,106 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools>=61.2", + "setuptools_scm[toml]>=3.4", + "wheel", +] -[tool.black] -line_length=88 +[project] +name = "django-haystack" +description = "Pluggable search for Django." +readme = "README.rst" +authors = [{name = "Daniel Lindsley", email = "daniel@toastdriven.com"}] +license = "BSD-3-Clause" +license-files = ["LICENSE"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Django", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Utilities", +] +dynamic = [ + "version", +] +dependencies = [ + "Django>=3.2", + "packaging", +] +[project.optional-dependencies] +elasticsearch = [ + "elasticsearch<8,>=5", +] +testing = [ + "coverage", + "geopy==2", + "pysolr>=3.7", + "python-dateutil", + "requests", + "whoosh<3.0,>=2.5.4", +] +[project.urls] +Documentation = "https://django-haystack.readthedocs.io" +Homepage = "http://haystacksearch.org/" +Source = "https://github.com/django-haystack/django-haystack" + +[tool.setuptools] +packages = [ + "haystack", + "haystack.backends", + "haystack.management", + "haystack.management.commands", + "haystack.templatetags", + "haystack.utils", +] +include-package-data = false +# test-suite = "test_haystack.run_tests.run_all" # validate-pyproject-toml will complain +zip-safe = false + +[tool.setuptools.package-data] +haystack = [ + "templates/panels/*", + "templates/search_configuration/*", +] + +[tool.setuptools_scm] +fallback_version = "0.0.dev0" +write_to = "haystack/version.py" [tool.isort] known_first_party = ["haystack", "test_haystack"] profile = "black" multi_line_output = 3 -[tool.setuptools_scm] -fallback_version = "0.0.dev0" -write_to = "haystack/version.py" +[tool.ruff] +extend-exclude = ["test_haystack/*"] +line-length = 162 +target-version = "py38" + +[tool.ruff.lint] +ignore = ["B018", "B028", "B904", "B905"] +select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] + +[tool.ruff.lint.isort] +known-first-party = ["haystack", "test_haystack"] + +[tool.ruff.lint.mccabe] +max-complexity = 14 + +[tool.ruff.lint.pylint] +max-args = 20 +max-branches = 40 +max-returns = 8 +max-statements = 91 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index bae09868b..000000000 --- a/setup.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[pep8] -line_length=88 -exclude=docs - -[flake8] -line_length=88 -exclude=docs,tests -ignore=E203, E501, W503, D - -[options] -setup_requires = - setuptools_scm diff --git a/setup.py b/setup.py deleted file mode 100644 index 6033e8dfd..000000000 --- a/setup.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -install_requires = ["Django>=2.2"] - -tests_require = [ - "pysolr>=3.7.0", - "whoosh>=2.5.4,<3.0", - "python-dateutil", - "geopy==2.0.0", - "nose", - "coverage", - "requests", -] - -setup( - name="django-haystack", - use_scm_version=True, - description="Pluggable search for Django.", - author="Daniel Lindsley", - author_email="daniel@toastdriven.com", - long_description=open("README.rst", "r").read(), - url="http://haystacksearch.org/", - packages=[ - "haystack", - "haystack.backends", - "haystack.management", - "haystack.management.commands", - "haystack.templatetags", - "haystack.utils", - ], - package_data={ - "haystack": ["templates/panels/*", "templates/search_configuration/*"] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Web Environment", - "Framework :: Django", - "Framework :: Django :: 2.2", - "Framework :: Django :: 3.1", - "Framework :: Django :: 3.2", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Utilities", - ], - zip_safe=False, - install_requires=install_requires, - tests_require=tests_require, - extras_require={ - "elasticsearch": ["elasticsearch>=5,<6"], - }, - test_suite="test_haystack.run_tests.run_all", -) diff --git a/test_haystack/__init__.py b/test_haystack/__init__.py index 8e2707352..e69de29bb 100644 --- a/test_haystack/__init__.py +++ b/test_haystack/__init__.py @@ -1,27 +0,0 @@ -import os - -test_runner = None -old_config = None - -os.environ["DJANGO_SETTINGS_MODULE"] = "test_haystack.settings" - - -import django - -django.setup() - - -def setup(): - global test_runner - global old_config - - from django.test.runner import DiscoverRunner - - test_runner = DiscoverRunner() - test_runner.setup_test_environment() - old_config = test_runner.setup_databases() - - -def teardown(): - test_runner.teardown_databases(old_config) - test_runner.teardown_test_environment() diff --git a/test_haystack/core/admin.py b/test_haystack/core/admin.py index 3e374bc6b..404dbefbe 100644 --- a/test_haystack/core/admin.py +++ b/test_haystack/core/admin.py @@ -5,10 +5,8 @@ from .models import MockModel +@admin.register(MockModel) class MockModelAdmin(SearchModelAdmin): haystack_connection = "solr" date_hierarchy = "pub_date" list_display = ("author", "pub_date") - - -admin.site.register(MockModel, MockModelAdmin) diff --git a/test_haystack/core/fixtures/base_data.json b/test_haystack/core/fixtures/base_data.json index c2b5ad37f..d4f3fdad5 100644 --- a/test_haystack/core/fixtures/base_data.json +++ b/test_haystack/core/fixtures/base_data.json @@ -1,88 +1,86 @@ [ - { - "pk": 1, - "model": "core.mocktag", - "fields": { - "name": "primary" + { + "pk": 1, + "model": "core.mocktag", + "fields": { + "name": "primary" + } + }, + { + "pk": 2, + "model": "core.mocktag", + "fields": { + "name": "secondary" + } + }, + { + "pk": 1, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "bar", + "pub_date": "2009-03-17 06:00:00", + "tag": 1 + } + }, + { + "pk": 2, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "bar", + "pub_date": "2009-03-17 07:00:00", + "tag": 1 + } + }, + { + "pk": 3, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "bar", + "pub_date": "2009-03-17 08:00:00", + "tag": 2 + } + }, + { + "pk": "sometext", + "model": "core.charpkmockmodel", + "fields": {} + }, + { + "pk": "1234", + "model": "core.charpkmockmodel", + "fields": {} + }, + { + "pk": 1, + "model": "core.afifthmockmodel", + "fields": { + "author": "sam1", + "deleted": false + } + }, + { + "pk": 2, + "model": "core.afifthmockmodel", + "fields": { + "author": "sam2", + "deleted": true + } + }, + { + "pk": "53554c58-7051-4350-bcc9-dad75eb248a9", + "model": "core.uuidmockmodel", + "fields": { + "characteristics": "some text that was indexed" + } + }, + { + "pk": "77554c58-7051-4350-bcc9-dad75eb24888", + "model": "core.uuidmockmodel", + "fields": { + "characteristics": "more text that was indexed" + } } - }, - { - "pk": 2, - "model": "core.mocktag", - "fields": { - "name": "secondary" - } - }, - { - "pk": 1, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "bar", - "pub_date": "2009-03-17 06:00:00", - "tag": 1 - } - }, - { - "pk": 2, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "bar", - "pub_date": "2009-03-17 07:00:00", - "tag": 1 - } - }, - { - "pk": 3, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "bar", - "pub_date": "2009-03-17 08:00:00", - "tag": 2 - } - }, - { - "pk": "sometext", - "model": "core.charpkmockmodel", - "fields": { - } - }, - { - "pk": "1234", - "model": "core.charpkmockmodel", - "fields": { - } - }, - { - "pk": 1, - "model": "core.afifthmockmodel", - "fields": { - "author": "sam1", - "deleted": false - } - }, - { - "pk": 2, - "model": "core.afifthmockmodel", - "fields": { - "author": "sam2", - "deleted": true - } - }, - { - "pk": "53554c58-7051-4350-bcc9-dad75eb248a9", - "model": "core.uuidmockmodel", - "fields": { - "characteristics": "some text that was indexed" - } - }, - { - "pk": "77554c58-7051-4350-bcc9-dad75eb24888", - "model": "core.uuidmockmodel", - "fields": { - "characteristics": "more text that was indexed" - } - } ] diff --git a/test_haystack/core/fixtures/bulk_data.json b/test_haystack/core/fixtures/bulk_data.json index 4e721d8d4..40bbf8a9d 100644 --- a/test_haystack/core/fixtures/bulk_data.json +++ b/test_haystack/core/fixtures/bulk_data.json @@ -1,262 +1,262 @@ [ - { - "pk": 1, - "model": "core.mocktag", - "fields": { - "name": "search_test" + { + "pk": 1, + "model": "core.mocktag", + "fields": { + "name": "search_test" + } + }, + { + "pk": 1, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:", + "pub_date": "2009-06-18 06:00:00", + "tag": 1 + } + }, + { + "pk": 2, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", + "pub_date": "2009-07-17 00:30:00", + "tag": 1 + } + }, + { + "pk": 3, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Every custom ``SearchIndex`` requires there be one and only one field with ``document=True``. This is the primary field that will get passed to the backend for indexing. For this field, you'll then need to create a template at ``search/indexes/myapp/note_text.txt``. This allows you to customize the document that will be passed to the search backend for indexing. A sample template might look like:", + "pub_date": "2009-06-18 08:00:00", + "tag": 1 + } + }, + { + "pk": 4, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "The exception to this is the TemplateField class. This take either no arguments or an explicit template name to populate their contents. You can find more information about them in the SearchIndex API reference.", + "pub_date": "2009-07-17 01:30:00", + "tag": 1 + } + }, + { + "pk": 5, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "This will pull in the default URLconf for Haystack. It consists of a single URLconf that points to a SearchView instance. You can change this class\u2019s behavior by passing it any of several keyword arguments or override it entirely with your own view.", + "pub_date": "2009-07-17 02:30:00", + "tag": 1 + } + }, + { + "pk": 6, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "This will create a default SearchIndex instance, search through all of your INSTALLED_APPS for search_indexes.py and register all SearchIndexes with the default SearchIndex. If autodiscovery and inclusion of all indexes is not desirable, you can manually register models in the following manner:", + "pub_date": "2009-07-17 03:30:00", + "tag": 1 + } + }, + { + "pk": 7, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchBackend class handles interaction directly with the backend. The search query it performs is usually fed to it from a SearchQuery class that has been built for that backend. This class must be at least partially implemented on a per-backend basis and is usually accompanied by a SearchQuery class within the same module.", + "pub_date": "2009-07-17 04:30:00", + "tag": 1 + } + }, + { + "pk": 8, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "Takes a query to search on and returns dictionary. The query should be a string that is appropriate syntax for the backend. The returned dictionary should contain the keys \u2018results\u2019 and \u2018hits\u2019. The \u2018results\u2019 value should be an iterable of populated SearchResult objects. The \u2018hits\u2019 should be an integer count of the number of matched results the search backend found. This method MUST be implemented by each backend, as it will be highly specific to each one.", + "pub_date": "2009-07-17 05:30:00", + "tag": 1 + } + }, + { + "pk": 9, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchQuery class acts as an intermediary between SearchQuerySet\u2018s abstraction and SearchBackend\u2018s actual search. Given the metadata provided by SearchQuerySet, SearchQuery build the actual query and interacts with the SearchBackend on SearchQuerySet\u2018s behalf. This class must be at least partially implemented on a per-backend basis, as portions are highly specific to the backend. It usually is bundled with the accompanying SearchBackend.", + "pub_date": "2009-07-17 06:30:00", + "tag": 1 + } + }, + { + "pk": 10, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Most people will NOT have to use this class directly. SearchQuerySet handles all interactions with SearchQuery objects and provides a nicer interface to work with. Should you need advanced/custom behavior, you can supply your version of SearchQuery that overrides/extends the class in the manner you see fit. SearchQuerySet objects take a kwarg parameter query where you can pass in your class.", + "pub_date": "2009-07-17 07:30:00", + "tag": 1 + } + }, + { + "pk": 11, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchQuery object maintains a list of QueryFilter objects. Each filter object supports what field it looks up against, what kind of lookup (i.e. the __\u2019s), what value it\u2019s looking for and if it\u2019s a AND/OR/NOT. The SearchQuery object\u2019s \u201cbuild_query\u201d method should then iterate over that list and convert that to a valid query for the search backend.", + "pub_date": "2009-07-17 08:30:00", + "tag": 1 + } + }, + { + "pk": 12, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The SearchSite provides a way to collect the SearchIndexes that are relevant to the current site, much like ModelAdmins in the admin app. This allows you to register indexes on models you don\u2019t control (reusable apps, django.contrib, etc.) as well as customize on a per-site basis what indexes should be available (different indexes for different sites, same codebase).", + "pub_date": "2009-07-17 09:30:00", + "tag": 1 + } + }, + { + "pk": 13, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "If you need to narrow the indexes that get registered, you will need to manipulate a SearchSite. There are two ways to go about this, via either register or unregister. If you want most of the indexes but want to forgo a specific one(s), you can setup the main site via autodiscover then simply unregister the one(s) you don\u2019t want.:", + "pub_date": "2009-07-17 10:30:00", + "tag": 1 + } + }, + { + "pk": 14, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The SearchIndex class allows the application developer a way to provide data to the backend in a structured format. Developers familiar with Django\u2019s Form or Model classes should find the syntax for indexes familiar. This class is arguably the most important part of integrating Haystack into your application, as it has a large impact on the quality of the search results and how easy it is for users to find what they\u2019re looking for. Care and effort should be put into making your indexes the best they can be.", + "pub_date": "2009-07-17 11:30:00", + "tag": 1 + } + }, + { + "pk": 15, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "Unlike relational databases, most search engines supported by Haystack are primarily document-based. They focus on a single text blob which they tokenize, analyze and index. When searching, this field is usually the primary one that is searched. Further, the schema used by most engines is the same for all types of data added, unlike a relational database that has a table schema for each chunk of data. It may be helpful to think of your search index as something closer to a key-value store instead of imagining it in terms of a RDBMS.", + "pub_date": "2009-07-17 12:30:00", + "tag": 1 + } + }, + { + "pk": 16, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Common uses include storing pertinent data information, categorizations of the document, author information and related data. By adding fields for these pieces of data, you provide a means to further narrow/filter search terms. This can be useful from either a UI perspective (a better advanced search form) or from a developer standpoint (section-dependent search, off-loading certain tasks to search, et cetera).", + "pub_date": "2009-07-17 13:30:00", + "tag": 1 + } + }, + { + "pk": 17, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Most search engines that were candidates for inclusion in Haystack all had a central concept of a document that they indexed. These documents form a corpus within which to primarily search. Because this ideal is so central and most of Haystack is designed to have pluggable backends, it is important to ensure that all engines have at least a bare minimum of the data they need to function.", + "pub_date": "2009-07-17 14:30:00", + "tag": 1 + } + }, + { + "pk": 18, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "As a result, when creating a SearchIndex, at least one field must be marked with document=True. This signifies to Haystack that whatever is placed in this field while indexing is to be the primary text the search engine indexes. The name of this field can be almost anything, but text is one of the more common names used.", + "pub_date": "2009-07-17 15:30:00", + "tag": 1 + } + }, + { + "pk": 19, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "One shortcoming of the use of search is that you rarely have all or the most up-to-date information about an object in the index. As a result, when retrieving search results, you will likely have to access the object in the database to provide better information. However, this can also hit the database quite heavily (think .get(pk=result.id) per object). If your search is popular, this can lead to a big performance hit. There are two ways to prevent this. The first way is SearchQuerySet.load_all, which tries to group all similar objects and pull them though one query instead of many. This still hits the DB and incurs a performance penalty.", + "pub_date": "2009-07-17 16:30:00", + "tag": 1 + } + }, + { + "pk": 20, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The other option is to leverage stored fields. By default, all fields in Haystack are both indexed (searchable by the engine) and stored (retained by the engine and presented in the results). By using a stored field, you can store commonly used data in such a way that you don\u2019t need to hit the database when processing the search result to get more information. By the way: Jenny's number is 867-5309", + "pub_date": "2009-07-17 17:30:00", + "tag": 1 + } + }, + { + "pk": 21, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "For example, one great way to leverage this is to pre-rendering an object\u2019s search result template DURING indexing. You define an additional field, render a template with it and it follows the main indexed record into the index. Then, when that record is pulled when it matches a query, you can simply display the contents of that field, which avoids the database hit.:", + "pub_date": "2009-07-17 18:30:00", + "tag": 1 + } + }, + { + "pk": 22, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "However, sometimes, even more control over what gets placed in your index is needed. To facilitate this, SearchIndex objects have a \u2018preparation\u2019 stage that populates data just before it is indexed. You can hook into this phase in several ways. This should be very familiar to developers who have used Django\u2019s forms before as it loosely follows similar concepts, though the emphasis here is less on cleansing data from user input and more on making the data friendly to the search backend.", + "pub_date": "2009-07-17 19:30:00", + "tag": 1 + } + }, + { + "pk": 23, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Each SearchIndex gets a prepare method, which handles collecting all the data. This method should return a dictionary that will be the final data used by the search backend. Overriding this method is useful if you need to collect more than one piece of data or need to incorporate additional data that is not well represented by a single SearchField. An example might look like:", + "pub_date": "2009-07-17 20:30:00", + "tag": 1 + } + }, + { + "pk": 1, + "model": "core.anothermockmodel", + "fields": { + "author": "daniel3", + "pub_date": "2009-07-17 21:30:00" + } + }, + { + "pk": 2, + "model": "core.anothermockmodel", + "fields": { + "author": "daniel3", + "pub_date": "2009-07-17 22:30:00" + } + }, + { + "pk": 1, + "model": "core.ScoreMockModel", + "fields": { + "score": "42" + } } - }, - { - "pk": 1, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:", - "pub_date": "2009-06-18 06:00:00", - "tag": 1 - } - }, - { - "pk": 2, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - "pub_date": "2009-07-17 00:30:00", - "tag": 1 - } - }, - { - "pk": 3, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Every custom ``SearchIndex`` requires there be one and only one field with ``document=True``. This is the primary field that will get passed to the backend for indexing. For this field, you'll then need to create a template at ``search/indexes/myapp/note_text.txt``. This allows you to customize the document that will be passed to the search backend for indexing. A sample template might look like:", - "pub_date": "2009-06-18 08:00:00", - "tag": 1 - } - }, - { - "pk": 4, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "The exception to this is the TemplateField class. This take either no arguments or an explicit template name to populate their contents. You can find more information about them in the SearchIndex API reference.", - "pub_date": "2009-07-17 01:30:00", - "tag": 1 - } - }, - { - "pk": 5, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "This will pull in the default URLconf for Haystack. It consists of a single URLconf that points to a SearchView instance. You can change this class’s behavior by passing it any of several keyword arguments or override it entirely with your own view.", - "pub_date": "2009-07-17 02:30:00", - "tag": 1 - } - }, - { - "pk": 6, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "This will create a default SearchIndex instance, search through all of your INSTALLED_APPS for search_indexes.py and register all SearchIndexes with the default SearchIndex. If autodiscovery and inclusion of all indexes is not desirable, you can manually register models in the following manner:", - "pub_date": "2009-07-17 03:30:00", - "tag": 1 - } - }, - { - "pk": 7, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchBackend class handles interaction directly with the backend. The search query it performs is usually fed to it from a SearchQuery class that has been built for that backend. This class must be at least partially implemented on a per-backend basis and is usually accompanied by a SearchQuery class within the same module.", - "pub_date": "2009-07-17 04:30:00", - "tag": 1 - } - }, - { - "pk": 8, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "Takes a query to search on and returns dictionary. The query should be a string that is appropriate syntax for the backend. The returned dictionary should contain the keys ‘results’ and ‘hits’. The ‘results’ value should be an iterable of populated SearchResult objects. The ‘hits’ should be an integer count of the number of matched results the search backend found. This method MUST be implemented by each backend, as it will be highly specific to each one.", - "pub_date": "2009-07-17 05:30:00", - "tag": 1 - } - }, - { - "pk": 9, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchQuery class acts as an intermediary between SearchQuerySet‘s abstraction and SearchBackend‘s actual search. Given the metadata provided by SearchQuerySet, SearchQuery build the actual query and interacts with the SearchBackend on SearchQuerySet‘s behalf. This class must be at least partially implemented on a per-backend basis, as portions are highly specific to the backend. It usually is bundled with the accompanying SearchBackend.", - "pub_date": "2009-07-17 06:30:00", - "tag": 1 - } - }, - { - "pk": 10, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Most people will NOT have to use this class directly. SearchQuerySet handles all interactions with SearchQuery objects and provides a nicer interface to work with. Should you need advanced/custom behavior, you can supply your version of SearchQuery that overrides/extends the class in the manner you see fit. SearchQuerySet objects take a kwarg parameter query where you can pass in your class.", - "pub_date": "2009-07-17 07:30:00", - "tag": 1 - } - }, - { - "pk": 11, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchQuery object maintains a list of QueryFilter objects. Each filter object supports what field it looks up against, what kind of lookup (i.e. the __’s), what value it’s looking for and if it’s a AND/OR/NOT. The SearchQuery object’s “build_query” method should then iterate over that list and convert that to a valid query for the search backend.", - "pub_date": "2009-07-17 08:30:00", - "tag": 1 - } - }, - { - "pk": 12, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The SearchSite provides a way to collect the SearchIndexes that are relevant to the current site, much like ModelAdmins in the admin app. This allows you to register indexes on models you don’t control (reusable apps, django.contrib, etc.) as well as customize on a per-site basis what indexes should be available (different indexes for different sites, same codebase).", - "pub_date": "2009-07-17 09:30:00", - "tag": 1 - } - }, - { - "pk": 13, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "If you need to narrow the indexes that get registered, you will need to manipulate a SearchSite. There are two ways to go about this, via either register or unregister. If you want most of the indexes but want to forgo a specific one(s), you can setup the main site via autodiscover then simply unregister the one(s) you don’t want.:", - "pub_date": "2009-07-17 10:30:00", - "tag": 1 - } - }, - { - "pk": 14, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The SearchIndex class allows the application developer a way to provide data to the backend in a structured format. Developers familiar with Django’s Form or Model classes should find the syntax for indexes familiar. This class is arguably the most important part of integrating Haystack into your application, as it has a large impact on the quality of the search results and how easy it is for users to find what they’re looking for. Care and effort should be put into making your indexes the best they can be.", - "pub_date": "2009-07-17 11:30:00", - "tag": 1 - } - }, - { - "pk": 15, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "Unlike relational databases, most search engines supported by Haystack are primarily document-based. They focus on a single text blob which they tokenize, analyze and index. When searching, this field is usually the primary one that is searched. Further, the schema used by most engines is the same for all types of data added, unlike a relational database that has a table schema for each chunk of data. It may be helpful to think of your search index as something closer to a key-value store instead of imagining it in terms of a RDBMS.", - "pub_date": "2009-07-17 12:30:00", - "tag": 1 - } - }, - { - "pk": 16, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Common uses include storing pertinent data information, categorizations of the document, author information and related data. By adding fields for these pieces of data, you provide a means to further narrow/filter search terms. This can be useful from either a UI perspective (a better advanced search form) or from a developer standpoint (section-dependent search, off-loading certain tasks to search, et cetera).", - "pub_date": "2009-07-17 13:30:00", - "tag": 1 - } - }, - { - "pk": 17, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Most search engines that were candidates for inclusion in Haystack all had a central concept of a document that they indexed. These documents form a corpus within which to primarily search. Because this ideal is so central and most of Haystack is designed to have pluggable backends, it is important to ensure that all engines have at least a bare minimum of the data they need to function.", - "pub_date": "2009-07-17 14:30:00", - "tag": 1 - } - }, - { - "pk": 18, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "As a result, when creating a SearchIndex, at least one field must be marked with document=True. This signifies to Haystack that whatever is placed in this field while indexing is to be the primary text the search engine indexes. The name of this field can be almost anything, but text is one of the more common names used.", - "pub_date": "2009-07-17 15:30:00", - "tag": 1 - } - }, - { - "pk": 19, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "One shortcoming of the use of search is that you rarely have all or the most up-to-date information about an object in the index. As a result, when retrieving search results, you will likely have to access the object in the database to provide better information. However, this can also hit the database quite heavily (think .get(pk=result.id) per object). If your search is popular, this can lead to a big performance hit. There are two ways to prevent this. The first way is SearchQuerySet.load_all, which tries to group all similar objects and pull them though one query instead of many. This still hits the DB and incurs a performance penalty.", - "pub_date": "2009-07-17 16:30:00", - "tag": 1 - } - }, - { - "pk": 20, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The other option is to leverage stored fields. By default, all fields in Haystack are both indexed (searchable by the engine) and stored (retained by the engine and presented in the results). By using a stored field, you can store commonly used data in such a way that you don’t need to hit the database when processing the search result to get more information. By the way: Jenny's number is 867-5309", - "pub_date": "2009-07-17 17:30:00", - "tag": 1 - } - }, - { - "pk": 21, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "For example, one great way to leverage this is to pre-rendering an object’s search result template DURING indexing. You define an additional field, render a template with it and it follows the main indexed record into the index. Then, when that record is pulled when it matches a query, you can simply display the contents of that field, which avoids the database hit.:", - "pub_date": "2009-07-17 18:30:00", - "tag": 1 - } - }, - { - "pk": 22, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "However, sometimes, even more control over what gets placed in your index is needed. To facilitate this, SearchIndex objects have a ‘preparation’ stage that populates data just before it is indexed. You can hook into this phase in several ways. This should be very familiar to developers who have used Django’s forms before as it loosely follows similar concepts, though the emphasis here is less on cleansing data from user input and more on making the data friendly to the search backend.", - "pub_date": "2009-07-17 19:30:00", - "tag": 1 - } - }, - { - "pk": 23, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Each SearchIndex gets a prepare method, which handles collecting all the data. This method should return a dictionary that will be the final data used by the search backend. Overriding this method is useful if you need to collect more than one piece of data or need to incorporate additional data that is not well represented by a single SearchField. An example might look like:", - "pub_date": "2009-07-17 20:30:00", - "tag": 1 - } - }, - { - "pk": 1, - "model": "core.anothermockmodel", - "fields": { - "author": "daniel3", - "pub_date": "2009-07-17 21:30:00" - } - }, - { - "pk": 2, - "model": "core.anothermockmodel", - "fields": { - "author": "daniel3", - "pub_date": "2009-07-17 22:30:00" - } - }, - { - "pk": 1, - "model": "core.ScoreMockModel", - "fields": { - "score": "42" - } - } ] diff --git a/test_haystack/core/templates/404.html b/test_haystack/core/templates/404.html index 838aa183a..21f5da261 100644 --- a/test_haystack/core/templates/404.html +++ b/test_haystack/core/templates/404.html @@ -1 +1 @@ -{% extends 'base.html' %} \ No newline at end of file +{% extends 'base.html' %} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt index d2cca8d7d..a8f8b85b3 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt @@ -1,2 +1,2 @@ Stored! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/search.html b/test_haystack/core/templates/search/search.html index 838aa183a..21f5da261 100644 --- a/test_haystack/core/templates/search/search.html +++ b/test_haystack/core/templates/search/search.html @@ -1 +1 @@ -{% extends 'base.html' %} \ No newline at end of file +{% extends 'base.html' %} diff --git a/test_haystack/core/templates/test_suggestion.html b/test_haystack/core/templates/test_suggestion.html index fa6240381..58df73ee3 100644 --- a/test_haystack/core/templates/test_suggestion.html +++ b/test_haystack/core/templates/test_suggestion.html @@ -1 +1 @@ -Suggestion: {{ suggestion }} \ No newline at end of file +Suggestion: {{ suggestion }} diff --git a/test_haystack/discovery/templates/search/indexes/bar_text.txt b/test_haystack/discovery/templates/search/indexes/bar_text.txt index 07070f0c9..4665cb553 100644 --- a/test_haystack/discovery/templates/search/indexes/bar_text.txt +++ b/test_haystack/discovery/templates/search/indexes/bar_text.txt @@ -1,2 +1,2 @@ {{ object.title }} -{{ object.body }} \ No newline at end of file +{{ object.body }} diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py deleted file mode 100644 index 67a9e9764..000000000 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -import unittest -import warnings - -from django.conf import settings - -from haystack.utils import log as logging - -warnings.simplefilter("ignore", Warning) - - -def setup(): - log = logging.getLogger("haystack") - try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch import Elasticsearch, exceptions - except ImportError: - log.error( - "Skipping ElasticSearch 2 tests: 'elasticsearch>=2.0.0,<3.0.0' not installed." - ) - raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") - - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - es = Elasticsearch(url) - try: - es.info() - except exceptions.ConnectionError as e: - log.error("elasticsearch not running on %r" % url, exc_info=True) - raise unittest.SkipTest("elasticsearch not running on %r" % url, e) diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py deleted file mode 100644 index aa2e9d7a5..000000000 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ /dev/null @@ -1,1816 +0,0 @@ -import datetime -import logging as std_logging -import operator -import pickle -import unittest -from decimal import Decimal - -import elasticsearch -from django.apps import apps -from django.conf import settings -from django.test import TestCase -from django.test.utils import override_settings - -from haystack import connections, indexes, reset_search_queries -from haystack.exceptions import SkipDocument -from haystack.inputs import AutoQuery -from haystack.models import SearchResult -from haystack.query import SQ, RelatedSearchQuerySet, SearchQuerySet -from haystack.utils import log as logging -from haystack.utils.loading import UnifiedIndex - -from ..core.models import AFourthMockModel, AnotherMockModel, ASixthMockModel, MockModel -from ..mocks import MockSearchResult - - -def clear_elasticsearch_index(): - # Wipe it clean. - raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - try: - raw_es.indices.delete( - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - raw_es.indices.refresh() - except elasticsearch.TransportError: - pass - - # Since we've just completely deleted the index, we'll reset setup_complete so the next access will - # correctly define the mappings: - connections["elasticsearch"].get_backend().setup_complete = False - - -class Elasticsearch2MockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2MockSearchIndexWithSkipDocument(Elasticsearch2MockSearchIndex): - def prepare_text(self, obj): - if obj.author == "daniel3": - raise SkipDocument - return "Indexed!\n%s" % obj.id - - -class Elasticsearch2MockSpellingIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - def prepare_text(self, obj): - return obj.foo - - -class Elasticsearch2MaintainTypeMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - month = indexes.CharField(indexed=False) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def prepare_month(self, obj): - return "%02d" % obj.pub_date.month - - def get_model(self): - return MockModel - - -class Elasticsearch2MockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2AnotherMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AnotherMockModel - - def prepare_text(self, obj): - return "You might be searching for the user %s" % obj.author - - -class Elasticsearch2BoostMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField( - document=True, - use_template=True, - template_name="search/indexes/core/mockmodel_template.txt", - ) - author = indexes.CharField(model_attr="author", weight=2.0) - editor = indexes.CharField(model_attr="editor") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AFourthMockModel - - def prepare(self, obj): - data = super().prepare(obj) - - if obj.pk == 4: - data["boost"] = 5.0 - - return data - - -class Elasticsearch2FacetingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - author = indexes.CharField(model_attr="author", faceted=True) - editor = indexes.CharField(model_attr="editor", faceted=True) - pub_date = indexes.DateField(model_attr="pub_date", faceted=True) - facet_field = indexes.FacetCharField(model_attr="author") - - def prepare_text(self, obj): - return "%s %s" % (obj.author, obj.editor) - - def get_model(self): - return AFourthMockModel - - -class Elasticsearch2RoundTripSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, default="") - name = indexes.CharField() - is_active = indexes.BooleanField() - post_count = indexes.IntegerField() - average_rating = indexes.FloatField() - price = indexes.DecimalField() - pub_date = indexes.DateField() - created = indexes.DateTimeField() - tags = indexes.MultiValueField() - sites = indexes.MultiValueField() - - def get_model(self): - return MockModel - - def prepare(self, obj): - prepped = super().prepare(obj) - prepped.update( - { - "text": "This is some example text.", - "name": "Mister Pants", - "is_active": True, - "post_count": 25, - "average_rating": 3.6, - "price": Decimal("24.99"), - "pub_date": datetime.date(2009, 11, 21), - "created": datetime.datetime(2009, 11, 21, 21, 31, 00), - "tags": ["staff", "outdoor", "activist", "scientist"], - "sites": [3, 5, 1], - } - ) - return prepped - - -class Elasticsearch2ComplexFacetsMockSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(document=True, default="") - name = indexes.CharField(faceted=True) - is_active = indexes.BooleanField(faceted=True) - post_count = indexes.IntegerField() - post_count_i = indexes.FacetIntegerField(facet_for="post_count") - average_rating = indexes.FloatField(faceted=True) - pub_date = indexes.DateField(faceted=True) - created = indexes.DateTimeField(faceted=True) - sites = indexes.MultiValueField(faceted=True) - - def get_model(self): - return MockModel - - -class Elasticsearch2AutocompleteMockModelSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - text_auto = indexes.EdgeNgramField(model_attr="foo") - name_auto = indexes.EdgeNgramField(model_attr="author") - - def get_model(self): - return MockModel - - -class Elasticsearch2SpatialSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="name", document=True) - location = indexes.LocationField() - - def prepare_location(self, obj): - return "%s,%s" % (obj.lat, obj.lon) - - def get_model(self): - return ASixthMockModel - - -class TestSettings(TestCase): - def test_kwargs_are_passed_on(self): - from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend - - backend = ElasticsearchSearchBackend( - "alias", - **{ - "URL": settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], - "INDEX_NAME": "testing", - "KWARGS": {"max_retries": 42}, - } - ) - - self.assertEqual(backend.conn.transport.max_retries, 42) - - -class Elasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.smmidni = Elasticsearch2MockSearchIndexWithSkipDocument() - self.smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - self.sb.silently_fail = True - - def raw_search(self, query): - try: - return self.raw_es.search( - q="*:*", - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"], - ) - except elasticsearch.TransportError: - return {} - - def test_non_silent(self): - bad_sb = connections["elasticsearch"].backend( - "bad", - URL="http://omg.wtf.bbq:1000/", - INDEX_NAME="whatver", - SILENTLY_FAIL=False, - TIMEOUT=1, - ) - - try: - bad_sb.update(self.smmi, self.sample_objs) - self.fail() - except: - pass - - try: - bad_sb.remove("core.mockmodel.1") - self.fail() - except: - pass - - try: - bad_sb.clear() - self.fail() - except: - pass - - try: - bad_sb.search("foo") - self.fail() - except: - pass - - def test_update_no_documents(self): - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - index_name = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True - ) - self.assertEqual(sb.update(self.smmi, []), None) - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=False - ) - try: - sb.update(self.smmi, []) - self.fail() - except: - pass - - def test_update(self): - self.sb.update(self.smmi, self.sample_objs) - - # Check what Elasticsearch thinks is there. - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=lambda x: x["id"], - ), - [ - { - "django_id": "1", - "django_ct": "core.mockmodel", - "name": "daniel1", - "name_exact": "daniel1", - "text": "Indexed!\n1", - "pub_date": "2009-02-24T00:00:00", - "id": "core.mockmodel.1", - }, - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_update_with_SkipDocument_raised(self): - self.sb.update(self.smmidni, self.sample_objs) - - # Check what Elasticsearch thinks is there. - res = self.raw_search("*:*")["hits"] - self.assertEqual(res["total"], 2) - self.assertListEqual( - sorted([x["_source"]["id"] for x in res["hits"]]), - ["core.mockmodel.1", "core.mockmodel.2"], - ) - - def test_remove(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 2) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=operator.itemgetter("django_id"), - ), - [ - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_remove_succeeds_on_404(self): - self.sb.silently_fail = False - self.sb.remove("core.mockmodel.421") - - def test_clear(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear() - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel, MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - def test_search(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - set([result.pk for result in self.sb.search("*:*")["results"]]), - {"2", "1", "3"}, - ) - - self.assertEqual(self.sb.search("", highlight=True), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("Index", highlight=True)["hits"], 3) - self.assertEqual( - sorted( - [ - result.highlighted[0] - for result in self.sb.search("Index", highlight=True)["results"] - ] - ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], - ) - - self.assertEqual(self.sb.search("Indx")["hits"], 0) - self.assertEqual(self.sb.search("indaxed")["spelling_suggestion"], "indexed") - self.assertEqual( - self.sb.search("arf", spelling_query="indexyd")["spelling_suggestion"], - "indexed", - ) - - self.assertEqual( - self.sb.search("", facets={"name": {}}), {"hits": 0, "results": []} - ) - results = self.sb.search("Index", facets={"name": {}}) - self.assertEqual(results["hits"], 3) - self.assertSetEqual( - set(results["facets"]["fields"]["name"]), - {("daniel3", 1), ("daniel2", 1), ("daniel1", 1)}, - ) - - self.assertEqual( - self.sb.search( - "", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ), - {"hits": 0, "results": []}, - ) - results = self.sb.search( - "Index", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ) - self.assertEqual(results["hits"], 3) - self.assertEqual( - results["facets"]["dates"]["pub_date"], - [(datetime.datetime(2009, 2, 1, 0, 0), 3)], - ) - - self.assertEqual( - self.sb.search("", query_facets=[("name", "[* TO e]")]), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", query_facets=[("name", "[* TO e]")]) - self.assertEqual(results["hits"], 3) - self.assertEqual(results["facets"]["queries"], {"name": 3}) - - self.assertEqual( - self.sb.search("", narrow_queries={"name:daniel1"}), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", narrow_queries={"name:daniel1"}) - self.assertEqual(results["hits"], 1) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sb.search("index", result_class=MockSearchResult)["results"][0], - MockSearchResult, - ) - ) - - # Check the use of ``limit_to_registered_models``. - self.assertEqual( - self.sb.search("", limit_to_registered_models=False), - {"hits": 0, "results": []}, - ) - self.assertEqual( - self.sb.search("*:*", limit_to_registered_models=False)["hits"], 3 - ) - self.assertEqual( - sorted( - [ - result.pk - for result in self.sb.search( - "*:*", limit_to_registered_models=False - )["results"] - ] - ), - ["1", "2", "3"], - ) - - # Stow. - old_limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = False - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - sorted([result.pk for result in self.sb.search("*:*")["results"]]), - ["1", "2", "3"], - ) - - # Restore. - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models - - def test_spatial_search_parameters(self): - from django.contrib.gis.geos import Point - - p1 = Point(1.23, 4.56) - kwargs = self.sb.build_search_kwargs( - "*:*", - distance_point={"field": "location", "point": p1}, - sort_by=(("distance", "desc"),), - ) - - self.assertIn("sort", kwargs) - self.assertEqual(1, len(kwargs["sort"])) - geo_d = kwargs["sort"][0]["_geo_distance"] - - # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be - # in the same order as we used to create the Point(): - # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4 - - self.assertDictEqual( - geo_d, {"location": [1.23, 4.56], "unit": "km", "order": "desc"} - ) - - def test_more_like_this(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - # A functional MLT example with enough data to work is below. Rely on - # this to ensure the API is correct enough. - self.assertEqual(self.sb.more_like_this(self.sample_objs[0])["hits"], 0) - self.assertEqual( - [ - result.pk - for result in self.sb.more_like_this(self.sample_objs[0])["results"] - ], - [], - ) - - def test_build_schema(self): - old_ui = connections["elasticsearch"].get_unified_index() - - (content_field_name, mapping) = self.sb.build_schema(old_ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 4 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - }, - ) - - ui = UnifiedIndex() - ui.build(indexes=[Elasticsearch2ComplexFacetsMockSearchIndex()]) - (content_field_name, mapping) = self.sb.build_schema(ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 15 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name": {"type": "string", "analyzer": "snowball"}, - "is_active_exact": {"type": "boolean"}, - "created": {"type": "date"}, - "post_count": {"type": "long"}, - "created_exact": {"type": "date"}, - "sites_exact": {"index": "not_analyzed", "type": "string"}, - "is_active": {"type": "boolean"}, - "sites": {"type": "string", "analyzer": "snowball"}, - "post_count_i": {"type": "long"}, - "average_rating": {"type": "float"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date_exact": {"type": "date"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - "pub_date": {"type": "date"}, - "average_rating_exact": {"type": "float"}, - }, - ) - - def test_verify_type(self): - old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - ui.build(indexes=[smtmmi]) - connections["elasticsearch"]._index = ui - sb = connections["elasticsearch"].get_backend() - sb.update(smtmmi, self.sample_objs) - - self.assertEqual(sb.search("*:*")["hits"], 3) - self.assertEqual( - [result.month for result in sb.search("*:*")["results"]], ["02", "02", "02"] - ) - connections["elasticsearch"]._index = old_ui - - -class CaptureHandler(std_logging.Handler): - logs_seen = [] - - def emit(self, record): - CaptureHandler.logs_seen.append(record) - - -class FailedElasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - # Stow. - # Point the backend at a URL that doesn't exist so we can watch the - # sparks fly. - self.old_es_url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = ( - "%s/foo/" % self.old_es_url - ) - self.cap = CaptureHandler() - logging.getLogger("haystack").addHandler(self.cap) - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(config.stream) - - # Setup the rest of the bits. - self.old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = ui - self.sb = connections["elasticsearch"].get_backend() - - def tearDown(self): - # Restore. - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = self.old_es_url - connections["elasticsearch"]._index = self.old_ui - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(self.cap) - logging.getLogger("haystack").addHandler(config.stream) - - @unittest.expectedFailure - def test_all_cases(self): - # Prior to the addition of the try/except bits, these would all fail miserably. - self.assertEqual(len(CaptureHandler.logs_seen), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(len(CaptureHandler.logs_seen), 1) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 2) - - self.sb.search("search") - self.assertEqual(len(CaptureHandler.logs_seen), 3) - - self.sb.more_like_this(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 4) - - self.sb.clear([MockModel]) - self.assertEqual(len(CaptureHandler.logs_seen), 5) - - self.sb.clear() - self.assertEqual(len(CaptureHandler.logs_seen), 6) - - -class LiveElasticsearch2SearchQueryTestCase(TestCase): - fixtures = ["base_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - self.sq = connections["elasticsearch"].get_query() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_log_query(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=False): - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=True): - # Redefine it to clear out the cached results. - self.sq = connections["elasticsearch"].query(using="elasticsearch") - self.sq.add_filter(SQ(name="bar")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - - # And again, for good measure. - self.sq = connections["elasticsearch"].query("elasticsearch") - self.sq.add_filter(SQ(name="bar")) - self.sq.add_filter(SQ(text="moof")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - self.assertEqual( - connections["elasticsearch"].queries[1]["query_string"], - "(name:(bar) AND text:(moof))", - ) - - -lssqstc_all_loaded = None - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SearchQuerySetTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - self.rsqs = RelatedSearchQuerySet("elasticsearch") - - # Ugly but not constantly reindexing saves us almost 50% runtime. - global lssqstc_all_loaded - - if lssqstc_all_loaded is None: - lssqstc_all_loaded = True - - # Wipe it clean. - clear_elasticsearch_index() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_load_all(self): - sqs = self.sqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - results = sorted([int(result.pk) for result in sqs]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_values_slicing(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends - - # The values will come back as strings because Hasytack doesn't assume PKs are integers. - # We'll prepare this set once since we're going to query the same results in multiple ways: - expected_pks = [str(i) for i in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]] - - results = self.sqs.all().order_by("pub_date").values("pk") - self.assertListEqual([i["pk"] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk") - self.assertListEqual([i[0] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk", flat=True) - self.assertListEqual(results[1:11], expected_pks) - - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_count(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - self.assertEqual(sqs.count(), 23) - self.assertEqual(sqs.count(), 23) - self.assertEqual(len(sqs), 23) - self.assertEqual(sqs.count(), 23) - # Should only execute one query to count the length of the result set. - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_manual_iter(self): - results = self.sqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = set([int(result.pk) for result in results._manual_iter()]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.sqs._cache_is_full(), False) - results = self.sqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test___and__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 & sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) AND (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar") - sqs = sqs3 & sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 3) - self.assertEqual( - sqs.query.build_query(), - "(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))", - ) - - def test___or__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 | sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) OR (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar").models(MockModel) - sqs = sqs3 | sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual( - sqs.query.build_query(), - "((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))", - ) - - def test_auto_query(self): - # Ensure bits in exact matches get escaped properly as well. - # This will break horrifically if escaping isn't working. - sqs = self.sqs.auto_query('"pants:rule"') - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual( - repr(sqs.query.query_filter), '' - ) - self.assertEqual(sqs.query.build_query(), '("pants\\:rule")') - self.assertEqual(len(sqs), 0) - - # Regressions - - def test_regression_proper_start_offsets(self): - sqs = self.sqs.filter(text="index") - self.assertNotEqual(sqs.count(), 0) - - id_counts = {} - - for item in sqs: - if item.id in id_counts: - id_counts[item.id] += 1 - else: - id_counts[item.id] = 1 - - for key, value in id_counts.items(): - if value > 1: - self.fail( - "Result with id '%s' seen more than once in the results." % key - ) - - def test_regression_raw_search_breaks_slicing(self): - sqs = self.sqs.raw_search("text:index") - page_1 = [result.pk for result in sqs[0:10]] - page_2 = [result.pk for result in sqs[10:20]] - - for pk in page_2: - if pk in page_1: - self.fail( - "Result with id '%s' seen more than once in the results." % pk - ) - - # RelatedSearchQuerySet Tests - - def test_related_load_all(self): - sqs = self.rsqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_related_load_all_queryset(self): - sqs = self.rsqs.load_all().order_by("pub_date") - self.assertEqual(len(sqs._load_all_querysets), 0) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24))) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual( - set([obj.object.id for obj in sqs]), - {12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20}, - ) - self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), {21, 22, 23}) - - def test_related_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.rsqs.all() - results = set([int(result.pk) for result in sqs]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - set([int(result.pk) for result in results[20:30]]), {21, 22, 23} - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_related_manual_iter(self): - results = self.rsqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = sorted([int(result.pk) for result in results._manual_iter()]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_related_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.rsqs._cache_is_full(), False) - results = self.rsqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_quotes_regression(self): - sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E") - # Should not have empty terms. - self.assertEqual(sqs.query.build_query(), "(44\xb048'40''N 20\xb028'32''E)") - # Should not cause Elasticsearch to 500. - self.assertEqual(sqs.count(), 0) - - sqs = self.sqs.auto_query("blazing") - self.assertEqual(sqs.query.build_query(), "(blazing)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("blazing saddles") - self.assertEqual(sqs.query.build_query(), "(blazing saddles)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles') - self.assertEqual(sqs.query.build_query(), '(\\"blazing saddles)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles"') - self.assertEqual(sqs.query.build_query(), '("blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing \'saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing \'saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" ')") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" '\\\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel brooks') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" \\"brooks)') - self.assertEqual(sqs.count(), 0) - - def test_query_generation(self): - sqs = self.sqs.filter( - SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")) - ) - self.assertEqual( - sqs.query.build_query(), "((hello world) OR title:(hello world))" - ) - - def test_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - sqs = self.sqs.all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - # Custom class. - sqs = self.sqs.result_class(MockSearchResult).all() - self.assertTrue(isinstance(sqs[0], MockSearchResult)) - - # Reset to default. - sqs = self.sqs.result_class(None).all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SpellingTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSpellingIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_spelling(self): - self.assertEqual( - self.sqs.auto_query("structurd").spelling_suggestion(), "structured" - ) - self.assertEqual(self.sqs.spelling_suggestion("structurd"), "structured") - self.assertEqual( - self.sqs.auto_query("srchindex instanc").spelling_suggestion(), - "searchindex instance", - ) - self.assertEqual( - self.sqs.spelling_suggestion("srchindex instanc"), "searchindex instance" - ) - - -class LiveElasticsearch2MoreLikeThisTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_more_like_this(self): - mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1)) - results = [result.pk for result in mlt] - self.assertEqual(mlt.count(), 11) - self.assertEqual( - set(results), {"10", "5", "2", "21", "4", "6", "23", "9", "14"} - ) - self.assertEqual(len(results), 10) - - alt_mlt = self.sqs.filter(name="daniel3").more_like_this( - MockModel.objects.get(pk=2) - ) - results = [result.pk for result in alt_mlt] - self.assertEqual(alt_mlt.count(), 9) - self.assertEqual( - set(results), {"2", "16", "3", "19", "4", "17", "10", "22", "23"} - ) - self.assertEqual(len(results), 9) - - alt_mlt_with_models = self.sqs.models(MockModel).more_like_this( - MockModel.objects.get(pk=1) - ) - results = [result.pk for result in alt_mlt_with_models] - self.assertEqual(alt_mlt_with_models.count(), 10) - self.assertEqual( - set(results), {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"} - ) - self.assertEqual(len(results), 10) - - if hasattr(MockModel.objects, "defer"): - # Make sure MLT works with deferred bits. - qs = MockModel.objects.defer("foo") - self.assertEqual(qs.query.deferred_loading[1], True) - deferred = self.sqs.models(MockModel).more_like_this(qs.get(pk=1)) - self.assertEqual(deferred.count(), 10) - self.assertEqual( - {result.pk for result in deferred}, - {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"}, - ) - self.assertEqual(len([result.pk for result in deferred]), 10) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sqs.result_class(MockSearchResult).more_like_this( - MockModel.objects.get(pk=1) - )[0], - MockSearchResult, - ) - ) - - -class LiveElasticsearch2AutocompleteTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2AutocompleteMockModelSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_build_schema(self): - self.sb = connections["elasticsearch"].get_backend() - content_name, mapping = self.sb.build_schema(self.ui.all_searchfields()) - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "text_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - }, - ) - - def test_autocomplete(self): - autocomplete = self.sqs.autocomplete(text_auto="mod") - self.assertEqual(autocomplete.count(), 16) - self.assertEqual( - set([result.pk for result in autocomplete]), - { - "1", - "12", - "6", - "14", - "7", - "4", - "23", - "17", - "13", - "18", - "20", - "22", - "19", - "15", - "10", - "2", - }, - ) - self.assertTrue("mod" in autocomplete[0].text.lower()) - self.assertTrue("mod" in autocomplete[1].text.lower()) - self.assertTrue("mod" in autocomplete[2].text.lower()) - self.assertTrue("mod" in autocomplete[3].text.lower()) - self.assertTrue("mod" in autocomplete[4].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete]), 16) - - # Test multiple words. - autocomplete_2 = self.sqs.autocomplete(text_auto="your mod") - self.assertEqual(autocomplete_2.count(), 13) - self.assertEqual( - set([result.pk for result in autocomplete_2]), - {"1", "6", "2", "14", "12", "13", "10", "19", "4", "20", "23", "22", "15"}, - ) - map_results = {result.pk: result for result in autocomplete_2} - self.assertTrue("your" in map_results["1"].text.lower()) - self.assertTrue("mod" in map_results["1"].text.lower()) - self.assertTrue("your" in map_results["6"].text.lower()) - self.assertTrue("mod" in map_results["6"].text.lower()) - self.assertTrue("your" in map_results["2"].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete_2]), 13) - - # Test multiple fields. - autocomplete_3 = self.sqs.autocomplete(text_auto="Django", name_auto="dan") - self.assertEqual(autocomplete_3.count(), 4) - self.assertEqual( - set([result.pk for result in autocomplete_3]), {"12", "1", "22", "14"} - ) - self.assertEqual(len([result.pk for result in autocomplete_3]), 4) - - # Test numbers in phrases - autocomplete_4 = self.sqs.autocomplete(text_auto="Jen 867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - # Test numbers alone - autocomplete_4 = self.sqs.autocomplete(text_auto="867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - -class LiveElasticsearch2RoundTripTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.srtsi = Elasticsearch2RoundTripSearchIndex() - self.ui.build(indexes=[self.srtsi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sqs = SearchQuerySet("elasticsearch") - - # Fake indexing. - mock = MockModel() - mock.id = 1 - self.sb.update(self.srtsi, [mock]) - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_round_trip(self): - results = self.sqs.filter(id="core.mockmodel.1") - - # Sanity check. - self.assertEqual(results.count(), 1) - - # Check the individual fields. - result = results[0] - self.assertEqual(result.id, "core.mockmodel.1") - self.assertEqual(result.text, "This is some example text.") - self.assertEqual(result.name, "Mister Pants") - self.assertEqual(result.is_active, True) - self.assertEqual(result.post_count, 25) - self.assertEqual(result.average_rating, 3.6) - self.assertEqual(result.price, "24.99") - self.assertEqual(result.pub_date, datetime.date(2009, 11, 21)) - self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00)) - self.assertEqual(result.tags, ["staff", "outdoor", "activist", "scientist"]) - self.assertEqual(result.sites, [3, 5, 1]) - - -class LiveElasticsearch2PickleTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_pickling(self): - results = self.sqs.all() - - for res in results: - # Make sure the cache is full. - pass - - in_a_pickle = pickle.dumps(results) - like_a_cuke = pickle.loads(in_a_pickle) - self.assertEqual(len(like_a_cuke), len(results)) - self.assertEqual(like_a_cuke[0].id, results[0].id) - - -class Elasticsearch2BoostBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2BoostMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sample_objs = [] - - for i in range(1, 5): - mock = AFourthMockModel() - mock.id = i - - if i % 2: - mock.author = "daniel" - mock.editor = "david" - else: - mock.author = "david" - mock.editor = "daniel" - - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def raw_search(self, query): - return self.raw_es.search( - q="*:*", index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - - def test_boost(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 4) - - results = SearchQuerySet(using="elasticsearch").filter( - SQ(author="daniel") | SQ(editor="daniel") - ) - - self.assertEqual( - set([result.id for result in results]), - { - "core.afourthmockmodel.4", - "core.afourthmockmodel.3", - "core.afourthmockmodel.1", - "core.afourthmockmodel.2", - }, - ) - - def test__to_python(self): - self.assertEqual(self.sb._to_python("abc"), "abc") - self.assertEqual(self.sb._to_python("1"), 1) - self.assertEqual(self.sb._to_python("2653"), 2653) - self.assertEqual(self.sb._to_python("25.5"), 25.5) - self.assertEqual(self.sb._to_python("[1, 2, 3]"), [1, 2, 3]) - self.assertEqual( - self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {"a": 1, "c": 3, "b": 2} - ) - self.assertEqual( - self.sb._to_python("2009-05-09T16:14:00"), - datetime.datetime(2009, 5, 9, 16, 14), - ) - self.assertEqual( - self.sb._to_python("2009-05-09T00:00:00"), - datetime.datetime(2009, 5, 9, 0, 0), - ) - self.assertEqual(self.sb._to_python(None), None) - - -class RecreateIndexTestCase(TestCase): - def setUp(self): - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - - def test_recreate_index(self): - clear_elasticsearch_index() - - sb = connections["elasticsearch"].get_backend() - sb.silently_fail = True - sb.setup() - - original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) - - sb.clear() - sb.setup() - - try: - updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) - except elasticsearch.NotFoundError: - self.fail("There is no mapping after recreating the index") - - self.assertEqual( - original_mapping, - updated_mapping, - "Mapping after recreating the index differs from the original one", - ) - - -class Elasticsearch2FacetingTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2FacetingMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 10): - mock = AFourthMockModel() - mock.id = i - if i > 5: - mock.editor = "George Taylor" - else: - mock.editor = "Perry White" - if i % 2: - mock.author = "Daniel Lindsley" - else: - mock.author = "Dan Watson" - mock.pub_date = datetime.date(2013, 9, (i % 4) + 1) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_facet(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 5), ("Dan Watson", 4)] - ) - self.assertEqual( - counts["fields"]["editor"], [("Perry White", 5), ("George Taylor", 4)] - ) - counts = ( - SearchQuerySet("elasticsearch") - .filter(content="white") - .facet("facet_field", order="reverse_count") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["facet_field"], [("Dan Watson", 2), ("Daniel Lindsley", 3)] - ) - - def test_multiple_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .narrow('editor_exact:"Perry White"') - .narrow('author_exact:"Daniel Lindsley"') - .facet("author") - .facet_counts() - ) - self.assertEqual(counts["fields"]["author"], [("Daniel Lindsley", 3)]) - - def test_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .narrow('editor_exact:"Perry White"') - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 3), ("Dan Watson", 2)] - ) - self.assertEqual(counts["fields"]["editor"], [("Perry White", 5)]) - - def test_date_facet(self): - self.sb.update(self.smmi, self.sample_objs) - start = datetime.date(2013, 9, 1) - end = datetime.date(2013, 9, 30) - # Facet by day - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="day") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], - [ - (datetime.datetime(2013, 9, 1), 2), - (datetime.datetime(2013, 9, 2), 3), - (datetime.datetime(2013, 9, 3), 2), - (datetime.datetime(2013, 9, 4), 2), - ], - ) - # By month - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="month") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], [(datetime.datetime(2013, 9, 1), 9)] - ) diff --git a/test_haystack/elasticsearch2_tests/test_inputs.py b/test_haystack/elasticsearch2_tests/test_inputs.py deleted file mode 100644 index af9f8f332..000000000 --- a/test_haystack/elasticsearch2_tests/test_inputs.py +++ /dev/null @@ -1,85 +0,0 @@ -from django.test import TestCase - -from haystack import connections, inputs - - -class Elasticsearch2InputTestCase(TestCase): - def setUp(self): - super().setUp() - self.query_obj = connections["elasticsearch"].get_query() - - def test_raw_init(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {}) - self.assertEqual(raw.post_process, False) - - raw = inputs.Raw("hello OR there, :you", test="really") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {"test": "really"}) - self.assertEqual(raw.post_process, False) - - def test_raw_prepare(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.prepare(self.query_obj), "hello OR there, :you") - - def test_clean_init(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.query_string, "hello OR there, :you") - self.assertEqual(clean.post_process, True) - - def test_clean_prepare(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.prepare(self.query_obj), "hello or there, \\:you") - - def test_exact_init(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.query_string, "hello OR there, :you") - self.assertEqual(exact.post_process, True) - - def test_exact_prepare(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.prepare(self.query_obj), '"hello OR there, :you"') - - exact = inputs.Exact("hello OR there, :you", clean=True) - self.assertEqual(exact.prepare(self.query_obj), '"hello or there, \\:you"') - - def test_not_init(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.query_string, "hello OR there, :you") - self.assertEqual(not_it.post_process, True) - - def test_not_prepare(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.prepare(self.query_obj), "NOT (hello or there, \\:you)") - - def test_autoquery_init(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual(autoquery.query_string, 'panic -don\'t "froody dude"') - self.assertEqual(autoquery.post_process, False) - - def test_autoquery_prepare(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual( - autoquery.prepare(self.query_obj), 'panic NOT don\'t "froody dude"' - ) - - def test_altparser_init(self): - altparser = inputs.AltParser("dismax") - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "") - self.assertEqual(altparser.kwargs, {}) - self.assertEqual(altparser.post_process, False) - - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "douglas adams") - self.assertEqual(altparser.kwargs, {"mm": 1, "qf": "author"}) - self.assertEqual(altparser.post_process, False) - - def test_altparser_prepare(self): - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual( - altparser.prepare(self.query_obj), - """{!dismax mm=1 qf=author v='douglas adams'}""", - ) diff --git a/test_haystack/elasticsearch2_tests/test_query.py b/test_haystack/elasticsearch2_tests/test_query.py deleted file mode 100644 index 5a0111d5b..000000000 --- a/test_haystack/elasticsearch2_tests/test_query.py +++ /dev/null @@ -1,247 +0,0 @@ -import datetime - -import elasticsearch -from django.contrib.gis.measure import D -from django.test import TestCase - -from haystack import connections -from haystack.inputs import Exact -from haystack.models import SearchResult -from haystack.query import SQ, SearchQuerySet - -from ..core.models import AnotherMockModel, MockModel - - -class Elasticsearch2SearchQueryTestCase(TestCase): - def setUp(self): - super().setUp() - self.sq = connections["elasticsearch"].get_query() - - def test_build_query_all(self): - self.assertEqual(self.sq.build_query(), "*:*") - - def test_build_query_single_word(self): - self.sq.add_filter(SQ(content="hello")) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_build_query_boolean(self): - self.sq.add_filter(SQ(content=True)) - self.assertEqual(self.sq.build_query(), "(True)") - - def test_regression_slash_search(self): - self.sq.add_filter(SQ(content="hello/")) - self.assertEqual(self.sq.build_query(), "(hello\\/)") - - def test_build_query_datetime(self): - self.sq.add_filter(SQ(content=datetime.datetime(2009, 5, 8, 11, 28))) - self.assertEqual(self.sq.build_query(), "(2009-05-08T11:28:00)") - - def test_build_query_multiple_words_and(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_filter(SQ(content="world")) - self.assertEqual(self.sq.build_query(), "((hello) AND (world))") - - def test_build_query_multiple_words_not(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) AND NOT ((world)))") - - def test_build_query_multiple_words_or(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) OR (hello))") - - def test_build_query_multiple_words_mixed(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual( - self.sq.build_query(), "(((why) OR (hello)) AND NOT ((world)))" - ) - - def test_build_query_phrase(self): - self.sq.add_filter(SQ(content="hello world")) - self.assertEqual(self.sq.build_query(), "(hello AND world)") - - self.sq.add_filter(SQ(content__exact="hello world")) - self.assertEqual( - self.sq.build_query(), '((hello AND world) AND ("hello world"))' - ) - - def test_build_query_boost(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_boost("world", 5) - self.assertEqual(self.sq.build_query(), "(hello) world^5") - - def test_build_query_multiple_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=Exact("2009-02-10 01:59:00"))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=Exact("2009-02-12 12:13:00"))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_multiple_filter_types_with_datetimes(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13, 0))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10T01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12T12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_in_filter_multiple_words(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=["A Famous Paper", "An Infamous Article"])) - self.assertEqual( - self.sq.build_query(), - '((why) AND title:("A Famous Paper" OR "An Infamous Article"))', - ) - - def test_build_query_in_filter_datetime(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__in=[datetime.datetime(2009, 7, 6, 1, 56, 21)])) - self.assertEqual( - self.sq.build_query(), '((why) AND pub_date:("2009-07-06T01:56:21"))' - ) - - def test_build_query_in_with_set(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in={"A Famous Paper", "An Infamous Article"})) - self.assertTrue("((why) AND title:(" in self.sq.build_query()) - self.assertTrue('"A Famous Paper"' in self.sq.build_query()) - self.assertTrue('"An Infamous Article"' in self.sq.build_query()) - - def test_build_query_wildcard_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__startswith="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack*))") - - def test_build_query_fuzzy_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__fuzzy="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack~))") - - def test_clean(self): - self.assertEqual(self.sq.clean("hello world"), "hello world") - self.assertEqual(self.sq.clean("hello AND world"), "hello and world") - self.assertEqual( - self.sq.clean( - r'hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world' - ), - 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world', - ) - self.assertEqual( - self.sq.clean("so please NOTe i am in a bAND and bORed"), - "so please NOTe i am in a bAND and bORed", - ) - - def test_build_query_with_models(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_model(MockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - self.sq.add_model(AnotherMockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_set_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - # Custom class. - class IttyBittyResult: - pass - - self.sq.set_result_class(IttyBittyResult) - self.assertTrue(issubclass(self.sq.result_class, IttyBittyResult)) - - # Reset to default. - self.sq.set_result_class(None) - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - def test_in_filter_values_list(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=[1, 2, 3])) - self.assertEqual(self.sq.build_query(), '((why) AND title:("1" OR "2" OR "3"))') - - def test_narrow_sq(self): - sqs = SearchQuerySet(using="elasticsearch").narrow(SQ(foo="moof")) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.narrow_queries), 1) - self.assertEqual(sqs.query.narrow_queries.pop(), "foo:(moof)") - - -class Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (0, 9, 9) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions < 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - {"distance": 0.5, "location_field": {"lat": 2.3456789, "lon": 1.2345678}}, - ) - - -class Elasticsearch2SearchQuerySpatialAfterReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (1, 0, 0) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions >= 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - { - "distance": "0.500000km", - "location_field": {"lat": 2.3456789, "lon": 1.2345678}, - }, - ) diff --git a/test_haystack/elasticsearch5_tests/__init__.py b/test_haystack/elasticsearch5_tests/__init__.py index 09f1ab176..5594ce332 100644 --- a/test_haystack/elasticsearch5_tests/__init__.py +++ b/test_haystack/elasticsearch5_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -29,3 +27,9 @@ def setup(): except exceptions.ConnectionError as e: log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch5_tests/test_backend.py b/test_haystack/elasticsearch5_tests/test_backend.py index 66b8af395..a4ed34fdd 100644 --- a/test_haystack/elasticsearch5_tests/test_backend.py +++ b/test_haystack/elasticsearch5_tests/test_backend.py @@ -343,7 +343,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -352,7 +352,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -361,7 +361,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -396,7 +396,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -405,7 +405,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -458,7 +458,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/elasticsearch7_tests/__init__.py b/test_haystack/elasticsearch7_tests/__init__.py index 6491d464a..24339ac89 100644 --- a/test_haystack/elasticsearch7_tests/__init__.py +++ b/test_haystack/elasticsearch7_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -29,3 +27,9 @@ def setup(): except exceptions.ConnectionError as e: log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch7_tests/test_backend.py b/test_haystack/elasticsearch7_tests/test_backend.py index f473e41cb..11edc66d5 100644 --- a/test_haystack/elasticsearch7_tests/test_backend.py +++ b/test_haystack/elasticsearch7_tests/test_backend.py @@ -344,7 +344,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -353,7 +353,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -362,7 +362,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -397,7 +397,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -406,7 +406,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -480,7 +480,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1", + "Indexed!\n2", + "Indexed!\n3", + ], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index 05c53d640..0ceb159dc 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -36,3 +34,9 @@ def setup(): % settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], e, ) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py index 73bcfddc5..7de53333c 100644 --- a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py +++ b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py @@ -229,7 +229,6 @@ def test_kwargs_are_passed_on(self): class ElasticSearchMockUnifiedIndex(UnifiedIndex): - spy_args = None def get_index(self, model_klass): @@ -360,7 +359,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -369,7 +368,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -378,7 +377,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -413,7 +412,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -422,7 +421,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -482,7 +481,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) self.assertEqual( sorted( @@ -495,9 +498,9 @@ def test_search(self): ] ), [ - "Indexed!\n1", - "Indexed!\n2", - "Indexed!\n3", + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", ], ) @@ -958,13 +961,13 @@ def test_count(self): def test_highlight(self): reset_search_queries() results = self.sqs.filter(content="index").highlight() - self.assertEqual(results[0].highlighted, ["Indexed!\n1"]) + self.assertEqual(results[0].highlighted, ["Indexed!\n1\n"]) def test_highlight_options(self): reset_search_queries() results = self.sqs.filter(content="index") results = results.highlight(pre_tags=[""], post_tags=[""]) - self.assertEqual(results[0].highlighted, ["Indexed!\n1"]) + self.assertEqual(results[0].highlighted, ["Indexed!\n1\n"]) def test_manual_iter(self): results = self.sqs.all() diff --git a/test_haystack/multipleindex/__init__.py b/test_haystack/multipleindex/__init__.py index d48e717da..0cd29ea56 100644 --- a/test_haystack/multipleindex/__init__.py +++ b/test_haystack/multipleindex/__init__.py @@ -1,24 +1,12 @@ -from django.apps import apps - -import haystack -from haystack.signals import RealtimeSignalProcessor +import os from ..utils import check_solr -_old_sp = None - -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() - global _old_sp - config = apps.get_app_config("haystack") - _old_sp = config.signal_processor - config.signal_processor = RealtimeSignalProcessor( - haystack.connections, haystack.connection_router + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern ) - - -def teardown(): - config = apps.get_app_config("haystack") - config.signal_processor.teardown() - config.signal_processor = _old_sp + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/multipleindex/tests.py b/test_haystack/multipleindex/tests.py index 5161a1f13..d4eda9b82 100644 --- a/test_haystack/multipleindex/tests.py +++ b/test_haystack/multipleindex/tests.py @@ -1,9 +1,10 @@ +from django.apps import apps from django.db import models -from haystack import connections +from haystack import connection_router, connections from haystack.exceptions import NotHandled from haystack.query import SearchQuerySet -from haystack.signals import BaseSignalProcessor +from haystack.signals import BaseSignalProcessor, RealtimeSignalProcessor from ..whoosh_tests.testcases import WhooshTestCase from .models import Bar, Foo @@ -191,6 +192,22 @@ def teardown(self): class SignalProcessorTestCase(WhooshTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + config = apps.get_app_config("haystack") + cls._old_sp = config.signal_processor + config.signal_processor = RealtimeSignalProcessor( + connections, connection_router + ) + + @classmethod + def tearDown(cls): + config = apps.get_app_config("haystack") + config.signal_processor.teardown() + config.signal_processor = cls._old_sp + super().tearDown() + def setUp(self): super().setUp() diff --git a/test_haystack/run_tests.py b/test_haystack/run_tests.py index 22f167637..85fa00a96 100755 --- a/test_haystack/run_tests.py +++ b/test_haystack/run_tests.py @@ -1,24 +1,17 @@ #!/usr/bin/env python +import os import sys -from os.path import abspath, dirname -import nose +import django +from django.core.management import call_command def run_all(argv=None): - sys.exitfunc = lambda: sys.stderr.write("Shutting down....\n") + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + os.environ["DJANGO_SETTINGS_MODULE"] = "test_haystack.settings" + django.setup() - # always insert coverage when running tests through setup.py - if argv is None: - argv = [ - "nosetests", - "--with-coverage", - "--cover-package=haystack", - "--cover-erase", - "--verbose", - ] - - nose.run_exit(argv=argv, defaultTest=abspath(dirname(__file__))) + call_command("test", sys.argv[1:]) if __name__ == "__main__": diff --git a/test_haystack/settings.py b/test_haystack/settings.py index c4234f547..7c658836a 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -8,6 +8,9 @@ "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "haystack_tests.db"} } +# Use BigAutoField as the default auto field for all models +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", @@ -34,6 +37,7 @@ "APP_DIRS": True, "OPTIONS": { "context_processors": [ + "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", ] @@ -91,13 +95,7 @@ try: import elasticsearch - if (2,) <= elasticsearch.__version__ <= (3,): - HAYSTACK_CONNECTIONS["elasticsearch"].update( - { - "ENGINE": "haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine" - } - ) - elif (5,) <= elasticsearch.__version__ <= (6,): + if (5,) <= elasticsearch.__version__ <= (6,): HAYSTACK_CONNECTIONS["elasticsearch"].update( { "ENGINE": "haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine" diff --git a/test_haystack/simple_tests/test_simple_backend.py b/test_haystack/simple_tests/test_simple_backend.py index e19662217..3f3df65e8 100644 --- a/test_haystack/simple_tests/test_simple_backend.py +++ b/test_haystack/simple_tests/test_simple_backend.py @@ -206,7 +206,6 @@ def test_more_like_this(self): self.assertEqual(self.backend.more_like_this(self.sample_objs[0])["hits"], 0) def test_score_field_collision(self): - index = connections["simple"].get_unified_index().get_index(ScoreMockModel) sample_objs = ScoreMockModel.objects.all() diff --git a/test_haystack/solr_tests/__init__.py b/test_haystack/solr_tests/__init__.py index 1b1d43036..0cd29ea56 100644 --- a/test_haystack/solr_tests/__init__.py +++ b/test_haystack/solr_tests/__init__.py @@ -1,9 +1,12 @@ -import warnings - -warnings.simplefilter("ignore", Warning) +import os from ..utils import check_solr -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh b/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh new file mode 100644 index 000000000..bf2b4fb9d --- /dev/null +++ b/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh @@ -0,0 +1,15 @@ +# figure out the solr container ID +SOLR_CONTAINER=`docker ps -f ancestor=solr:6 --format '{{.ID}}'` + +LOCAL_CONFDIR=./test_haystack/solr_tests/server/confdir +CONTAINER_CONFDIR=/opt/solr/server/solr/collection1/conf + +# set up a solr core +docker exec $SOLR_CONTAINER ./bin/solr create -c collection1 -p 8983 -n basic_config +# copy the testing schema to the collection and fix permissions +docker cp $LOCAL_CONFDIR/solrconfig.xml $SOLR_CONTAINER:$CONTAINER_CONFDIR/solrconfig.xml +docker cp $LOCAL_CONFDIR/schema.xml $SOLR_CONTAINER:$CONTAINER_CONFDIR/schema.xml +docker exec $SOLR_CONTAINER mv $CONTAINER_CONFDIR/managed-schema $CONTAINER_CONFDIR/managed-schema.old +docker exec -u root $SOLR_CONTAINER chown -R solr:solr /opt/solr/server/solr/collection1 +# reload the solr core +curl "http://localhost:9001/solr/admin/cores?action=RELOAD&core=collection1" diff --git a/test_haystack/solr_tests/server/wait-for-solr b/test_haystack/solr_tests/server/wait-for-solr index 3b2f69a25..88446d939 100755 --- a/test_haystack/solr_tests/server/wait-for-solr +++ b/test_haystack/solr_tests/server/wait-for-solr @@ -8,7 +8,7 @@ import requests max_retries = 100 retry_count = 0 retry_delay = 15 -status_url = 'http://localhost:9001/solr/collection1/admin/ping' +status_url = "http://localhost:9001/solr/collection1/admin/ping" while retry_count < max_retries: @@ -20,14 +20,18 @@ while retry_count < max_retries: if status_code == 200: sys.exit(0) except Exception as exc: - print('Unhandled exception requesting %s: %s' % (status_url, exc), file=sys.stderr) + print( + "Unhandled exception requesting %s: %s" % (status_url, exc), file=sys.stderr + ) retry_count += 1 - print('Waiting {0} seconds for Solr to start (retry #{1}, status {2})'.format(retry_delay, - retry_count, - status_code), - file=sys.stderr) + print( + "Waiting {0} seconds for Solr to start (retry #{1}, status {2})".format( + retry_delay, retry_count, status_code + ), + file=sys.stderr, + ) time.sleep(retry_delay) diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index 6e82ea6f0..cab7b88b1 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -10,7 +10,7 @@ from django.conf import settings from django.test import TestCase from django.test.utils import override_settings -from pkg_resources import parse_version +from packaging.version import Version from haystack import connections, indexes, reset_search_queries from haystack.exceptions import SkipDocument @@ -267,7 +267,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00Z", "id": "core.mockmodel.1", }, @@ -276,7 +276,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00Z", "id": "core.mockmodel.2", }, @@ -285,7 +285,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00Z", "id": "core.mockmodel.3", }, @@ -321,7 +321,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00Z", "id": "core.mockmodel.2", }, @@ -330,7 +330,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00Z", "id": "core.mockmodel.3", }, @@ -400,7 +400,11 @@ def test_search(self): result.highlighted["text"][0] for result in self.sb.search("Index", highlight=True)["results"] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) # shortened highlighting options @@ -416,7 +420,7 @@ def test_search(self): "results" ] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + ["Indexed!\n1\n", "Indexed!\n2\n", "Indexed!\n3\n"], ) # full-form highlighting options @@ -428,7 +432,7 @@ def test_search(self): "results" ] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + ["Indexed!\n1\n", "Indexed!\n2\n", "Indexed!\n3\n"], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) @@ -1216,6 +1220,21 @@ def test_related_load_all_queryset(self): self.assertEqual([obj.object.id for obj in sqs], list(range(11, 24))) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) + def test_related_load_all_with_empty_model_results(self): + another_index = SolrAnotherMockModelSearchIndex() + another_index.update("solr") + self.ui.build(indexes=[self.smmi, another_index]) + + sqs = self.rsqs.order_by("id") + assert len(list(sqs)) == 25 + sqs = sqs.all().load_all_queryset( + AnotherMockModel, AnotherMockModel.objects.none() + ) + sqs = sqs.load_all() + # two AnotherMockModel objects are skipped, so only 23 results now + # (but those results are still present and weren't skipped) + assert len(list(sqs)) == 23 + def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections["solr"].queries), 0) @@ -1646,7 +1665,7 @@ def test_boost(self): @unittest.skipIf( - parse_version(pysolr.__version__) < parse_version("3.1.1"), + Version(pysolr.__version__) < Version("3.1.1"), "content extraction requires pysolr > 3.1.1", ) class LiveSolrContentExtractionTestCase(TestCase): diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 6c6a537e0..73ad57c74 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -1,5 +1,7 @@ import datetime import os +import shutil +import tempfile from io import StringIO from tempfile import mkdtemp from unittest.mock import patch @@ -202,7 +204,6 @@ def test_multiprocessing(self): self.assertEqual(self.solr.search("*:*").hits, 0) def test_build_schema_wrong_backend(self): - settings.HAYSTACK_CONNECTIONS["whoosh"] = { "ENGINE": "haystack.backends.whoosh_backend.WhooshEngine", "PATH": mkdtemp(prefix="dummy-path-"), @@ -214,12 +215,14 @@ def test_build_schema_wrong_backend(self): ) def test_build_schema(self): - # Stow. oldhdf = constants.DOCUMENT_FIELD oldui = connections["solr"].get_unified_index() oldurl = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + conf_dir = tempfile.mkdtemp() + with open(os.path.join(conf_dir, "managed-schema"), "w+") as fp: + pass try: needle = "Th3S3cr3tK3y" constants.DOCUMENT_FIELD = ( @@ -236,10 +239,6 @@ def test_build_schema(self): rendered_file = StringIO() - script_dir = os.path.realpath(os.path.dirname(__file__)) - conf_dir = os.path.join( - script_dir, "server", "solr", "server", "solr", "mgmnt", "conf" - ) schema_file = os.path.join(conf_dir, "schema.xml") solrconfig_file = os.path.join(conf_dir, "solrconfig.xml") @@ -263,16 +262,23 @@ def test_build_schema(self): os.path.isfile(os.path.join(conf_dir, "managed-schema.old")) ) - call_command("build_solr_schema", using="solr", reload_core=True) + with patch( + "haystack.management.commands.build_solr_schema.requests.get" + ) as mock_request: + call_command("build_solr_schema", using="solr", reload_core=True) - os.rename(schema_file, "%s.bak" % schema_file) - self.assertRaises( - CommandError, - call_command, - "build_solr_schema", - using="solr", - reload_core=True, - ) + with patch( + "haystack.management.commands.build_solr_schema.requests.get" + ) as mock_request: + mock_request.return_value.ok = False + + self.assertRaises( + CommandError, + call_command, + "build_solr_schema", + using="solr", + reload_core=True, + ) call_command("build_solr_schema", using="solr", filename=schema_file) with open(schema_file) as s: @@ -282,6 +288,49 @@ def test_build_schema(self): constants.DOCUMENT_FIELD = oldhdf connections["solr"]._index = oldui settings.HAYSTACK_CONNECTIONS["solr"]["URL"] = oldurl + shutil.rmtree(conf_dir, ignore_errors=True) + + def test_build_solr_schema_reload_core_without_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL does not have a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Remove trailing slash if present + updated_url = ( + current_url.rstrip("/") if current_url.endswith("/") else current_url + ) + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + + def test_build_solr_schema_reload_core_with_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL has a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Add a trailing slash if not present + updated_url = current_url if current_url.endswith("/") else current_url + "/" + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message class AppModelManagementCommandTestCase(TestCase): diff --git a/test_haystack/spatial/__init__.py b/test_haystack/spatial/__init__.py index 02a7dd78a..0cd29ea56 100644 --- a/test_haystack/spatial/__init__.py +++ b/test_haystack/spatial/__init__.py @@ -1,5 +1,12 @@ +import os + from ..utils import check_solr -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/spatial/test_spatial.py b/test_haystack/spatial/test_spatial.py index 8218f9bf8..6d0fbc12a 100644 --- a/test_haystack/spatial/test_spatial.py +++ b/test_haystack/spatial/test_spatial.py @@ -106,6 +106,7 @@ def setUp(self): super().setUp() self.ui = connections[self.using].get_unified_index() + self.ui.reset() self.checkindex = self.ui.get_index(Checkin) self.checkindex.reindex(using=self.using) self.sqs = SearchQuerySet().using(self.using) diff --git a/test_haystack/test_app_using_appconfig/__init__.py b/test_haystack/test_app_using_appconfig/__init__.py index 30a0d2351..e69de29bb 100644 --- a/test_haystack/test_app_using_appconfig/__init__.py +++ b/test_haystack/test_app_using_appconfig/__init__.py @@ -1 +0,0 @@ -default_app_config = "test_app_using_appconfig.apps.SimpleTestAppConfig" diff --git a/test_haystack/test_app_using_appconfig/migrations/0001_initial.py b/test_haystack/test_app_using_appconfig/migrations/0001_initial.py index 1f9b7051e..309b49009 100644 --- a/test_haystack/test_app_using_appconfig/migrations/0001_initial.py +++ b/test_haystack/test_app_using_appconfig/migrations/0001_initial.py @@ -2,7 +2,6 @@ class Migration(migrations.Migration): - dependencies = [] operations = [ diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py new file mode 100644 index 000000000..f4808f68c --- /dev/null +++ b/test_haystack/test_django_config_detection.py @@ -0,0 +1,21 @@ +"""""" + +import unittest + +import django +from django.test import TestCase + +import haystack + + +class AppConfigCompatibilityTestCase(TestCase): + @unittest.skipIf( + django.VERSION >= (3, 2), "default_app_config is deprecated since django 3.2." + ) + def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): + has_default_appconfig_attr = hasattr(haystack, "default_app_config") + self.assertTrue(has_default_appconfig_attr) + + def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): + has_default_appconfig_attr = hasattr(haystack, "default_app_config") + self.assertFalse(has_default_appconfig_attr) diff --git a/test_haystack/test_fields.py b/test_haystack/test_fields.py index bb5cf3f4b..8f5b36301 100644 --- a/test_haystack/test_fields.py +++ b/test_haystack/test_fields.py @@ -597,7 +597,7 @@ def test_prepare(self): template3 = CharField(use_template=True) template3.instance_name = "template" - self.assertEqual(template3.prepare(mock), "Indexed!\n1") + self.assertEqual(template3.prepare(mock), "Indexed!\n1\n") template4 = CharField(use_template=True, template_name="search/indexes/foo.txt") template4.instance_name = "template" diff --git a/test_haystack/test_indexes.py b/test_haystack/test_indexes.py index 74e4e7755..6e6ee2d2d 100644 --- a/test_haystack/test_indexes.py +++ b/test_haystack/test_indexes.py @@ -168,7 +168,7 @@ def setUp(self): self.sample_docs = { "core.mockmodel.1": { - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "django_id": "1", "django_ct": "core.mockmodel", "extra": "Stored!\n1", @@ -177,7 +177,7 @@ def setUp(self): "id": "core.mockmodel.1", }, "core.mockmodel.2": { - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "django_id": "2", "django_ct": "core.mockmodel", "extra": "Stored!\n2", @@ -186,7 +186,7 @@ def setUp(self): "id": "core.mockmodel.2", }, "core.mockmodel.3": { - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "django_id": "3", "django_ct": "core.mockmodel", "extra": "Stored!\n3", @@ -687,8 +687,7 @@ class Meta: def get_index_fieldname(self, f): if f.name == "author": return "author_bar" - else: - return f.name + return f.name class YetAnotherBasicModelSearchIndex(indexes.ModelSearchIndex, indexes.Indexable): diff --git a/test_haystack/test_management_commands.py b/test_haystack/test_management_commands.py index 5d55de3a1..b66faf38f 100644 --- a/test_haystack/test_management_commands.py +++ b/test_haystack/test_management_commands.py @@ -77,8 +77,8 @@ def test_rebuild_index(self, mock_handle_clear, mock_handle_update): self.assertTrue(mock_handle_clear.called) self.assertTrue(mock_handle_update.called) - @patch("haystack.management.commands.update_index.Command.handle") - @patch("haystack.management.commands.clear_index.Command.handle") + @patch("haystack.management.commands.update_index.Command.handle", return_value="") + @patch("haystack.management.commands.clear_index.Command.handle", return_value="") def test_rebuild_index_nocommit(self, *mocks): call_command("rebuild_index", interactive=False, commit=False) @@ -92,7 +92,7 @@ def test_rebuild_index_nocommit(self, *mocks): @patch("haystack.management.commands.clear_index.Command.handle", return_value="") @patch("haystack.management.commands.update_index.Command.handle", return_value="") - def test_rebuild_index_nocommit(self, update_mock, clear_mock): + def test_rebuild_index_nocommit_two(self, update_mock, clear_mock): """ Confirm that command-line option parsing produces the same results as using call_command() directly, mostly as a sanity check for the logic in rebuild_index which combines the option_lists for its diff --git a/test_haystack/test_managers.py b/test_haystack/test_managers.py index 3784217cd..cc600752e 100644 --- a/test_haystack/test_managers.py +++ b/test_haystack/test_managers.py @@ -242,11 +242,11 @@ def spelling_suggestion(self): def test_values(self): sqs = self.search_index.objects.auto_query("test").values("id") - self.assert_(isinstance(sqs, ValuesSearchQuerySet)) + self.assertIsInstance(sqs, ValuesSearchQuerySet) def test_valueslist(self): sqs = self.search_index.objects.auto_query("test").values_list("id") - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) class CustomManagerTestCase(TestCase): diff --git a/test_haystack/test_query.py b/test_haystack/test_query.py index ffe35c19a..c66d38427 100644 --- a/test_haystack/test_query.py +++ b/test_haystack/test_query.py @@ -95,6 +95,12 @@ def test_simple_nesting(self): class BaseSearchQueryTestCase(TestCase): fixtures = ["base_data.json", "bulk_data.json"] + @classmethod + def setUpClass(cls): + for connection in connections.all(): + connection.get_unified_index().reset() + super().setUpClass() + def setUp(self): super().setUp() self.bsq = BaseSearchQuery() @@ -442,7 +448,7 @@ def test_len(self): def test_repr(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) - self.assertRegexpMatches( + self.assertRegex( repr(self.msqs), r"^, using=None>$", @@ -967,18 +973,18 @@ def test_or_and(self): class ValuesQuerySetTestCase(SearchQuerySetTestCase): def test_values_sqs(self): sqs = self.msqs.auto_query("test").values("id") - self.assert_(isinstance(sqs, ValuesSearchQuerySet)) + self.assertIsInstance(sqs, ValuesSearchQuerySet) # We'll do a basic test to confirm that slicing works as expected: - self.assert_(isinstance(sqs[0], dict)) - self.assert_(isinstance(sqs[0:5][0], dict)) + self.assertIsInstance(sqs[0], dict) + self.assertIsInstance(sqs[0:5][0], dict) def test_valueslist_sqs(self): sqs = self.msqs.auto_query("test").values_list("id") - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) - self.assert_(isinstance(sqs[0], (list, tuple))) - self.assert_(isinstance(sqs[0:1][0], (list, tuple))) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) + self.assertIsInstance(sqs[0], (list, tuple)) + self.assertIsInstance(sqs[0:1][0], (list, tuple)) self.assertRaises( TypeError, @@ -989,12 +995,12 @@ def test_valueslist_sqs(self): ) flat_sqs = self.msqs.auto_query("test").values_list("id", flat=True) - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) # Note that this will actually be None because a mocked sqs lacks # anything else: - self.assert_(flat_sqs[0] is None) - self.assert_(flat_sqs[0:1][0] is None) + self.assertIsNone(flat_sqs[0]) + self.assertIsNone(flat_sqs[0:1][0]) class EmptySearchQuerySetTestCase(TestCase): diff --git a/test_haystack/whoosh_tests/test_forms.py b/test_haystack/whoosh_tests/test_forms.py index 9899807c4..64be222fc 100644 --- a/test_haystack/whoosh_tests/test_forms.py +++ b/test_haystack/whoosh_tests/test_forms.py @@ -1,4 +1,5 @@ """Tests for Whoosh spelling suggestions""" + from django.conf import settings from django.http import HttpRequest @@ -37,4 +38,4 @@ def test_view_suggestion(self): mock = HttpRequest() mock.GET["q"] = "exampl" resp = view(mock) - self.assertEqual(resp.content, b"Suggestion: example") + self.assertEqual(resp.content, b"Suggestion: example\n") diff --git a/test_haystack/whoosh_tests/test_whoosh_backend.py b/test_haystack/whoosh_tests/test_whoosh_backend.py index fd5f56e14..5de276b5e 100644 --- a/test_haystack/whoosh_tests/test_whoosh_backend.py +++ b/test_haystack/whoosh_tests/test_whoosh_backend.py @@ -1,12 +1,11 @@ import os import unittest -from datetime import timedelta +from datetime import date, datetime, timedelta from decimal import Decimal from django.conf import settings from django.test import TestCase from django.test.utils import override_settings -from django.utils.datetime_safe import date, datetime from whoosh.analysis import SpaceSeparatedTokenizer, SubstitutionFilter from whoosh.fields import BOOLEAN, DATETIME, KEYWORD, NUMERIC, TEXT from whoosh.qparser import QueryParser @@ -115,6 +114,7 @@ def get_model(self): return MockModel +@override_settings(USE_TZ=False) class WhooshSearchBackendTestCase(WhooshTestCase): fixtures = ["bulk_data.json"] diff --git a/tox.ini b/tox.ini index 9eefabfc8..7868aec7d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,39 @@ [tox] envlist = docs - py35-django2.2-es{1.x,2.x,5.x,7.x} - py{36,37,38,py}-django{2.2,3.0}-es{1.x,2.x,5.x,7.x} + py{38,39,310,311,312}-django{3.2,4.2,5.0,5.1}-es7.x +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 + +[gh-actions:env] +DJANGO = + 3.2: django3.2 + 4.2: django4.2 + 5.0: django5.0 + 5.1: django5.1 [testenv] commands = python test_haystack/solr_tests/server/wait-for-solr - python {toxinidir}/setup.py test + coverage run {toxinidir}/test_haystack/run_tests.py deps = + pysolr>=3.7.0 + whoosh>=2.5.4,<3.0 + python-dateutil + geopy==2.0.0 + coverage requests - django2.2: Django>=2.2,<3.0 - django3.0: Django>=3.0,<3.1 + setuptools; python_version >= "3.12" # Can be removed on pysolr >= v3.10 + django3.2: Django>=3.2,<3.3 + django4.2: Django>=4.2,<4.3 + django5.0: Django>=5.0,<5.1 + django5.1: Django>=5.1,<5.2 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6