From 1b464ac4b128acbc23808dc796ffa7801a6ebb7e Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Fri, 25 Oct 2019 09:42:44 -0300 Subject: [PATCH 001/162] fix facet.date API is gone in Solr 6.6, replaced with facet.range - issue #1572 --- haystack/backends/solr_backend.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index d1f4cc664..92ab57b91 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -283,14 +283,14 @@ def build_search_kwargs( if date_facets is not None: kwargs["facet"] = "on" - kwargs["facet.date"] = date_facets.keys() - kwargs["facet.date.other"] = "none" + kwargs["facet.range"] = date_facets.keys() + kwargs["facet.range.other"] = "none" for key, value in date_facets.items(): - kwargs["f.%s.facet.date.start" % key] = self.conn._from_python( + kwargs["f.%s.facet.range.start" % key] = self.conn._from_python( value.get("start_date") ) - kwargs["f.%s.facet.date.end" % key] = self.conn._from_python( + kwargs["f.%s.facet.range.end" % key] = self.conn._from_python( value.get("end_date") ) gap_by_string = value.get("gap_by").upper() @@ -299,7 +299,7 @@ def build_search_kwargs( if value.get("gap_amount") != 1: gap_string += "S" - kwargs["f.%s.facet.date.gap" % key] = "+%s/%s" % ( + kwargs["f.%s.facet.range.gap" % key] = "+%s/%s" % ( gap_string, gap_by_string, ) @@ -486,6 +486,7 @@ def _process_results( "fields": raw_results.facets.get("facet_fields", {}), "dates": raw_results.facets.get("facet_dates", {}), "queries": raw_results.facets.get("facet_queries", {}), + "ranges": raw_results.facets.get("facet_ranges", {}), } for key in ["fields"]: @@ -499,6 +500,14 @@ def _process_results( ) ) + for key in ['ranges']: + for facet_field in facets[key]: + # Convert to a two-tuple, as Solr's json format returns a list of + # pairs. + facets[key][facet_field] = list( + zip(facets[key][facet_field]['counts'][::2], + facets[key][facet_field]['counts'][1::2])) + if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) From 32df6a2ac16cbf754241a729c9d3c90e257f4e8e Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Fri, 25 Oct 2019 09:45:24 -0300 Subject: [PATCH 002/162] add attribute `date_facet_fields` on generic_views receiving list of kwargs to date_facet - issue #1572 --- haystack/generic_views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/haystack/generic_views.py b/haystack/generic_views.py index 74cdfc9c9..3c2625e21 100644 --- a/haystack/generic_views.py +++ b/haystack/generic_views.py @@ -111,6 +111,10 @@ def get_queryset(self): qs = super(FacetedSearchMixin, self).get_queryset() for field in self.facet_fields: qs = qs.facet(field) + + for field in self.date_facet_fields: + qs = qs.date_facet(**field) + return qs From 9bb3c07121be579742fb3115c8545fd33821b8d4 Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Fri, 25 Oct 2019 11:06:52 -0300 Subject: [PATCH 003/162] set default None to new attribute `date_facet_fields` PR #1690 and issue #1572 --- haystack/generic_views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/haystack/generic_views.py b/haystack/generic_views.py index 3c2625e21..65c1324f3 100644 --- a/haystack/generic_views.py +++ b/haystack/generic_views.py @@ -96,6 +96,7 @@ class FacetedSearchMixin(SearchMixin): form_class = FacetedSearchForm facet_fields = None + date_facet_fields = None def get_form_kwargs(self): kwargs = super(FacetedSearchMixin, self).get_form_kwargs() From 0b3108c59cdd33c258510421e207264e1cc2177f Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Fri, 25 Oct 2019 18:17:49 -0300 Subject: [PATCH 004/162] add HAYSTACK_DATE_FACET_FIELD setting to `date_facet` support on Solr >= 6.6. Default is `range`. Olders set `date` PR #1690 and issue #1572 --- haystack/backends/solr_backend.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 92ab57b91..4f95890b6 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -281,16 +281,21 @@ def build_search_kwargs( "f.%s.facet.%s" % (facet_field, key) ] = self.conn._from_python(value) + # Support to `date_facet` on Solr >= 6.6. Olders set `date` + date_facet_field = getattr( + settings, "HAYSTACK_DATE_FACET_FIELD", "range" + ) + if date_facets is not None: kwargs["facet"] = "on" - kwargs["facet.range"] = date_facets.keys() - kwargs["facet.range.other"] = "none" + kwargs["facet.%s" % date_facet_field ] = date_facets.keys() + kwargs["facet.%s.other" % date_facet_field ] = "none" for key, value in date_facets.items(): - kwargs["f.%s.facet.range.start" % key] = self.conn._from_python( + kwargs["f.%s.facet.%s.start" % (key, date_facet_field)] = self.conn._from_python( value.get("start_date") ) - kwargs["f.%s.facet.range.end" % key] = self.conn._from_python( + kwargs["f.%s.facet.%s.end" % (key, date_facet_field)] = self.conn._from_python( value.get("end_date") ) gap_by_string = value.get("gap_by").upper() @@ -299,7 +304,7 @@ def build_search_kwargs( if value.get("gap_amount") != 1: gap_string += "S" - kwargs["f.%s.facet.range.gap" % key] = "+%s/%s" % ( + kwargs["f.%s.facet.%s.gap" % (key, date_facet_field)] = "+%s/%s" % ( gap_string, gap_by_string, ) From c984992ee2246ed7bbfe9b4e767ab17513fac3f8 Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Sat, 26 Oct 2019 17:15:33 -0300 Subject: [PATCH 005/162] changed setting to HAYSTACK_CONNECTIONS parameter by connection (ex.: `HAYSTACK_CONNECTIONS["default"]["DATE_FACET_FIELD"]`).\n Used to `date_facet` support on Solr >= 6.6. Default is `range`. Olders set `date` PR #1690 and issue #1572 --- haystack/backends/solr_backend.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 4f95890b6..6638281a5 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -71,6 +71,9 @@ def __init__(self, connection_alias, **connection_options): self.collate = connection_options.get("COLLATE_SPELLING", True) + # Support to `date_facet` on Solr >= 6.6. Olders set `date` + self.date_facet_field = connection_options.get("DATE_FACET_FIELD", "range") + self.conn = Solr( connection_options["URL"], timeout=self.timeout, @@ -281,21 +284,16 @@ def build_search_kwargs( "f.%s.facet.%s" % (facet_field, key) ] = self.conn._from_python(value) - # Support to `date_facet` on Solr >= 6.6. Olders set `date` - date_facet_field = getattr( - settings, "HAYSTACK_DATE_FACET_FIELD", "range" - ) - if date_facets is not None: kwargs["facet"] = "on" - kwargs["facet.%s" % date_facet_field ] = date_facets.keys() - kwargs["facet.%s.other" % date_facet_field ] = "none" + kwargs["facet.%s" % self.date_facet_field ] = date_facets.keys() + kwargs["facet.%s.other" % self.date_facet_field ] = "none" for key, value in date_facets.items(): - kwargs["f.%s.facet.%s.start" % (key, date_facet_field)] = self.conn._from_python( + kwargs["f.%s.facet.%s.start" % (key, self.date_facet_field)] = self.conn._from_python( value.get("start_date") ) - kwargs["f.%s.facet.%s.end" % (key, date_facet_field)] = self.conn._from_python( + kwargs["f.%s.facet.%s.end" % (key, self.date_facet_field)] = self.conn._from_python( value.get("end_date") ) gap_by_string = value.get("gap_by").upper() @@ -304,7 +302,7 @@ def build_search_kwargs( if value.get("gap_amount") != 1: gap_string += "S" - kwargs["f.%s.facet.%s.gap" % (key, date_facet_field)] = "+%s/%s" % ( + kwargs["f.%s.facet.%s.gap" % (key, self.date_facet_field)] = "+%s/%s" % ( gap_string, gap_by_string, ) From f26238a97ad26a0f2ecf59d5dc5582c175df41ad Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Sat, 26 Oct 2019 18:42:14 -0300 Subject: [PATCH 006/162] doc to `DATE_FACET_FIELD` setting to HAYSTACK_CONNECTIONS parameter by connection (ex.: `HAYSTACK_CONNECTIONS["default"]["DATE_FACET_FIELD"]`).\n Used to `date_facet` support on Solr >= 6.6. Default is `range`. Olders set `date` PR #1690 and issue #1572 --- docs/settings.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/settings.rst b/docs/settings.rst index b1a2eb5ae..a82514cbc 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -81,6 +81,8 @@ Additionally, each backend may have additional options it requires: * ``URL`` - The URL to the Solr core. e.g. http://localhost:9001/solr/collection1 * ``ADMIN_URL`` - The URL to the administrative functions. e.g. http://localhost:9001/solr/admin/cores + * ``DATE_FACET_FIELD`` - Support to `date_facet` on Solr >= 6.6. Olders set `date`. + Default is `range` * Whoosh From 0b05bdc6f38166ce1748ebd106107999a50f1477 Mon Sep 17 00:00:00 2001 From: LaborAutonomo Date: Sat, 26 Oct 2019 18:57:07 -0300 Subject: [PATCH 007/162] fix doc to ``DATE_FACET_FIELD`` setting to HAYSTACK_CONNECTIONS parameter by connection (ex.: ``HAYSTACK_CONNECTIONS["default"]["DATE_FACET_FIELD"]``).\n Used to ``date_facet`` support on Solr >= 6.6. Default is ``range``. Olders set ``date`` PR #1690 and issue #1572 --- docs/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/settings.rst b/docs/settings.rst index a82514cbc..2208e6e37 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -81,8 +81,6 @@ Additionally, each backend may have additional options it requires: * ``URL`` - The URL to the Solr core. e.g. http://localhost:9001/solr/collection1 * ``ADMIN_URL`` - The URL to the administrative functions. e.g. http://localhost:9001/solr/admin/cores - * ``DATE_FACET_FIELD`` - Support to `date_facet` on Solr >= 6.6. Olders set `date`. - Default is `range` * Whoosh @@ -109,6 +107,8 @@ The following options are optional: don't want indexed or for when you want to replace an index. * ``KWARGS`` - (Solr and ElasticSearch) Any additional keyword arguments that should be passed on to the underlying client library. +* ``DATE_FACET_FIELD`` - (Solr-only) Support to ``date_facet`` on Solr >= 6.6. + Olders set ``date``. Default is ``range``. ``HAYSTACK_ROUTERS`` From f7a15974573eb32eb34bf170b6efbba79f353a41 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:16:14 -0400 Subject: [PATCH 008/162] Restore convenience import for haystack.utils.highlighter --- haystack/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/haystack/utils/__init__.py b/haystack/utils/__init__.py index 6e335352c..de7207f0c 100644 --- a/haystack/utils/__init__.py +++ b/haystack/utils/__init__.py @@ -4,6 +4,7 @@ from django.conf import settings from haystack.constants import DJANGO_CT, DJANGO_ID, ID +from haystack.utils.highlighting import Highlighter # noqa=F401 IDENTIFIER_REGEX = re.compile(r"^[\w\d_]+\.[\w\d_]+\.[\w\d-]+$") From 424648009edfa5c7e58cc4e9e838bbb39238f37f Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:25:39 -0400 Subject: [PATCH 009/162] Enable pre-commit for local quality checks This does not yet introduce flake8 because we have some stylistic cleanups first. --- .pre-commit-config.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..1008976dc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +exclude: ".*/vendor/.*" +repos: + - repo: https://github.com/pre-commit/mirrors-isort + rev: v5.9.3 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-added-large-files + args: ["--maxkb=128"] + - id: check-ast + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + - id: pretty-format-json + args: ["--autofix", "--no-sort-keys", "--indent=4"] + - id: trailing-whitespace From e7ea0f9715312b0f6bafc1da134acb8f5f99bdf8 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:25:54 -0400 Subject: [PATCH 010/162] Strip unnecessary execute bits --- haystack/management/commands/update_index.py | 0 haystack/utils/app_loading.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 haystack/management/commands/update_index.py mode change 100755 => 100644 haystack/utils/app_loading.py diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py old mode 100755 new mode 100644 diff --git a/haystack/utils/app_loading.py b/haystack/utils/app_loading.py old mode 100755 new mode 100644 From 9518fe1c88302f2e7e3ad50efcfd7c694f3fc203 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:26:17 -0400 Subject: [PATCH 011/162] Roboformatting --- .github/issue_template.md | 2 +- .github/pull_request_template.md | 2 +- .github/workflows/test.yml | 1 - docs/faceting.rst | 16 +- docs/faq.rst | 4 +- docs/haystack_theme/layout.html | 2 +- docs/haystack_theme/static/documentation.css | 2 +- docs/haystack_theme/theme.conf | 2 +- docs/installing_search_engines.rst | 8 +- docs/multiple_index.rst | 26 +- docs/rich_content_extraction.rst | 2 +- docs/searchquery_api.rst | 2 +- docs/templatetags.rst | 10 +- docs/toc.rst | 1 - .../indexes/bare_bones_app/cat_text.txt | 2 +- .../search/indexes/regular_app/dog_text.txt | 2 +- haystack/utils/__init__.py | 2 +- test_haystack/core/fixtures/base_data.json | 168 +++--- test_haystack/core/fixtures/bulk_data.json | 518 +++++++++--------- test_haystack/core/templates/404.html | 2 +- .../search/indexes/core/mockmodel_content.txt | 2 +- .../search/indexes/core/mockmodel_extra.txt | 2 +- .../indexes/core/mockmodel_template.txt | 2 +- .../search/indexes/core/mockmodel_text.txt | 2 +- .../core/templates/search/search.html | 2 +- .../core/templates/test_suggestion.html | 2 +- .../templates/search/indexes/bar_text.txt | 2 +- test_haystack/solr_tests/server/wait-for-solr | 16 +- 28 files changed, 402 insertions(+), 402 deletions(-) diff --git a/.github/issue_template.md b/.github/issue_template.md index 88490948a..cdef8e6bd 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -15,4 +15,4 @@ * Search engine version: * Python version: * Django version: -* Haystack version: \ No newline at end of file +* Haystack version: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 35d92349f..620b1ff84 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,3 @@ # Hey, thanks for contributing to Haystack. Please review [the contributor guidelines](https://django-haystack.readthedocs.io/en/latest/contributing.html) and confirm that [the tests pass](https://django-haystack.readthedocs.io/en/latest/running_tests.html) with at least one search engine. -# Once your pull request has been submitted, the full test suite will be executed on https://github.com/django-haystack/django-haystack/actions/workflows/test.yml. Pull requests with passing tests are far more likely to be reviewed and merged. \ No newline at end of file +# Once your pull request has been submitted, the full test suite will be executed on https://github.com/django-haystack/django-haystack/actions/workflows/test.yml. Pull requests with passing tests are far more likely to be reviewed and merged. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 066e5d6ff..4001699bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,4 +56,3 @@ jobs: python setup.py clean build install - name: Run test run: coverage run setup.py test - diff --git a/docs/faceting.rst b/docs/faceting.rst index a9e39f29f..c5cd20d5b 100644 --- a/docs/faceting.rst +++ b/docs/faceting.rst @@ -26,12 +26,12 @@ capabilities. The general workflow in this regard is: Faceting can be difficult, especially in providing the user with the right number of options and/or the right areas to be able to drill into. This is unique to every situation and demands following what real users need. - + You may want to consider logging queries and looking at popular terms to help you narrow down how you can help your users. Haystack provides functionality so that all of the above steps are possible. -From the ground up, let's build a faceted search setup. This assumes that you +From the ground up, let's build a faceted search setup. This assumes that you have been to work through the :doc:`tutorial` and have a working Haystack installation. The same setup from the :doc:`tutorial` applies here. @@ -214,8 +214,8 @@ URLconf should resemble:: from django.urls import path from haystack.forms import FacetedSearchForm from haystack.views import FacetedSearchView - - + + urlpatterns = [ path('', FacetedSearchView(form_class=FacetedSearchForm, facet_fields=['author']), name='haystack_search'), ] @@ -243,11 +243,11 @@ might look like this:: - + {% if query %}

By Author

- +
{% if facets.fields.author %} @@ -262,12 +262,12 @@ might look like this::
- + {% for result in page.object_list %}

{{ result.object.title }}

- +

{{ result.object.body|truncatewords:80 }}

{% empty %} diff --git a/docs/faq.rst b/docs/faq.rst index 94fd0d343..05481f107 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -32,7 +32,7 @@ When should I not be using Haystack? ==================================== * Non-Model-based data. If you just want to index random data (flat files, - alternate sources, etc.), Haystack isn't a good solution. Haystack is very + alternate sources, etc.), Haystack isn't a good solution. Haystack is very ``Model``-based and doesn't work well outside of that use case. * Ultra-high volume. Because of the very nature of Haystack (abstraction layer), there's more overhead involved. This makes it portable, but as with all @@ -111,7 +111,7 @@ Several possibilities on this. #. We're not aware of the engine If you think we may not be aware of the engine you'd like, please tell us - about it (preferably via the group - + about it (preferably via the group - http://groups.google.com/group/django-haystack/). Be sure to check through the backends (in case it wasn't documented) and search the history on the group to minimize duplicates. diff --git a/docs/haystack_theme/layout.html b/docs/haystack_theme/layout.html index e1d4ab39e..b342cb597 100644 --- a/docs/haystack_theme/layout.html +++ b/docs/haystack_theme/layout.html @@ -19,4 +19,4 @@

Haystack

  • Spelling Suggestions
    • -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/docs/haystack_theme/static/documentation.css b/docs/haystack_theme/static/documentation.css index 3e9492cd0..6fa063311 100644 --- a/docs/haystack_theme/static/documentation.css +++ b/docs/haystack_theme/static/documentation.css @@ -26,4 +26,4 @@ div.sphinxsidebar ul ul { padding-left: 10px; margin-left: 10px; } div.bodywrapper { margin: 0px; } div.highlight-python, div.highlight { background-color: #262511; margin-bottom: 10px; padding: 10px; } div.footer { background-color:#262511; font-size: 90%; padding: 10px; } -table thead { background-color: #053211; border-bottom: 1px solid #262511; } \ No newline at end of file +table thead { background-color: #053211; border-bottom: 1px solid #262511; } diff --git a/docs/haystack_theme/theme.conf b/docs/haystack_theme/theme.conf index 3161b4d41..89e03bbda 100644 --- a/docs/haystack_theme/theme.conf +++ b/docs/haystack_theme/theme.conf @@ -1,2 +1,2 @@ [theme] -inherit = basic \ No newline at end of file +inherit = basic diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 3f8a1e1a2..47753773a 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -61,7 +61,7 @@ You'll also need to install the ``pysolr`` client library from PyPI:: More Like This -------------- -On Solr 6.X+ "More Like This" functionality is enabled by default. To enable +On Solr 6.X+ "More Like This" functionality is enabled by default. To enable the "More Like This" functionality on earlier versions of Solr, you'll need to enable the ``MoreLikeThisHandler``. Add the following line to your ``solrconfig.xml`` file within the ``config`` tag:: @@ -93,7 +93,7 @@ Then, you enable it in Solr by adding the following line to your ``solrconfig.xml`` file within the ``config`` tag:: - + text_general default @@ -117,14 +117,14 @@ Then change your default handler from:: 10 - + ... to ...:: explicit 10 - + default on true diff --git a/docs/multiple_index.rst b/docs/multiple_index.rst index f295db207..3fde249d1 100644 --- a/docs/multiple_index.rst +++ b/docs/multiple_index.rst @@ -84,7 +84,7 @@ Haystack ships with a ``DefaultRouter`` enabled. It looks like:: class DefaultRouter(BaseRouter): def for_read(self, **hints): return DEFAULT_ALIAS - + def for_write(self, **hints): return DEFAULT_ALIAS @@ -118,20 +118,20 @@ Master-Slave Example The ``MasterRouter`` & ``SlaveRouter`` might look like:: from haystack import routers - - + + class MasterRouter(routers.BaseRouter): def for_write(self, **hints): return 'master' - + def for_read(self, **hints): return None - - + + class SlaveRouter(routers.BaseRouter): def for_write(self, **hints): return None - + def for_read(self, **hints): return 'slave' @@ -139,12 +139,12 @@ The observant might notice that since the methods don't overlap, this could be combined into one ``Router`` like so:: from haystack import routers - - + + class MasterSlaveRouter(routers.BaseRouter): def for_write(self, **hints): return 'master' - + def for_read(self, **hints): return 'slave' @@ -160,13 +160,13 @@ For this, the ``SearchQuerySet`` class allows for manually selecting the index via the ``SearchQuerySet.using`` method:: from haystack.query import SearchQuerySet - + # Uses the routers' opinion. sqs = SearchQuerySet().auto_query('banana') - + # Forces the default. sqs = SearchQuerySet().using('default').auto_query('banana') - + # Forces the slave connection (presuming it was setup). sqs = SearchQuerySet().using('slave').auto_query('banana') diff --git a/docs/rich_content_extraction.rst b/docs/rich_content_extraction.rst index 19d672bbb..ef6f03bec 100644 --- a/docs/rich_content_extraction.rst +++ b/docs/rich_content_extraction.rst @@ -65,4 +65,4 @@ template, modified or intermixed with database content as appropriate: {% endfor %} {% endfor %} - {{ extracted.contents|striptags|safe }} \ No newline at end of file + {{ extracted.contents|striptags|safe }} diff --git a/docs/searchquery_api.rst b/docs/searchquery_api.rst index 305557e06..8704f6fab 100644 --- a/docs/searchquery_api.rst +++ b/docs/searchquery_api.rst @@ -236,7 +236,7 @@ Adds a boosted term and the amount to boost it to the query. Runs a raw query (no parsing) against the backend. -This method causes the ``SearchQuery`` to ignore the standard query-generating +This method causes the ``SearchQuery`` to ignore the standard query-generating facilities, running only what was provided instead. Note that any kwargs passed along will override anything provided diff --git a/docs/templatetags.rst b/docs/templatetags.rst index 71d6e0842..f76e7edf7 100644 --- a/docs/templatetags.rst +++ b/docs/templatetags.rst @@ -12,7 +12,7 @@ special features available to templates. ============= Takes a block of text and highlights words from a provided query within that -block of text. Optionally accepts arguments to provide the HTML tag to wrap +block of text. Optionally accepts arguments to provide the HTML tag to wrap highlighted word in, a CSS class to use with the tag and a maximum length of the blurb in characters. @@ -27,11 +27,11 @@ Example:: # Highlight summary with default behavior. {% highlight result.summary with query %} - + # Highlight summary but wrap highlighted words with a div and the # following CSS class. {% highlight result.summary with query html_tag "div" css_class "highlight_me_please" %} - + # Highlight summary but only show 40 words. {% highlight result.summary with query max_length 40 %} @@ -57,10 +57,10 @@ Example:: # Pull a full SearchQuerySet (lazy loaded) of similar content. {% more_like_this entry as related_content %} - + # Pull just the top 5 similar pieces of content. {% more_like_this entry as related_content limit 5 %} - + # Pull just the top 5 similar entries or comments. {% more_like_this entry as related_content for "blog.entry,comments.comment" limit 5 %} diff --git a/docs/toc.rst b/docs/toc.rst index 46ed9bba8..06ab413dd 100644 --- a/docs/toc.rst +++ b/docs/toc.rst @@ -50,4 +50,3 @@ Indices and tables ================== * :ref:`search` - diff --git a/example_project/templates/search/indexes/bare_bones_app/cat_text.txt b/example_project/templates/search/indexes/bare_bones_app/cat_text.txt index db8321ea5..10f379a16 100644 --- a/example_project/templates/search/indexes/bare_bones_app/cat_text.txt +++ b/example_project/templates/search/indexes/bare_bones_app/cat_text.txt @@ -1,2 +1,2 @@ {{ object.name }} -{{ object.bio }} \ No newline at end of file +{{ object.bio }} diff --git a/example_project/templates/search/indexes/regular_app/dog_text.txt b/example_project/templates/search/indexes/regular_app/dog_text.txt index 232490549..c796c047c 100644 --- a/example_project/templates/search/indexes/regular_app/dog_text.txt +++ b/example_project/templates/search/indexes/regular_app/dog_text.txt @@ -4,4 +4,4 @@ {% for toy in object.toys.all %} {{ toy.name }} -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/haystack/utils/__init__.py b/haystack/utils/__init__.py index de7207f0c..b0b0d082a 100644 --- a/haystack/utils/__init__.py +++ b/haystack/utils/__init__.py @@ -4,7 +4,7 @@ from django.conf import settings from haystack.constants import DJANGO_CT, DJANGO_ID, ID -from haystack.utils.highlighting import Highlighter # noqa=F401 +from haystack.utils.highlighting import Highlighter # noqa=F401 IDENTIFIER_REGEX = re.compile(r"^[\w\d_]+\.[\w\d_]+\.[\w\d-]+$") diff --git a/test_haystack/core/fixtures/base_data.json b/test_haystack/core/fixtures/base_data.json index c2b5ad37f..d4f3fdad5 100644 --- a/test_haystack/core/fixtures/base_data.json +++ b/test_haystack/core/fixtures/base_data.json @@ -1,88 +1,86 @@ [ - { - "pk": 1, - "model": "core.mocktag", - "fields": { - "name": "primary" + { + "pk": 1, + "model": "core.mocktag", + "fields": { + "name": "primary" + } + }, + { + "pk": 2, + "model": "core.mocktag", + "fields": { + "name": "secondary" + } + }, + { + "pk": 1, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "bar", + "pub_date": "2009-03-17 06:00:00", + "tag": 1 + } + }, + { + "pk": 2, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "bar", + "pub_date": "2009-03-17 07:00:00", + "tag": 1 + } + }, + { + "pk": 3, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "bar", + "pub_date": "2009-03-17 08:00:00", + "tag": 2 + } + }, + { + "pk": "sometext", + "model": "core.charpkmockmodel", + "fields": {} + }, + { + "pk": "1234", + "model": "core.charpkmockmodel", + "fields": {} + }, + { + "pk": 1, + "model": "core.afifthmockmodel", + "fields": { + "author": "sam1", + "deleted": false + } + }, + { + "pk": 2, + "model": "core.afifthmockmodel", + "fields": { + "author": "sam2", + "deleted": true + } + }, + { + "pk": "53554c58-7051-4350-bcc9-dad75eb248a9", + "model": "core.uuidmockmodel", + "fields": { + "characteristics": "some text that was indexed" + } + }, + { + "pk": "77554c58-7051-4350-bcc9-dad75eb24888", + "model": "core.uuidmockmodel", + "fields": { + "characteristics": "more text that was indexed" + } } - }, - { - "pk": 2, - "model": "core.mocktag", - "fields": { - "name": "secondary" - } - }, - { - "pk": 1, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "bar", - "pub_date": "2009-03-17 06:00:00", - "tag": 1 - } - }, - { - "pk": 2, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "bar", - "pub_date": "2009-03-17 07:00:00", - "tag": 1 - } - }, - { - "pk": 3, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "bar", - "pub_date": "2009-03-17 08:00:00", - "tag": 2 - } - }, - { - "pk": "sometext", - "model": "core.charpkmockmodel", - "fields": { - } - }, - { - "pk": "1234", - "model": "core.charpkmockmodel", - "fields": { - } - }, - { - "pk": 1, - "model": "core.afifthmockmodel", - "fields": { - "author": "sam1", - "deleted": false - } - }, - { - "pk": 2, - "model": "core.afifthmockmodel", - "fields": { - "author": "sam2", - "deleted": true - } - }, - { - "pk": "53554c58-7051-4350-bcc9-dad75eb248a9", - "model": "core.uuidmockmodel", - "fields": { - "characteristics": "some text that was indexed" - } - }, - { - "pk": "77554c58-7051-4350-bcc9-dad75eb24888", - "model": "core.uuidmockmodel", - "fields": { - "characteristics": "more text that was indexed" - } - } ] diff --git a/test_haystack/core/fixtures/bulk_data.json b/test_haystack/core/fixtures/bulk_data.json index 4e721d8d4..40bbf8a9d 100644 --- a/test_haystack/core/fixtures/bulk_data.json +++ b/test_haystack/core/fixtures/bulk_data.json @@ -1,262 +1,262 @@ [ - { - "pk": 1, - "model": "core.mocktag", - "fields": { - "name": "search_test" + { + "pk": 1, + "model": "core.mocktag", + "fields": { + "name": "search_test" + } + }, + { + "pk": 1, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:", + "pub_date": "2009-06-18 06:00:00", + "tag": 1 + } + }, + { + "pk": 2, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", + "pub_date": "2009-07-17 00:30:00", + "tag": 1 + } + }, + { + "pk": 3, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Every custom ``SearchIndex`` requires there be one and only one field with ``document=True``. This is the primary field that will get passed to the backend for indexing. For this field, you'll then need to create a template at ``search/indexes/myapp/note_text.txt``. This allows you to customize the document that will be passed to the search backend for indexing. A sample template might look like:", + "pub_date": "2009-06-18 08:00:00", + "tag": 1 + } + }, + { + "pk": 4, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "The exception to this is the TemplateField class. This take either no arguments or an explicit template name to populate their contents. You can find more information about them in the SearchIndex API reference.", + "pub_date": "2009-07-17 01:30:00", + "tag": 1 + } + }, + { + "pk": 5, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "This will pull in the default URLconf for Haystack. It consists of a single URLconf that points to a SearchView instance. You can change this class\u2019s behavior by passing it any of several keyword arguments or override it entirely with your own view.", + "pub_date": "2009-07-17 02:30:00", + "tag": 1 + } + }, + { + "pk": 6, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "This will create a default SearchIndex instance, search through all of your INSTALLED_APPS for search_indexes.py and register all SearchIndexes with the default SearchIndex. If autodiscovery and inclusion of all indexes is not desirable, you can manually register models in the following manner:", + "pub_date": "2009-07-17 03:30:00", + "tag": 1 + } + }, + { + "pk": 7, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchBackend class handles interaction directly with the backend. The search query it performs is usually fed to it from a SearchQuery class that has been built for that backend. This class must be at least partially implemented on a per-backend basis and is usually accompanied by a SearchQuery class within the same module.", + "pub_date": "2009-07-17 04:30:00", + "tag": 1 + } + }, + { + "pk": 8, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "Takes a query to search on and returns dictionary. The query should be a string that is appropriate syntax for the backend. The returned dictionary should contain the keys \u2018results\u2019 and \u2018hits\u2019. The \u2018results\u2019 value should be an iterable of populated SearchResult objects. The \u2018hits\u2019 should be an integer count of the number of matched results the search backend found. This method MUST be implemented by each backend, as it will be highly specific to each one.", + "pub_date": "2009-07-17 05:30:00", + "tag": 1 + } + }, + { + "pk": 9, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchQuery class acts as an intermediary between SearchQuerySet\u2018s abstraction and SearchBackend\u2018s actual search. Given the metadata provided by SearchQuerySet, SearchQuery build the actual query and interacts with the SearchBackend on SearchQuerySet\u2018s behalf. This class must be at least partially implemented on a per-backend basis, as portions are highly specific to the backend. It usually is bundled with the accompanying SearchBackend.", + "pub_date": "2009-07-17 06:30:00", + "tag": 1 + } + }, + { + "pk": 10, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Most people will NOT have to use this class directly. SearchQuerySet handles all interactions with SearchQuery objects and provides a nicer interface to work with. Should you need advanced/custom behavior, you can supply your version of SearchQuery that overrides/extends the class in the manner you see fit. SearchQuerySet objects take a kwarg parameter query where you can pass in your class.", + "pub_date": "2009-07-17 07:30:00", + "tag": 1 + } + }, + { + "pk": 11, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "The SearchQuery object maintains a list of QueryFilter objects. Each filter object supports what field it looks up against, what kind of lookup (i.e. the __\u2019s), what value it\u2019s looking for and if it\u2019s a AND/OR/NOT. The SearchQuery object\u2019s \u201cbuild_query\u201d method should then iterate over that list and convert that to a valid query for the search backend.", + "pub_date": "2009-07-17 08:30:00", + "tag": 1 + } + }, + { + "pk": 12, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The SearchSite provides a way to collect the SearchIndexes that are relevant to the current site, much like ModelAdmins in the admin app. This allows you to register indexes on models you don\u2019t control (reusable apps, django.contrib, etc.) as well as customize on a per-site basis what indexes should be available (different indexes for different sites, same codebase).", + "pub_date": "2009-07-17 09:30:00", + "tag": 1 + } + }, + { + "pk": 13, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "If you need to narrow the indexes that get registered, you will need to manipulate a SearchSite. There are two ways to go about this, via either register or unregister. If you want most of the indexes but want to forgo a specific one(s), you can setup the main site via autodiscover then simply unregister the one(s) you don\u2019t want.:", + "pub_date": "2009-07-17 10:30:00", + "tag": 1 + } + }, + { + "pk": 14, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The SearchIndex class allows the application developer a way to provide data to the backend in a structured format. Developers familiar with Django\u2019s Form or Model classes should find the syntax for indexes familiar. This class is arguably the most important part of integrating Haystack into your application, as it has a large impact on the quality of the search results and how easy it is for users to find what they\u2019re looking for. Care and effort should be put into making your indexes the best they can be.", + "pub_date": "2009-07-17 11:30:00", + "tag": 1 + } + }, + { + "pk": 15, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "Unlike relational databases, most search engines supported by Haystack are primarily document-based. They focus on a single text blob which they tokenize, analyze and index. When searching, this field is usually the primary one that is searched. Further, the schema used by most engines is the same for all types of data added, unlike a relational database that has a table schema for each chunk of data. It may be helpful to think of your search index as something closer to a key-value store instead of imagining it in terms of a RDBMS.", + "pub_date": "2009-07-17 12:30:00", + "tag": 1 + } + }, + { + "pk": 16, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Common uses include storing pertinent data information, categorizations of the document, author information and related data. By adding fields for these pieces of data, you provide a means to further narrow/filter search terms. This can be useful from either a UI perspective (a better advanced search form) or from a developer standpoint (section-dependent search, off-loading certain tasks to search, et cetera).", + "pub_date": "2009-07-17 13:30:00", + "tag": 1 + } + }, + { + "pk": 17, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Most search engines that were candidates for inclusion in Haystack all had a central concept of a document that they indexed. These documents form a corpus within which to primarily search. Because this ideal is so central and most of Haystack is designed to have pluggable backends, it is important to ensure that all engines have at least a bare minimum of the data they need to function.", + "pub_date": "2009-07-17 14:30:00", + "tag": 1 + } + }, + { + "pk": 18, + "model": "core.mockmodel", + "fields": { + "author": "daniel1", + "foo": "As a result, when creating a SearchIndex, at least one field must be marked with document=True. This signifies to Haystack that whatever is placed in this field while indexing is to be the primary text the search engine indexes. The name of this field can be almost anything, but text is one of the more common names used.", + "pub_date": "2009-07-17 15:30:00", + "tag": 1 + } + }, + { + "pk": 19, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "One shortcoming of the use of search is that you rarely have all or the most up-to-date information about an object in the index. As a result, when retrieving search results, you will likely have to access the object in the database to provide better information. However, this can also hit the database quite heavily (think .get(pk=result.id) per object). If your search is popular, this can lead to a big performance hit. There are two ways to prevent this. The first way is SearchQuerySet.load_all, which tries to group all similar objects and pull them though one query instead of many. This still hits the DB and incurs a performance penalty.", + "pub_date": "2009-07-17 16:30:00", + "tag": 1 + } + }, + { + "pk": 20, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "The other option is to leverage stored fields. By default, all fields in Haystack are both indexed (searchable by the engine) and stored (retained by the engine and presented in the results). By using a stored field, you can store commonly used data in such a way that you don\u2019t need to hit the database when processing the search result to get more information. By the way: Jenny's number is 867-5309", + "pub_date": "2009-07-17 17:30:00", + "tag": 1 + } + }, + { + "pk": 21, + "model": "core.mockmodel", + "fields": { + "author": "daniel2", + "foo": "For example, one great way to leverage this is to pre-rendering an object\u2019s search result template DURING indexing. You define an additional field, render a template with it and it follows the main indexed record into the index. Then, when that record is pulled when it matches a query, you can simply display the contents of that field, which avoids the database hit.:", + "pub_date": "2009-07-17 18:30:00", + "tag": 1 + } + }, + { + "pk": 22, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "However, sometimes, even more control over what gets placed in your index is needed. To facilitate this, SearchIndex objects have a \u2018preparation\u2019 stage that populates data just before it is indexed. You can hook into this phase in several ways. This should be very familiar to developers who have used Django\u2019s forms before as it loosely follows similar concepts, though the emphasis here is less on cleansing data from user input and more on making the data friendly to the search backend.", + "pub_date": "2009-07-17 19:30:00", + "tag": 1 + } + }, + { + "pk": 23, + "model": "core.mockmodel", + "fields": { + "author": "daniel3", + "foo": "Each SearchIndex gets a prepare method, which handles collecting all the data. This method should return a dictionary that will be the final data used by the search backend. Overriding this method is useful if you need to collect more than one piece of data or need to incorporate additional data that is not well represented by a single SearchField. An example might look like:", + "pub_date": "2009-07-17 20:30:00", + "tag": 1 + } + }, + { + "pk": 1, + "model": "core.anothermockmodel", + "fields": { + "author": "daniel3", + "pub_date": "2009-07-17 21:30:00" + } + }, + { + "pk": 2, + "model": "core.anothermockmodel", + "fields": { + "author": "daniel3", + "pub_date": "2009-07-17 22:30:00" + } + }, + { + "pk": 1, + "model": "core.ScoreMockModel", + "fields": { + "score": "42" + } } - }, - { - "pk": 1, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:", - "pub_date": "2009-06-18 06:00:00", - "tag": 1 - } - }, - { - "pk": 2, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - "pub_date": "2009-07-17 00:30:00", - "tag": 1 - } - }, - { - "pk": 3, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Every custom ``SearchIndex`` requires there be one and only one field with ``document=True``. This is the primary field that will get passed to the backend for indexing. For this field, you'll then need to create a template at ``search/indexes/myapp/note_text.txt``. This allows you to customize the document that will be passed to the search backend for indexing. A sample template might look like:", - "pub_date": "2009-06-18 08:00:00", - "tag": 1 - } - }, - { - "pk": 4, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "The exception to this is the TemplateField class. This take either no arguments or an explicit template name to populate their contents. You can find more information about them in the SearchIndex API reference.", - "pub_date": "2009-07-17 01:30:00", - "tag": 1 - } - }, - { - "pk": 5, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "This will pull in the default URLconf for Haystack. It consists of a single URLconf that points to a SearchView instance. You can change this class’s behavior by passing it any of several keyword arguments or override it entirely with your own view.", - "pub_date": "2009-07-17 02:30:00", - "tag": 1 - } - }, - { - "pk": 6, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "This will create a default SearchIndex instance, search through all of your INSTALLED_APPS for search_indexes.py and register all SearchIndexes with the default SearchIndex. If autodiscovery and inclusion of all indexes is not desirable, you can manually register models in the following manner:", - "pub_date": "2009-07-17 03:30:00", - "tag": 1 - } - }, - { - "pk": 7, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchBackend class handles interaction directly with the backend. The search query it performs is usually fed to it from a SearchQuery class that has been built for that backend. This class must be at least partially implemented on a per-backend basis and is usually accompanied by a SearchQuery class within the same module.", - "pub_date": "2009-07-17 04:30:00", - "tag": 1 - } - }, - { - "pk": 8, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "Takes a query to search on and returns dictionary. The query should be a string that is appropriate syntax for the backend. The returned dictionary should contain the keys ‘results’ and ‘hits’. The ‘results’ value should be an iterable of populated SearchResult objects. The ‘hits’ should be an integer count of the number of matched results the search backend found. This method MUST be implemented by each backend, as it will be highly specific to each one.", - "pub_date": "2009-07-17 05:30:00", - "tag": 1 - } - }, - { - "pk": 9, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchQuery class acts as an intermediary between SearchQuerySet‘s abstraction and SearchBackend‘s actual search. Given the metadata provided by SearchQuerySet, SearchQuery build the actual query and interacts with the SearchBackend on SearchQuerySet‘s behalf. This class must be at least partially implemented on a per-backend basis, as portions are highly specific to the backend. It usually is bundled with the accompanying SearchBackend.", - "pub_date": "2009-07-17 06:30:00", - "tag": 1 - } - }, - { - "pk": 10, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Most people will NOT have to use this class directly. SearchQuerySet handles all interactions with SearchQuery objects and provides a nicer interface to work with. Should you need advanced/custom behavior, you can supply your version of SearchQuery that overrides/extends the class in the manner you see fit. SearchQuerySet objects take a kwarg parameter query where you can pass in your class.", - "pub_date": "2009-07-17 07:30:00", - "tag": 1 - } - }, - { - "pk": 11, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "The SearchQuery object maintains a list of QueryFilter objects. Each filter object supports what field it looks up against, what kind of lookup (i.e. the __’s), what value it’s looking for and if it’s a AND/OR/NOT. The SearchQuery object’s “build_query” method should then iterate over that list and convert that to a valid query for the search backend.", - "pub_date": "2009-07-17 08:30:00", - "tag": 1 - } - }, - { - "pk": 12, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The SearchSite provides a way to collect the SearchIndexes that are relevant to the current site, much like ModelAdmins in the admin app. This allows you to register indexes on models you don’t control (reusable apps, django.contrib, etc.) as well as customize on a per-site basis what indexes should be available (different indexes for different sites, same codebase).", - "pub_date": "2009-07-17 09:30:00", - "tag": 1 - } - }, - { - "pk": 13, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "If you need to narrow the indexes that get registered, you will need to manipulate a SearchSite. There are two ways to go about this, via either register or unregister. If you want most of the indexes but want to forgo a specific one(s), you can setup the main site via autodiscover then simply unregister the one(s) you don’t want.:", - "pub_date": "2009-07-17 10:30:00", - "tag": 1 - } - }, - { - "pk": 14, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The SearchIndex class allows the application developer a way to provide data to the backend in a structured format. Developers familiar with Django’s Form or Model classes should find the syntax for indexes familiar. This class is arguably the most important part of integrating Haystack into your application, as it has a large impact on the quality of the search results and how easy it is for users to find what they’re looking for. Care and effort should be put into making your indexes the best they can be.", - "pub_date": "2009-07-17 11:30:00", - "tag": 1 - } - }, - { - "pk": 15, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "Unlike relational databases, most search engines supported by Haystack are primarily document-based. They focus on a single text blob which they tokenize, analyze and index. When searching, this field is usually the primary one that is searched. Further, the schema used by most engines is the same for all types of data added, unlike a relational database that has a table schema for each chunk of data. It may be helpful to think of your search index as something closer to a key-value store instead of imagining it in terms of a RDBMS.", - "pub_date": "2009-07-17 12:30:00", - "tag": 1 - } - }, - { - "pk": 16, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Common uses include storing pertinent data information, categorizations of the document, author information and related data. By adding fields for these pieces of data, you provide a means to further narrow/filter search terms. This can be useful from either a UI perspective (a better advanced search form) or from a developer standpoint (section-dependent search, off-loading certain tasks to search, et cetera).", - "pub_date": "2009-07-17 13:30:00", - "tag": 1 - } - }, - { - "pk": 17, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Most search engines that were candidates for inclusion in Haystack all had a central concept of a document that they indexed. These documents form a corpus within which to primarily search. Because this ideal is so central and most of Haystack is designed to have pluggable backends, it is important to ensure that all engines have at least a bare minimum of the data they need to function.", - "pub_date": "2009-07-17 14:30:00", - "tag": 1 - } - }, - { - "pk": 18, - "model": "core.mockmodel", - "fields": { - "author": "daniel1", - "foo": "As a result, when creating a SearchIndex, at least one field must be marked with document=True. This signifies to Haystack that whatever is placed in this field while indexing is to be the primary text the search engine indexes. The name of this field can be almost anything, but text is one of the more common names used.", - "pub_date": "2009-07-17 15:30:00", - "tag": 1 - } - }, - { - "pk": 19, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "One shortcoming of the use of search is that you rarely have all or the most up-to-date information about an object in the index. As a result, when retrieving search results, you will likely have to access the object in the database to provide better information. However, this can also hit the database quite heavily (think .get(pk=result.id) per object). If your search is popular, this can lead to a big performance hit. There are two ways to prevent this. The first way is SearchQuerySet.load_all, which tries to group all similar objects and pull them though one query instead of many. This still hits the DB and incurs a performance penalty.", - "pub_date": "2009-07-17 16:30:00", - "tag": 1 - } - }, - { - "pk": 20, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "The other option is to leverage stored fields. By default, all fields in Haystack are both indexed (searchable by the engine) and stored (retained by the engine and presented in the results). By using a stored field, you can store commonly used data in such a way that you don’t need to hit the database when processing the search result to get more information. By the way: Jenny's number is 867-5309", - "pub_date": "2009-07-17 17:30:00", - "tag": 1 - } - }, - { - "pk": 21, - "model": "core.mockmodel", - "fields": { - "author": "daniel2", - "foo": "For example, one great way to leverage this is to pre-rendering an object’s search result template DURING indexing. You define an additional field, render a template with it and it follows the main indexed record into the index. Then, when that record is pulled when it matches a query, you can simply display the contents of that field, which avoids the database hit.:", - "pub_date": "2009-07-17 18:30:00", - "tag": 1 - } - }, - { - "pk": 22, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "However, sometimes, even more control over what gets placed in your index is needed. To facilitate this, SearchIndex objects have a ‘preparation’ stage that populates data just before it is indexed. You can hook into this phase in several ways. This should be very familiar to developers who have used Django’s forms before as it loosely follows similar concepts, though the emphasis here is less on cleansing data from user input and more on making the data friendly to the search backend.", - "pub_date": "2009-07-17 19:30:00", - "tag": 1 - } - }, - { - "pk": 23, - "model": "core.mockmodel", - "fields": { - "author": "daniel3", - "foo": "Each SearchIndex gets a prepare method, which handles collecting all the data. This method should return a dictionary that will be the final data used by the search backend. Overriding this method is useful if you need to collect more than one piece of data or need to incorporate additional data that is not well represented by a single SearchField. An example might look like:", - "pub_date": "2009-07-17 20:30:00", - "tag": 1 - } - }, - { - "pk": 1, - "model": "core.anothermockmodel", - "fields": { - "author": "daniel3", - "pub_date": "2009-07-17 21:30:00" - } - }, - { - "pk": 2, - "model": "core.anothermockmodel", - "fields": { - "author": "daniel3", - "pub_date": "2009-07-17 22:30:00" - } - }, - { - "pk": 1, - "model": "core.ScoreMockModel", - "fields": { - "score": "42" - } - } ] diff --git a/test_haystack/core/templates/404.html b/test_haystack/core/templates/404.html index 838aa183a..21f5da261 100644 --- a/test_haystack/core/templates/404.html +++ b/test_haystack/core/templates/404.html @@ -1 +1 @@ -{% extends 'base.html' %} \ No newline at end of file +{% extends 'base.html' %} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_content.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt index d2cca8d7d..a8f8b85b3 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_extra.txt @@ -1,2 +1,2 @@ Stored! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_template.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt b/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt index 837f4871b..9700a3fe0 100644 --- a/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt +++ b/test_haystack/core/templates/search/indexes/core/mockmodel_text.txt @@ -1,2 +1,2 @@ Indexed! -{{ object.pk }} \ No newline at end of file +{{ object.pk }} diff --git a/test_haystack/core/templates/search/search.html b/test_haystack/core/templates/search/search.html index 838aa183a..21f5da261 100644 --- a/test_haystack/core/templates/search/search.html +++ b/test_haystack/core/templates/search/search.html @@ -1 +1 @@ -{% extends 'base.html' %} \ No newline at end of file +{% extends 'base.html' %} diff --git a/test_haystack/core/templates/test_suggestion.html b/test_haystack/core/templates/test_suggestion.html index fa6240381..58df73ee3 100644 --- a/test_haystack/core/templates/test_suggestion.html +++ b/test_haystack/core/templates/test_suggestion.html @@ -1 +1 @@ -Suggestion: {{ suggestion }} \ No newline at end of file +Suggestion: {{ suggestion }} diff --git a/test_haystack/discovery/templates/search/indexes/bar_text.txt b/test_haystack/discovery/templates/search/indexes/bar_text.txt index 07070f0c9..4665cb553 100644 --- a/test_haystack/discovery/templates/search/indexes/bar_text.txt +++ b/test_haystack/discovery/templates/search/indexes/bar_text.txt @@ -1,2 +1,2 @@ {{ object.title }} -{{ object.body }} \ No newline at end of file +{{ object.body }} diff --git a/test_haystack/solr_tests/server/wait-for-solr b/test_haystack/solr_tests/server/wait-for-solr index 3b2f69a25..88446d939 100755 --- a/test_haystack/solr_tests/server/wait-for-solr +++ b/test_haystack/solr_tests/server/wait-for-solr @@ -8,7 +8,7 @@ import requests max_retries = 100 retry_count = 0 retry_delay = 15 -status_url = 'http://localhost:9001/solr/collection1/admin/ping' +status_url = "http://localhost:9001/solr/collection1/admin/ping" while retry_count < max_retries: @@ -20,14 +20,18 @@ while retry_count < max_retries: if status_code == 200: sys.exit(0) except Exception as exc: - print('Unhandled exception requesting %s: %s' % (status_url, exc), file=sys.stderr) + print( + "Unhandled exception requesting %s: %s" % (status_url, exc), file=sys.stderr + ) retry_count += 1 - print('Waiting {0} seconds for Solr to start (retry #{1}, status {2})'.format(retry_delay, - retry_count, - status_code), - file=sys.stderr) + print( + "Waiting {0} seconds for Solr to start (retry #{1}, status {2})".format( + retry_delay, retry_count, status_code + ), + file=sys.stderr, + ) time.sleep(retry_delay) From 0b7b2743d0b3cf4a1f98e96bc00f45dc83bc66c9 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:28:30 -0400 Subject: [PATCH 012/162] Remove black + isort GitHub action This can be handled as a subset of the pre-commit checks --- .github/workflows/black+isort.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .github/workflows/black+isort.yml diff --git a/.github/workflows/black+isort.yml b/.github/workflows/black+isort.yml deleted file mode 100644 index 20ea48d90..000000000 --- a/.github/workflows/black+isort.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: black+isort - -on: [pull_request, push] - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install tools - run: pip install black isort - - name: Run black+isort - run: | - black --check --diff . - isort --check . From 2782c063d75e02eef5e82be95b18fe5196139f36 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 27 Aug 2021 18:35:56 -0400 Subject: [PATCH 013/162] Update tests which depended on missing trailing newlines --- .../elasticsearch2_tests/test_backend.py | 16 +++++++----- .../elasticsearch5_tests/test_backend.py | 16 +++++++----- .../elasticsearch7_tests/test_backend.py | 16 +++++++----- .../test_elasticsearch_backend.py | 26 +++++++++++-------- test_haystack/solr_tests/test_solr_backend.py | 18 ++++++++----- test_haystack/test_fields.py | 2 +- test_haystack/test_indexes.py | 6 ++--- test_haystack/whoosh_tests/test_forms.py | 2 +- 8 files changed, 61 insertions(+), 41 deletions(-) diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py index aa2e9d7a5..0ec9608b0 100644 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ b/test_haystack/elasticsearch2_tests/test_backend.py @@ -343,7 +343,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -352,7 +352,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -361,7 +361,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -396,7 +396,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -405,7 +405,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -458,7 +458,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/elasticsearch5_tests/test_backend.py b/test_haystack/elasticsearch5_tests/test_backend.py index 66b8af395..a4ed34fdd 100644 --- a/test_haystack/elasticsearch5_tests/test_backend.py +++ b/test_haystack/elasticsearch5_tests/test_backend.py @@ -343,7 +343,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -352,7 +352,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -361,7 +361,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -396,7 +396,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -405,7 +405,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -458,7 +458,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/elasticsearch7_tests/test_backend.py b/test_haystack/elasticsearch7_tests/test_backend.py index f473e41cb..11edc66d5 100644 --- a/test_haystack/elasticsearch7_tests/test_backend.py +++ b/test_haystack/elasticsearch7_tests/test_backend.py @@ -344,7 +344,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -353,7 +353,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -362,7 +362,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -397,7 +397,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -406,7 +406,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -480,7 +480,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1", + "Indexed!\n2", + "Indexed!\n3", + ], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py index 73bcfddc5..665b00cea 100644 --- a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py +++ b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py @@ -360,7 +360,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00", "id": "core.mockmodel.1", }, @@ -369,7 +369,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -378,7 +378,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -413,7 +413,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00", "id": "core.mockmodel.2", }, @@ -422,7 +422,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00", "id": "core.mockmodel.3", }, @@ -482,7 +482,11 @@ def test_search(self): for result in self.sb.search("Index", highlight=True)["results"] ] ), - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) self.assertEqual( sorted( @@ -495,9 +499,9 @@ def test_search(self): ] ), [ - "Indexed!\n1", - "Indexed!\n2", - "Indexed!\n3", + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", ], ) @@ -958,13 +962,13 @@ def test_count(self): def test_highlight(self): reset_search_queries() results = self.sqs.filter(content="index").highlight() - self.assertEqual(results[0].highlighted, ["Indexed!\n1"]) + self.assertEqual(results[0].highlighted, ["Indexed!\n1\n"]) def test_highlight_options(self): reset_search_queries() results = self.sqs.filter(content="index") results = results.highlight(pre_tags=[""], post_tags=[""]) - self.assertEqual(results[0].highlighted, ["Indexed!\n1"]) + self.assertEqual(results[0].highlighted, ["Indexed!\n1\n"]) def test_manual_iter(self): results = self.sqs.all() diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index 6e82ea6f0..d20347e7e 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -267,7 +267,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel1", "name_exact": "daniel1", - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "pub_date": "2009-02-24T00:00:00Z", "id": "core.mockmodel.1", }, @@ -276,7 +276,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00Z", "id": "core.mockmodel.2", }, @@ -285,7 +285,7 @@ def test_update(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00Z", "id": "core.mockmodel.3", }, @@ -321,7 +321,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel2", "name_exact": "daniel2", - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "pub_date": "2009-02-23T00:00:00Z", "id": "core.mockmodel.2", }, @@ -330,7 +330,7 @@ def test_remove(self): "django_ct": "core.mockmodel", "name": "daniel3", "name_exact": "daniel3", - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "pub_date": "2009-02-22T00:00:00Z", "id": "core.mockmodel.3", }, @@ -400,7 +400,11 @@ def test_search(self): result.highlighted["text"][0] for result in self.sb.search("Index", highlight=True)["results"] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + [ + "Indexed!\n1\n", + "Indexed!\n2\n", + "Indexed!\n3\n", + ], ) # shortened highlighting options @@ -428,7 +432,7 @@ def test_search(self): "results" ] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + ["Indexed!\n1\n", "Indexed!\n2\n", "Indexed!\n3\n"], ) self.assertEqual(self.sb.search("Indx")["hits"], 0) diff --git a/test_haystack/test_fields.py b/test_haystack/test_fields.py index bb5cf3f4b..8f5b36301 100644 --- a/test_haystack/test_fields.py +++ b/test_haystack/test_fields.py @@ -597,7 +597,7 @@ def test_prepare(self): template3 = CharField(use_template=True) template3.instance_name = "template" - self.assertEqual(template3.prepare(mock), "Indexed!\n1") + self.assertEqual(template3.prepare(mock), "Indexed!\n1\n") template4 = CharField(use_template=True, template_name="search/indexes/foo.txt") template4.instance_name = "template" diff --git a/test_haystack/test_indexes.py b/test_haystack/test_indexes.py index 74e4e7755..19481ea51 100644 --- a/test_haystack/test_indexes.py +++ b/test_haystack/test_indexes.py @@ -168,7 +168,7 @@ def setUp(self): self.sample_docs = { "core.mockmodel.1": { - "text": "Indexed!\n1", + "text": "Indexed!\n1\n", "django_id": "1", "django_ct": "core.mockmodel", "extra": "Stored!\n1", @@ -177,7 +177,7 @@ def setUp(self): "id": "core.mockmodel.1", }, "core.mockmodel.2": { - "text": "Indexed!\n2", + "text": "Indexed!\n2\n", "django_id": "2", "django_ct": "core.mockmodel", "extra": "Stored!\n2", @@ -186,7 +186,7 @@ def setUp(self): "id": "core.mockmodel.2", }, "core.mockmodel.3": { - "text": "Indexed!\n3", + "text": "Indexed!\n3\n", "django_id": "3", "django_ct": "core.mockmodel", "extra": "Stored!\n3", diff --git a/test_haystack/whoosh_tests/test_forms.py b/test_haystack/whoosh_tests/test_forms.py index 9899807c4..204d14f46 100644 --- a/test_haystack/whoosh_tests/test_forms.py +++ b/test_haystack/whoosh_tests/test_forms.py @@ -37,4 +37,4 @@ def test_view_suggestion(self): mock = HttpRequest() mock.GET["q"] = "exampl" resp = view(mock) - self.assertEqual(resp.content, b"Suggestion: example") + self.assertEqual(resp.content, b"Suggestion: example\n") From cce4189573d2e121e0b37a9b4e47ee15cd0f716b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 19:32:11 +0000 Subject: [PATCH 014/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - https://github.com/pre-commit/mirrors-isort → https://github.com/PyCQA/isort - [github.com/PyCQA/isort: v5.9.3 → 5.9.3](https://github.com/PyCQA/isort/compare/v5.9.3...5.9.3) - [github.com/psf/black: 21.7b0 → 21.8b0](https://github.com/psf/black/compare/21.7b0...21.8b0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1008976dc..495f2ed4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,11 @@ exclude: ".*/vendor/.*" repos: - - repo: https://github.com/pre-commit/mirrors-isort - rev: v5.9.3 + - repo: https://github.com/PyCQA/isort + rev: 5.9.3 hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.7b0 + rev: 21.8b0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From f6070e827b041a0a1839fb0a0da8683d7fd3f913 Mon Sep 17 00:00:00 2001 From: fabiopiovam Date: Tue, 31 Aug 2021 16:05:20 -0300 Subject: [PATCH 015/162] add AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 60a8e82a4..29f6dbf00 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,3 +118,4 @@ Thanks to * João Junior (@joaojunior) and Bruno Marques (@ElSaico) for Elasticsearch 2.x support * Alex Tomkins (@tomkins) for various patches * Martin Pauly (@mpauly) for Django 2.0 support + * Fábio Piovam (@fabiopiovam) for date_facet on Solr 6.6+ From fe2ecd20ca949bcb235286ece66fb2721c4fd7ff Mon Sep 17 00:00:00 2001 From: fabiopiovam Date: Tue, 31 Aug 2021 16:16:44 -0300 Subject: [PATCH 016/162] add AUTHORS --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 29f6dbf00..4ea16e074 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,4 +118,10 @@ Thanks to * João Junior (@joaojunior) and Bruno Marques (@ElSaico) for Elasticsearch 2.x support * Alex Tomkins (@tomkins) for various patches * Martin Pauly (@mpauly) for Django 2.0 support + * Ryan Jarvis (@cabalist) for some code cleanup + * Dulmandakh Sukhbaatar (@dulmandakh) for GitHub Actions support, and flake8, black, isort checks. + * Deniz Dogan (@denizdogan) for adding support for the ``analyzer`` parameter for the Whoosh backend + * parruc for basic Whoosh faceting support + * Jens Kadenbach (audax) for updating and testing Whoosh faceting support + * Alejandro Sedeño (asedeno) trying the Whoosh faceting thing again * Fábio Piovam (@fabiopiovam) for date_facet on Solr 6.6+ From 99091621fb3b7fac0750aea7e9ee92cb6ba8d901 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 19:22:16 +0000 Subject: [PATCH 017/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/backends/solr_backend.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index d8425338e..dc929bf33 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -281,23 +281,25 @@ def build_search_kwargs( if date_facets is not None: kwargs["facet"] = "on" - kwargs["facet.%s" % self.date_facet_field ] = date_facets.keys() - kwargs["facet.%s.other" % self.date_facet_field ] = "none" + kwargs["facet.%s" % self.date_facet_field] = date_facets.keys() + kwargs["facet.%s.other" % self.date_facet_field] = "none" for key, value in date_facets.items(): - kwargs["f.%s.facet.%s.start" % (key, self.date_facet_field)] = self.conn._from_python( - value.get("start_date") - ) - kwargs["f.%s.facet.%s.end" % (key, self.date_facet_field)] = self.conn._from_python( - value.get("end_date") - ) + kwargs[ + "f.%s.facet.%s.start" % (key, self.date_facet_field) + ] = self.conn._from_python(value.get("start_date")) + kwargs[ + "f.%s.facet.%s.end" % (key, self.date_facet_field) + ] = self.conn._from_python(value.get("end_date")) gap_by_string = value.get("gap_by").upper() gap_string = "%d%s" % (value.get("gap_amount"), gap_by_string) if value.get("gap_amount") != 1: gap_string += "S" - kwargs["f.%s.facet.%s.gap" % (key, self.date_facet_field)] = "+%s/%s" % ( + kwargs[ + "f.%s.facet.%s.gap" % (key, self.date_facet_field) + ] = "+%s/%s" % ( gap_string, gap_by_string, ) @@ -498,13 +500,16 @@ def _process_results( ) ) - for key in ['ranges']: + for key in ["ranges"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( - zip(facets[key][facet_field]['counts'][::2], - facets[key][facet_field]['counts'][1::2])) + zip( + facets[key][facet_field]["counts"][::2], + facets[key][facet_field]["counts"][1::2], + ) + ) if self.include_spelling and hasattr(raw_results, "spellcheck"): try: From 469d561e36c2f4c1a9472c87bf948663b05afd78 Mon Sep 17 00:00:00 2001 From: fabiopiovam Date: Tue, 31 Aug 2021 16:39:40 -0300 Subject: [PATCH 018/162] update AUTHORS --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 4ea16e074..1842e588d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,4 +124,4 @@ Thanks to * parruc for basic Whoosh faceting support * Jens Kadenbach (audax) for updating and testing Whoosh faceting support * Alejandro Sedeño (asedeno) trying the Whoosh faceting thing again - * Fábio Piovam (@fabiopiovam) for date_facet on Solr 6.6+ + * Fábio Piovam (fabiopiovam) for date_facet on Solr 6.6+ From 4c08720d3621364bf0c92c18921eb9b541d154ca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Sep 2021 20:06:50 +0000 Subject: [PATCH 019/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 21.8b0 → 21.9b0](https://github.com/psf/black/compare/21.8b0...21.9b0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 495f2ed4f..70f3b02a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.8b0 + rev: 21.9b0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From a1eabc0337bb91890042b779c0314853f6fad54b Mon Sep 17 00:00:00 2001 From: Joshua Brooks Date: Thu, 14 Oct 2021 23:37:40 +0900 Subject: [PATCH 020/162] use ngettext not ungettext --- haystack/admin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/haystack/admin.py b/haystack/admin.py index 83ebe398e..22b0a34f1 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -4,7 +4,7 @@ from django.core.paginator import InvalidPage, Paginator from django.shortcuts import render from django.utils.encoding import force_str -from django.utils.translation import ungettext +from django.utils.translation import ngettext from haystack import connections from haystack.constants import DEFAULT_ALIAS @@ -116,12 +116,12 @@ def changelist_view(self, request, extra_context=None): else: action_form = None - selection_note = ungettext( + selection_note = ngettext( "0 of %(count)d selected", "of %(count)d selected", len(changelist.result_list), ) - selection_note_all = ungettext( + selection_note_all = ngettext( "%(total_count)s selected", "All %(total_count)s selected", changelist.result_count, From ed9a742c14705f5e9693d3b93ed3a598b088a8b1 Mon Sep 17 00:00:00 2001 From: Edmund <2623895+edmundlam@users.noreply.github.com> Date: Tue, 19 Oct 2021 11:58:49 -0400 Subject: [PATCH 021/162] Fix typo in tutorial document --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 89bbe7587..b902b7894 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -4,7 +4,7 @@ Getting Started with Haystack ============================= -Search is a topic of ever increasing importance. Users increasing rely on search +Search is a topic of ever increasing importance. Users increasingly rely on search to separate signal from noise and find what they're looking for quickly. In addition, search can provide insight into what things are popular (many searches), what things are difficult to find on the site and ways you can From 97e20f866b7ba694b12eb5d08d90222a94fe6d9b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 20:35:03 +0000 Subject: [PATCH 022/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 21.9b0 → 21.10b0](https://github.com/psf/black/compare/21.9b0...21.10b0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70f3b02a5..2fa6069ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.9b0 + rev: 21.10b0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From b6dd72e6b5c97b782f5436b7bb4e8227ba6e3b06 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Tue, 2 Nov 2021 11:21:13 -0400 Subject: [PATCH 023/162] Update doc link to Whoosh Closes #1757 --- docs/backend_support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backend_support.rst b/docs/backend_support.rst index e32d99f44..083f46b87 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -15,7 +15,7 @@ Supported Backends .. _Solr: http://lucene.apache.org/solr/ .. _ElasticSearch: http://elasticsearch.org/ -.. _Whoosh: https://github.com/mchaput/whoosh/ +.. _Whoosh: https://github.com/whoosh-community/whoosh/ .. _Xapian: http://xapian.org/ From 86f4d9a8045abb311ef399d39daae11e3b0c9b3d Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 27 Dec 2021 12:05:54 +0800 Subject: [PATCH 024/162] forms.py: Fix Django 4.0 compatibility --- haystack/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/haystack/forms.py b/haystack/forms.py index af9dd6964..7490b6f1d 100644 --- a/haystack/forms.py +++ b/haystack/forms.py @@ -1,5 +1,4 @@ from django import forms -from django.utils.encoding import smart_text from django.utils.text import capfirst from django.utils.translation import gettext_lazy as _ @@ -8,11 +7,12 @@ from haystack.query import EmptySearchQuerySet, SearchQuerySet from haystack.utils import get_model_ct from haystack.utils.app_loading import haystack_get_model +from django.utils.encoding import smart_str def model_choices(using=DEFAULT_ALIAS): choices = [ - (get_model_ct(m), capfirst(smart_text(m._meta.verbose_name_plural))) + (get_model_ct(m), capfirst(smart_str(m._meta.verbose_name_plural))) for m in connections[using].get_unified_index().get_indexed_models() ] return sorted(choices, key=lambda x: x[1]) From fccf6ae00e8cde934db88dc6d9b79a8e4156f0d7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Mon, 27 Dec 2021 17:05:43 +0800 Subject: [PATCH 025/162] Add Django 4.0 to test matrix --- .github/workflows/test.yml | 12 ++++++++++++ tox.ini | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4001699bb..fcce393d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,18 @@ jobs: - django-version: 2.2 python-version: 3.5 elastic-version: '7.13.1' + - django-version: '4.0' + python-version: 3.8 + elastic-version: 5.5 + - django-version: '4.0' + python-version: 3.8 + elastic-version: '7.13.1' + - django-version: '4.0' + python-version: 3.9 + elastic-version: 5.5 + - django-version: '4.0' + python-version: 3.9 + elastic-version: '7.13.1' services: elastic: image: elasticsearch:${{ matrix.elastic-version }} diff --git a/tox.ini b/tox.ini index 9eefabfc8..2cf5472a9 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = docs py35-django2.2-es{1.x,2.x,5.x,7.x} - py{36,37,38,py}-django{2.2,3.0}-es{1.x,2.x,5.x,7.x} + py{36,37,38,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} [testenv] @@ -13,6 +13,9 @@ deps = requests django2.2: Django>=2.2,<3.0 django3.0: Django>=3.0,<3.1 + django3.1: Django>=3.1,<3.2 + django3.2: Django>=3.2,<3.3 + django4.0: Django>=4.0,<4.1 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6 From 92525bd79ac508b576ddc41bcc98600730ee3637 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Dec 2021 09:10:37 +0000 Subject: [PATCH 026/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/forms.py b/haystack/forms.py index 7490b6f1d..b7dd1e28c 100644 --- a/haystack/forms.py +++ b/haystack/forms.py @@ -1,4 +1,5 @@ from django import forms +from django.utils.encoding import smart_str from django.utils.text import capfirst from django.utils.translation import gettext_lazy as _ @@ -7,7 +8,6 @@ from haystack.query import EmptySearchQuerySet, SearchQuerySet from haystack.utils import get_model_ct from haystack.utils.app_loading import haystack_get_model -from django.utils.encoding import smart_str def model_choices(using=DEFAULT_ALIAS): From 0964383817384cc4a3ecdbfd5fe75db8f853f6f7 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Sun, 9 Jan 2022 16:46:53 +0800 Subject: [PATCH 027/162] tox.ini: Add py39 and py310 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2cf5472a9..1b79eb3dd 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = docs py35-django2.2-es{1.x,2.x,5.x,7.x} - py{36,37,38,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} + py{36,37,38,39,310,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} [testenv] From 182b4c423b44cfa855869b76b0933200672bbbfc Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 19 Jan 2022 17:25:26 +0100 Subject: [PATCH 028/162] fix admin search_fields: page_num --- haystack/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/admin.py b/haystack/admin.py index 22b0a34f1..feeb1f3f3 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -41,7 +41,7 @@ def get_results(self, request): # Get the list of objects to display on this page. try: - result_list = paginator.page(self.page_num + 1).object_list + result_list = paginator.page(self.page_num).object_list # Grab just the Django models, since that's what everything else is # expecting. result_list = [result.object for result in result_list] From 3f2b2925012798ae23adada10ebf23827f6e6afc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 22:32:29 +0000 Subject: [PATCH 029/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/isort: 5.9.3 → 5.10.1](https://github.com/PyCQA/isort/compare/5.9.3...5.10.1) - [github.com/psf/black: 21.10b0 → 22.1.0](https://github.com/psf/black/compare/21.10b0...22.1.0) - [github.com/pre-commit/pre-commit-hooks: v4.0.1 → v4.1.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.0.1...v4.1.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fa6069ae..5c4116a6d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/PyCQA/isort - rev: 5.9.3 + rev: 5.10.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 21.10b0 + rev: 22.1.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.1.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] From b9bf3113689574c4d20e7332c703da385143387f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 22:32:51 +0000 Subject: [PATCH 030/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/management/commands/update_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py index 6dc9155f5..da50644bc 100644 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -150,7 +150,7 @@ def do_update( LOG.warning(error_msg, error_context, exc_info=True) # If going to try again, sleep a bit before - time.sleep(2 ** retries) + time.sleep(2**retries) # Clear out the DB connections queries because it bloats up RAM. reset_queries() From d4902f64c04e5caa5e9db2e85fe9d5f3bc293d50 Mon Sep 17 00:00:00 2001 From: Greg Baker Date: Tue, 30 Nov 2021 12:10:43 -0800 Subject: [PATCH 031/162] document Elasticsearch 7.x compatibility --- docs/backend_support.rst | 2 +- docs/installing_search_engines.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/backend_support.rst b/docs/backend_support.rst index 083f46b87..59614a83b 100644 --- a/docs/backend_support.rst +++ b/docs/backend_support.rst @@ -50,7 +50,7 @@ ElasticSearch * Stored (non-indexed) fields * Highlighting * Spatial search -* Requires: `elasticsearch-py `_ 1.x, 2.x, or 5.X. +* Requires: `elasticsearch-py `_ 1.x, 2.x, 5.X, or 7.X. Whoosh ------ diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 47753773a..50bd6fb06 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -153,7 +153,7 @@ Elasticsearch is similar to Solr — another Java application using Lucene — b focused on ease of deployment and clustering. See https://www.elastic.co/products/elasticsearch for more information. -Haystack currently supports Elasticsearch 1.x, 2.x, and 5.x. +Haystack currently supports Elasticsearch 1.x, 2.x, 5.x, and 7.x. Follow the instructions on https://www.elastic.co/downloads/elasticsearch to download and install Elasticsearch and configure it for your environment. @@ -161,7 +161,7 @@ download and install Elasticsearch and configure it for your environment. You'll also need to install the Elasticsearch binding: elasticsearch_ for the appropriate backend version — for example:: - $ pip install "elasticsearch>=5,<6" + $ pip install "elasticsearch>=7,<8" .. _elasticsearch: https://pypi.python.org/pypi/elasticsearch/ From 0b40ceb4bfa4a5e1bb03ddaba4a6dea2b75e0ddb Mon Sep 17 00:00:00 2001 From: Nakarin Hansawattana Date: Fri, 14 Jan 2022 20:23:26 +0700 Subject: [PATCH 032/162] Update setup.py for Elasticsearch 7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6033e8dfd..3224ed2a1 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ install_requires=install_requires, tests_require=tests_require, extras_require={ - "elasticsearch": ["elasticsearch>=5,<6"], + "elasticsearch": ["elasticsearch>=5,<8"], }, test_suite="test_haystack.run_tests.run_all", ) From 51a664069e025f388bc6458e7dc5e250f2f8e5b9 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 2 Feb 2022 23:02:28 +0100 Subject: [PATCH 033/162] Remove the who_uses docs page Such a page is a nightmare to maintain on the long term. --- docs/index.rst | 1 - docs/toc.rst | 1 - docs/who_uses.rst | 367 ---------------------------------------------- 3 files changed, 369 deletions(-) delete mode 100644 docs/who_uses.rst diff --git a/docs/index.rst b/docs/index.rst index 747fdf733..f5267e9c2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,7 +43,6 @@ you up and running: glossary management_commands faq - who_uses other_apps installing_search_engines debugging diff --git a/docs/toc.rst b/docs/toc.rst index 06ab413dd..4ae1c846a 100644 --- a/docs/toc.rst +++ b/docs/toc.rst @@ -15,7 +15,6 @@ Table Of Contents installing_search_engines settings faq - who_uses other_apps debugging diff --git a/docs/who_uses.rst b/docs/who_uses.rst deleted file mode 100644 index dc8a13c1a..000000000 --- a/docs/who_uses.rst +++ /dev/null @@ -1,367 +0,0 @@ -.. _ref-who-uses: - -Sites Using Haystack -==================== - -The following sites are a partial list of people using Haystack. I'm always -interested in adding more sites, so please find me (``daniellindsley``) via -IRC or the mailing list thread. - - -LJWorld/Lawrence.com/KUSports ------------------------------ - -For all things search-related. - -Using: Solr - -* http://www2.ljworld.com/search/ -* http://www2.ljworld.com/search/vertical/news.story/ -* http://www2.ljworld.com/marketplace/ -* http://www.lawrence.com/search/ -* http://www.kusports.com/search/ - - -AltWeeklies ------------ - -Providing an API to story aggregation. - -Using: Whoosh - -* http://www.northcoastjournal.com/altweeklies/documentation/ - - -Teachoo ------------ - -Teachoo uses Haystack for its site search. - -Using: Elasticsearch - -* https://www.teachoo.com/ - - -Trapeze -------- - -Various projects. - -Using: Xapian - -* http://www.trapeze.com/ -* http://www.windmobile.ca/ -* http://www.bonefishgrill.com/ -* http://www.canadiantire.ca/ (Portions of) - - -Vickerey.com ------------- - -For (really well done) search & faceting. - -Using: Solr - -* http://store.vickerey.com/products/search/ - - -Eldarion --------- - -Various projects. - -Using: Solr - -* http://eldarion.com/ - - -Sunlight Labs -------------- - -For general search. - -Using: Whoosh & Solr - -* http://sunlightlabs.com/ -* http://subsidyscope.com/ - - -NASA ----- - -For general search. - -Using: Solr - -* An internal site called SMD Spacebook 1.1. -* http://science.nasa.gov/ - - -AllForLocal ------------ - -For general search. - -* http://www.allforlocal.com/ - - -HUGE ----- - -Various projects. - -Using: Solr - -* http://hugeinc.com/ -* http://houselogic.com/ - - -Brick Design ------------- - -For search on Explore. - -Using: Solr - -* http://bricksf.com/ -* http://explore.org/ - - -Winding Road ------------- - -For general search. - -Using: Solr - -* http://www.windingroad.com/ - - -Reddit ------- - -For Reddit Gifts. - -Using: Whoosh - -* http://redditgifts.com/ - - -Pegasus News ------------- - -For general search. - -Using: Xapian - -* http://www.pegasusnews.com/ - - -Rampframe ---------- - -For general search. - -Using: Xapian - -* http://www.rampframe.com/ - - -Forkinit --------- - -For general search, model-specific search and suggestions via MLT. - -Using: Solr - -* http://forkinit.com/ - - -Structured Abstraction ----------------------- - -For general search. - -Using: Xapian - -* http://www.structuredabstraction.com/ -* http://www.delivergood.org/ - - -CustomMade ----------- - -For general search. - -Using: Solr - -* http://www.custommade.com/ - - -University of the Andes, Dept. of Political Science ---------------------------------------------------- - -For general search & section-specific search. Developed by Monoku. - -Using: Solr - -* http://www.congresovisible.org/ -* http://www.monoku.com/ - - -Christchurch Art Gallery ------------------------- - -For general search & section-specific search. - -Using: Solr - -* http://christchurchartgallery.org.nz/search/ -* http://christchurchartgallery.org.nz/collection/browse/ - - -DevCheatSheet.com ------------------ - -For general search. - -Using: Xapian - -* http://devcheatsheet.com/ - - -TodasLasRecetas ---------------- - -For search, faceting & More Like This. - -Using: Solr - -* http://www.todaslasrecetas.es/receta/s/?q=langostinos -* http://www.todaslasrecetas.es/receta/9526/brochetas-de-langostinos - - -AstroBin --------- - -For general search. - -Using: Solr - -* http://www.astrobin.com/ - - -European Paper Company ----------------------- - -For general search. - -Using: ??? - -* http://europeanpaper.com/ - - -mtn-op ------- - -For general search. - -Using: ??? - -* http://mountain-op.com/ - - -Crate ------ - -Crate is a PyPI mirror/replacement. It's using Haystack to power all search & -faceted navigation on the site. - -Using: Elasticsearch - -* https://crate.io/ - - -Pix Populi ----------- - -Pix Populi is a popular French photo sharing site. - -Using: Solr - -* http://www.pix-populi.fr/ - - -LocalWiki ----------- - -LocalWiki is a tool for collaborating in local, geographic communities. -It's using Haystack to power search on every LocalWiki instance. - -Using: Solr - -* http://localwiki.org/ - - -Pitchup -------- - -For faceting, geo and autocomplete. - -Using: ??? - -* http://www.pitchup.com/search/ - - -Gidsy ------ - -Gidsy makes it easy for anyone to organize and find exciting things -to do everywhere in the world. - -For activity search, area pages, forums and private messages. - -Using: Elasticsearch - -* https://gidsy.com/ -* https://gidsy.com/search/ -* https://gidsy.com/forum/ - - -GroundCity ----------- - -Groundcity is a Romanian dynamic real estate site. - -For real estate, forums and comments. - -Using: Whoosh - -* http://groundcity.ro/cautare/ - - -Docket Alarm ------------- - -Docket Alarm allows people to search court dockets across -the country. With it, you can search court dockets in the International Trade -Commission (ITC), the Patent Trial and Appeal Board (PTAB) and All Federal -Courts. - -Using: Elasticsearch - -* https://www.docketalarm.com/search/ITC -* https://www.docketalarm.com/search/PTAB -* https://www.docketalarm.com/search/dockets - - -Educreations -------------- - -Educreations makes it easy for anyone to teach what they know and learn -what they don't with a recordable whiteboard. Haystack is used to -provide search across users and lessons. - -Using: Solr - -* http://www.educreations.com/browse/ From 77ada257c4b76b1e2d97b803599c6f1690057849 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 2 Feb 2022 21:54:00 +0100 Subject: [PATCH 034/162] [WIP] Use standard Django test runner --- .github/workflows/test.yml | 6 +++-- docs/running_tests.rst | 14 ++++------ setup.py | 1 - test_haystack/__init__.py | 27 ------------------- .../elasticsearch2_tests/__init__.py | 12 ++++++--- .../elasticsearch5_tests/__init__.py | 12 ++++++--- .../elasticsearch7_tests/__init__.py | 12 ++++++--- test_haystack/elasticsearch_tests/__init__.py | 12 ++++++--- test_haystack/multipleindex/__init__.py | 24 +++++------------ test_haystack/multipleindex/tests.py | 21 +++++++++++++-- test_haystack/run_tests.py | 21 +++++---------- test_haystack/solr_tests/__init__.py | 11 +++++--- test_haystack/spatial/__init__.py | 9 ++++++- tox.ini | 8 ++++-- 14 files changed, 94 insertions(+), 96 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcce393d1..7444c776e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,8 +63,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel - pip install coverage requests + pip install coverage requests tox tox-gh-actions pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} python setup.py clean build install - name: Run test - run: coverage run setup.py test + run: tox -v + env: + DJANGO: ${{ matrix.django-version }} diff --git a/docs/running_tests.rst b/docs/running_tests.rst index 76d4daea8..f016a91e9 100644 --- a/docs/running_tests.rst +++ b/docs/running_tests.rst @@ -29,17 +29,13 @@ the errors persist. To run just a portion of the tests you can use the script ``run_tests.py`` and just specify the files or directories you wish to run, for example:: - cd test_haystack - ./run_tests.py whoosh_tests test_loading.py + python test_haystack/run_tests.py whoosh_tests test_loading.py -The ``run_tests.py`` script is just a tiny wrapper around the nose_ library and -any options you pass to it will be passed on; including ``--help`` to get a -list of possible options:: +The ``run_tests.py`` script is just a tiny wrapper around the Django test +command and any options you pass to it will be passed on; including ``--help`` +to get a list of possible options:: - cd test_haystack - ./run_tests.py --help - -.. _nose: https://nose.readthedocs.io/en/latest/ + python test_haystack/run_tests.py --help Configuring Solr ================ diff --git a/setup.py b/setup.py index 3224ed2a1..d58b52ddd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,6 @@ "whoosh>=2.5.4,<3.0", "python-dateutil", "geopy==2.0.0", - "nose", "coverage", "requests", ] diff --git a/test_haystack/__init__.py b/test_haystack/__init__.py index 8e2707352..e69de29bb 100644 --- a/test_haystack/__init__.py +++ b/test_haystack/__init__.py @@ -1,27 +0,0 @@ -import os - -test_runner = None -old_config = None - -os.environ["DJANGO_SETTINGS_MODULE"] = "test_haystack.settings" - - -import django - -django.setup() - - -def setup(): - global test_runner - global old_config - - from django.test.runner import DiscoverRunner - - test_runner = DiscoverRunner() - test_runner.setup_test_environment() - old_config = test_runner.setup_databases() - - -def teardown(): - test_runner.teardown_databases(old_config) - test_runner.teardown_test_environment() diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py index 67a9e9764..38fa24fbc 100644 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ b/test_haystack/elasticsearch2_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -29,3 +27,9 @@ def setup(): except exceptions.ConnectionError as e: log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch5_tests/__init__.py b/test_haystack/elasticsearch5_tests/__init__.py index 09f1ab176..5594ce332 100644 --- a/test_haystack/elasticsearch5_tests/__init__.py +++ b/test_haystack/elasticsearch5_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -29,3 +27,9 @@ def setup(): except exceptions.ConnectionError as e: log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch7_tests/__init__.py b/test_haystack/elasticsearch7_tests/__init__.py index 6491d464a..24339ac89 100644 --- a/test_haystack/elasticsearch7_tests/__init__.py +++ b/test_haystack/elasticsearch7_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -29,3 +27,9 @@ def setup(): except exceptions.ConnectionError as e: log.error("elasticsearch not running on %r" % url, exc_info=True) raise unittest.SkipTest("elasticsearch not running on %r" % url, e) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/elasticsearch_tests/__init__.py b/test_haystack/elasticsearch_tests/__init__.py index 05c53d640..0ceb159dc 100644 --- a/test_haystack/elasticsearch_tests/__init__.py +++ b/test_haystack/elasticsearch_tests/__init__.py @@ -1,14 +1,12 @@ +import os import unittest -import warnings from django.conf import settings from haystack.utils import log as logging -warnings.simplefilter("ignore", Warning) - -def setup(): +def load_tests(loader, standard_tests, pattern): log = logging.getLogger("haystack") try: import elasticsearch @@ -36,3 +34,9 @@ def setup(): % settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], e, ) + + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/multipleindex/__init__.py b/test_haystack/multipleindex/__init__.py index d48e717da..0cd29ea56 100644 --- a/test_haystack/multipleindex/__init__.py +++ b/test_haystack/multipleindex/__init__.py @@ -1,24 +1,12 @@ -from django.apps import apps - -import haystack -from haystack.signals import RealtimeSignalProcessor +import os from ..utils import check_solr -_old_sp = None - -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() - global _old_sp - config = apps.get_app_config("haystack") - _old_sp = config.signal_processor - config.signal_processor = RealtimeSignalProcessor( - haystack.connections, haystack.connection_router + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern ) - - -def teardown(): - config = apps.get_app_config("haystack") - config.signal_processor.teardown() - config.signal_processor = _old_sp + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/multipleindex/tests.py b/test_haystack/multipleindex/tests.py index 5161a1f13..d4eda9b82 100644 --- a/test_haystack/multipleindex/tests.py +++ b/test_haystack/multipleindex/tests.py @@ -1,9 +1,10 @@ +from django.apps import apps from django.db import models -from haystack import connections +from haystack import connection_router, connections from haystack.exceptions import NotHandled from haystack.query import SearchQuerySet -from haystack.signals import BaseSignalProcessor +from haystack.signals import BaseSignalProcessor, RealtimeSignalProcessor from ..whoosh_tests.testcases import WhooshTestCase from .models import Bar, Foo @@ -191,6 +192,22 @@ def teardown(self): class SignalProcessorTestCase(WhooshTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + config = apps.get_app_config("haystack") + cls._old_sp = config.signal_processor + config.signal_processor = RealtimeSignalProcessor( + connections, connection_router + ) + + @classmethod + def tearDown(cls): + config = apps.get_app_config("haystack") + config.signal_processor.teardown() + config.signal_processor = cls._old_sp + super().tearDown() + def setUp(self): super().setUp() diff --git a/test_haystack/run_tests.py b/test_haystack/run_tests.py index 22f167637..85fa00a96 100755 --- a/test_haystack/run_tests.py +++ b/test_haystack/run_tests.py @@ -1,24 +1,17 @@ #!/usr/bin/env python +import os import sys -from os.path import abspath, dirname -import nose +import django +from django.core.management import call_command def run_all(argv=None): - sys.exitfunc = lambda: sys.stderr.write("Shutting down....\n") + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + os.environ["DJANGO_SETTINGS_MODULE"] = "test_haystack.settings" + django.setup() - # always insert coverage when running tests through setup.py - if argv is None: - argv = [ - "nosetests", - "--with-coverage", - "--cover-package=haystack", - "--cover-erase", - "--verbose", - ] - - nose.run_exit(argv=argv, defaultTest=abspath(dirname(__file__))) + call_command("test", sys.argv[1:]) if __name__ == "__main__": diff --git a/test_haystack/solr_tests/__init__.py b/test_haystack/solr_tests/__init__.py index 1b1d43036..0cd29ea56 100644 --- a/test_haystack/solr_tests/__init__.py +++ b/test_haystack/solr_tests/__init__.py @@ -1,9 +1,12 @@ -import warnings - -warnings.simplefilter("ignore", Warning) +import os from ..utils import check_solr -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/test_haystack/spatial/__init__.py b/test_haystack/spatial/__init__.py index 02a7dd78a..0cd29ea56 100644 --- a/test_haystack/spatial/__init__.py +++ b/test_haystack/spatial/__init__.py @@ -1,5 +1,12 @@ +import os + from ..utils import check_solr -def setup(): +def load_tests(loader, standard_tests, pattern): check_solr() + package_tests = loader.discover( + start_dir=os.path.dirname(__file__), pattern=pattern + ) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/tox.ini b/tox.ini index 1b79eb3dd..e2b2e711b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,19 @@ [tox] envlist = docs - py35-django2.2-es{1.x,2.x,5.x,7.x} py{36,37,38,39,310,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} [testenv] commands = python test_haystack/solr_tests/server/wait-for-solr - python {toxinidir}/setup.py test + coverage run {toxinidir}/test_haystack/run_tests.py deps = + pysolr>=3.7.0 + whoosh>=2.5.4,<3.0 + python-dateutil + geopy==2.0.0 + coverage requests django2.2: Django>=2.2,<3.0 django3.0: Django>=3.0,<3.1 From 6e7f0682c7c6381bbbd1e07d8f070106d5fbc67e Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 3 Feb 2022 08:05:42 +0100 Subject: [PATCH 035/162] Remove support for Python 3.5 --- .github/workflows/test.yml | 12 ------------ README.rst | 2 +- setup.py | 1 - tox.ini | 1 - 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcce393d1..a7d67d3d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,18 +12,6 @@ jobs: python-version: [3.6, 3.7, 3.8, 3.9] elastic-version: [1.7, 2.4, 5.5, '7.13.1'] include: - - django-version: 2.2 - python-version: 3.5 - elastic-version: 1.7 - - django-version: 2.2 - python-version: 3.5 - elastic-version: 2.4 - - django-version: 2.2 - python-version: 3.5 - elastic-version: 5.5 - - django-version: 2.2 - python-version: 3.5 - elastic-version: '7.13.1' - django-version: '4.0' python-version: 3.8 elastic-version: 5.5 diff --git a/README.rst b/README.rst index ae447fa08..22afa29b1 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,7 @@ Requirements Haystack has a relatively easily-met set of requirements. -* Python 3.5+ +* Python 3.6+ * A supported version of Django: https://www.djangoproject.com/download/#supported-versions Additionally, each backend has its own requirements. You should refer to diff --git a/setup.py b/setup.py index 3224ed2a1..a096f4019 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,6 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/tox.ini b/tox.ini index 1b79eb3dd..d4ec71035 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] envlist = docs - py35-django2.2-es{1.x,2.x,5.x,7.x} py{36,37,38,39,310,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} From a6b65adc9c7cde2b8a10300aa86e51421ffc6dc7 Mon Sep 17 00:00:00 2001 From: Andrii Oriekhov Date: Thu, 3 Mar 2022 18:36:24 +0200 Subject: [PATCH 036/162] add GitHub URL for PyPi --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 3224ed2a1..bf7a2d4b6 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,10 @@ author_email="daniel@toastdriven.com", long_description=open("README.rst", "r").read(), url="http://haystacksearch.org/", + project_urls={ + "Documentation": "https://django-haystack.readthedocs.io", + "Source": "https://github.com/django-haystack/django-haystack", + }, packages=[ "haystack", "haystack.backends", From 24f49a7ab75e48dc32ba5a380d4a69bb5133f9ca Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Sun, 20 Mar 2022 12:17:15 +0100 Subject: [PATCH 037/162] default_app_config compatibility. --- haystack/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/haystack/__init__.py b/haystack/__init__.py index 6282f322f..22bf9bba7 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -1,3 +1,4 @@ +import django from django.conf import settings from django.core.exceptions import ImproperlyConfigured from pkg_resources import DistributionNotFound, get_distribution, parse_version @@ -15,7 +16,10 @@ __version__ = "0.0.dev0" version_info = parse_version(__version__) -default_app_config = "haystack.apps.HaystackConfig" + +if django.VERSION < (3, 2): + # default_app_config is deprecated since django 3.2. + default_app_config = "haystack.apps.HaystackConfig" # Help people clean up from 1.X. From d4f24f40e91534a803402be596aa2faa0749d35a Mon Sep 17 00:00:00 2001 From: deadly-panda Date: Sun, 20 Mar 2022 12:17:27 +0100 Subject: [PATCH 038/162] default_app_config compatibility unittest. --- test_haystack/test_django_config_detection.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test_haystack/test_django_config_detection.py diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py new file mode 100644 index 000000000..3b99e4014 --- /dev/null +++ b/test_haystack/test_django_config_detection.py @@ -0,0 +1,19 @@ +"""""" +from django.test import TestCase +import unittest +import django + +import haystack + + +class AppConfigCompatibilityTestCase(TestCase): + @unittest.skipIf(django.VERSION >= (3, 2), "default_app_config is deprecated since django 3.2.") + def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): + has_default_appconfig_attr = hasattr(haystack, "default_app_config") + self.assertTrue(has_default_appconfig_attr) + + @unittest.skipIf(django.VERSION < (3, 2), "default_app_config should be used in versions prior to django 3.2.") + def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): + has_default_appconfig_attr = hasattr(haystack, "default_app_config") + self.assertFalse(has_default_appconfig_attr) + From d6f9444d4172928035e3aa02e8b4698498f7ff34 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 20 Mar 2022 11:23:33 +0000 Subject: [PATCH 039/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/__init__.py | 2 +- test_haystack/test_django_config_detection.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/haystack/__init__.py b/haystack/__init__.py index 22bf9bba7..94b8f4674 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -18,7 +18,7 @@ if django.VERSION < (3, 2): - # default_app_config is deprecated since django 3.2. + # default_app_config is deprecated since django 3.2. default_app_config = "haystack.apps.HaystackConfig" diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py index 3b99e4014..31241a48f 100644 --- a/test_haystack/test_django_config_detection.py +++ b/test_haystack/test_django_config_detection.py @@ -1,19 +1,24 @@ """""" -from django.test import TestCase import unittest + import django +from django.test import TestCase import haystack class AppConfigCompatibilityTestCase(TestCase): - @unittest.skipIf(django.VERSION >= (3, 2), "default_app_config is deprecated since django 3.2.") + @unittest.skipIf( + django.VERSION >= (3, 2), "default_app_config is deprecated since django 3.2." + ) def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): - has_default_appconfig_attr = hasattr(haystack, "default_app_config") + has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertTrue(has_default_appconfig_attr) - @unittest.skipIf(django.VERSION < (3, 2), "default_app_config should be used in versions prior to django 3.2.") + @unittest.skipIf( + django.VERSION < (3, 2), + "default_app_config should be used in versions prior to django 3.2.", + ) def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): - has_default_appconfig_attr = hasattr(haystack, "default_app_config") + has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertFalse(has_default_appconfig_attr) - From 69c99efc9528292d3d7f64e5fadf73d150224bb1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Apr 2022 20:44:31 +0000 Subject: [PATCH 040/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 22.1.0 → 22.3.0](https://github.com/psf/black/compare/22.1.0...22.3.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c4116a6d..d83e41609 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 22.1.0 + rev: 22.3.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 6517eaad2a04000161c09a3c6568b89f7db30b21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 21:50:39 +0000 Subject: [PATCH 041/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.1.0 → v4.2.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.1.0...v4.2.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d83e41609..026e9ec74 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] From 14e824c862ab6c9db7bc3c3acf343cd0ab08953c Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Tue, 3 May 2022 14:10:57 -0400 Subject: [PATCH 042/162] Generic views: handle undefined date facet fields values This resolves a regression introduced in #1690. --- haystack/generic_views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/haystack/generic_views.py b/haystack/generic_views.py index d24db369c..2b981a4d1 100644 --- a/haystack/generic_views.py +++ b/haystack/generic_views.py @@ -109,8 +109,9 @@ def get_queryset(self): for field in self.facet_fields: qs = qs.facet(field) - for field in self.date_facet_fields: - qs = qs.date_facet(**field) + if self.date_facet_fields: + for field in self.date_facet_fields: + qs = qs.date_facet(**field) return qs From 7ce2330bbb8bd96d0021eb3ebd015a0a84bacd21 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Tue, 19 Jul 2022 22:30:06 +1000 Subject: [PATCH 043/162] docs: Fix a few typos There are small typos in: - docs/searchindex_api.rst - haystack/backends/elasticsearch_backend.py - haystack/backends/solr_backend.py - haystack/backends/whoosh_backend.py Fixes: - Should read `incorporate` rather than `incorportate`. - Should read `assumes` rather than `asssumes`. - Should read `analogous` rather than `analagous`. Signed-off-by: Tim Gates --- docs/searchindex_api.rst | 2 +- haystack/backends/elasticsearch_backend.py | 4 ++-- haystack/backends/solr_backend.py | 2 +- haystack/backends/whoosh_backend.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/searchindex_api.rst b/docs/searchindex_api.rst index 3f32c1b24..a537e1cda 100644 --- a/docs/searchindex_api.rst +++ b/docs/searchindex_api.rst @@ -352,7 +352,7 @@ non-existent), merely an example of how to extend existing fields. .. note:: - This method is analagous to Django's ``Field.clean`` methods. + This method is analogous to Django's ``Field.clean`` methods. Adding New Fields diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index c2fb47f5f..95ae40971 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -305,7 +305,7 @@ def clear(self, models=None, commit=True): for model in models: models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - # Delete by query in Elasticsearch asssumes you're dealing with + # Delete by query in Elasticsearch assumes you're dealing with # a ``query`` root object. :/ query = { "query": {"query_string": {"query": " OR ".join(models_to_delete)}} @@ -971,7 +971,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] if isinstance(prepared_value, str): diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index dc929bf33..267f04ca2 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -819,7 +819,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] for possible_value in prepared_value.split(" "): diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 5c06e8750..e636148cf 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -1019,7 +1019,7 @@ def build_query_fragment(self, field, filter_type, value): if value.input_type_name == "exact": query_frag = prepared_value else: - # Iterate over terms & incorportate the converted form of each into the query. + # Iterate over terms & incorporate the converted form of each into the query. terms = [] if isinstance(prepared_value, str): From 5dd25e9d466a0d0c5d73cdb92e8c54e5e8005cf6 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 18 Sep 2022 08:28:08 +0200 Subject: [PATCH 044/162] Fix typo --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index c1ca45c26..7d8f0934f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -115,7 +115,7 @@ If you've been granted the commit bit, here's how to shepherd the changes in: * ``git merge --squash`` is a good tool for performing this, as is ``git rebase -i HEAD~N``. - * This is done to prevent anyone using the git repo from accidently pulling + * This is done to prevent anyone using the git repo from accidentally pulling work-in-progress commits. * Commit messages should use past tense, describe what changed & thank anyone From 8ca77121e582bca8a7c7b34bd4cca3686d9dd0c5 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 21 Nov 2022 16:51:21 +0100 Subject: [PATCH 045/162] Fix flake8: logging.error(exc_info=True) --> logging.exception() --- haystack/backends/elasticsearch2_backend.py | 20 +++---- haystack/backends/elasticsearch5_backend.py | 20 +++---- haystack/backends/elasticsearch7_backend.py | 20 +++---- haystack/backends/elasticsearch_backend.py | 57 ++++++++------------ haystack/backends/solr_backend.py | 52 +++++++----------- haystack/backends/whoosh_backend.py | 25 ++++----- haystack/management/commands/update_index.py | 2 +- haystack/templatetags/more_like_this.py | 8 +-- 8 files changed, 77 insertions(+), 127 deletions(-) diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py index 97c8cca15..ce744107f 100644 --- a/haystack/backends/elasticsearch2_backend.py +++ b/haystack/backends/elasticsearch2_backend.py @@ -79,21 +79,17 @@ def clear(self, models=None, commit=True): ) self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -321,15 +317,13 @@ def more_like_this( **self._get_doc_type_option(), **params, ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} diff --git a/haystack/backends/elasticsearch5_backend.py b/haystack/backends/elasticsearch5_backend.py index 2eedc1ad3..3afe11347 100644 --- a/haystack/backends/elasticsearch5_backend.py +++ b/haystack/backends/elasticsearch5_backend.py @@ -75,21 +75,17 @@ def clear(self, models=None, commit=True): ) self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -411,15 +407,13 @@ def more_like_this( **self._get_doc_type_option(), **params, ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} diff --git a/haystack/backends/elasticsearch7_backend.py b/haystack/backends/elasticsearch7_backend.py index dd9c9933d..161a9038a 100644 --- a/haystack/backends/elasticsearch7_backend.py +++ b/haystack/backends/elasticsearch7_backend.py @@ -143,21 +143,17 @@ def clear(self, models=None, commit=True): ) self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -479,15 +475,13 @@ def more_like_this( raw_results = self.conn.search( body=mlt_query, index=self.index_name, _source=True, **params ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index 95ae40971..6c708f4f3 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -199,13 +199,11 @@ def update(self, index, iterable, commit=True): if not self.setup_complete: try: self.setup() - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to add documents to Elasticsearch: %s", e, exc_info=True - ) + self.log.exception("Failed to add documents to Elasticsearch") return prepped_docs = [] @@ -223,16 +221,15 @@ def update(self, index, iterable, commit=True): prepped_docs.append(final_data) except SkipDocument: self.log.debug("Indexing for object `%s` skipped", obj) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( - "%s while preparing object for update" % e.__class__.__name__, - exc_info=True, + self.log.exception( + "Preparing object for update", extra={"data": {"index": index, "object": get_identifier(obj)}}, ) @@ -252,15 +249,13 @@ def remove(self, obj_or_string, commit=True): if not self.setup_complete: try: self.setup() - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Elasticsearch: %s", + self.log.exception( + "Failed to remove document '%s' from Elasticsearch", doc_id, - e, - exc_info=True, ) return @@ -274,15 +269,13 @@ def remove(self, obj_or_string, commit=True): if commit: self.conn.indices.refresh(index=self.index_name) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Elasticsearch: %s", + self.log.exception( + "Failed to remove document '%s' from Elasticsearch", doc_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -315,21 +308,17 @@ def clear(self, models=None, commit=True): body=query, **self._get_doc_type_option(), ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Elasticsearch index of models '%s': %s", + self.log.exception( + "Failed to clear Elasticsearch index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error( - "Failed to clear Elasticsearch index: %s", e, exc_info=True - ) + self.log.exception("Failed to clear Elasticsearch index") def build_search_kwargs( self, @@ -588,15 +577,13 @@ def search(self, query_string, **kwargs): _source=True, **self._get_doc_type_option(), ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to query Elasticsearch using '%s': %s", + self.log.exception( + "Failed to query Elasticsearch using '%s'", query_string, - e, - exc_info=True, ) raw_results = {} @@ -652,15 +639,13 @@ def more_like_this( **self._get_doc_type_option(), **params, ) - except elasticsearch.TransportError as e: + except elasticsearch.TransportError: if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Elasticsearch for document '%s': %s", + self.log.exception( + "Failed to fetch More Like This from Elasticsearch for document '%s'", doc_id, - e, - exc_info=True, ) raw_results = {} diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 267f04ca2..12bfd2f3b 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -91,20 +91,19 @@ def update(self, index, iterable, commit=True): # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( + self.log.exception( "UnicodeDecodeError while preparing object for update", - exc_info=True, extra={"data": {"index": index, "object": get_identifier(obj)}}, ) if len(docs) > 0: try: self.conn.add(docs, commit=commit, boost=index.get_field_weights()) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error("Failed to add documents to Solr: %s", e, exc_info=True) + self.log.exception("Failed to add documents to Solr") def remove(self, obj_or_string, commit=True): solr_id = get_identifier(obj_or_string) @@ -112,15 +111,13 @@ def remove(self, obj_or_string, commit=True): try: kwargs = {"commit": commit, "id": solr_id} self.conn.delete(**kwargs) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Solr: %s", + self.log.exception( + "Failed to remove document '%s' from Solr", solr_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -142,19 +139,17 @@ def clear(self, models=None, commit=True): if commit: # Run an optimize post-clear. http://wiki.apache.org/solr/FAQ#head-9aafb5d8dff5308e8ea4fcf4b71f19f029c4bb99 self.conn.optimize() - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Solr index of models '%s': %s", + self.log.exception( + "Failed to clear Solr index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error("Failed to clear Solr index: %s", e, exc_info=True) + self.log.exception("Failed to clear Solr index") @log_query def search(self, query_string, **kwargs): @@ -165,13 +160,11 @@ def search(self, query_string, **kwargs): try: raw_results = self.conn.search(query_string, **search_kwargs) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to query Solr using '%s': %s", query_string, e, exc_info=True - ) + self.log.exception("Failed to query Solr using '%s'", query_string) raw_results = EmptyResults() return self._process_results( @@ -450,15 +443,12 @@ def more_like_this( try: raw_results = self.conn.more_like_this(query, field_name, **params) - except (IOError, SolrError) as e: + except (IOError, SolrError): if not self.silently_fail: raise - self.log.error( - "Failed to fetch More Like This from Solr for document '%s': %s", - query, - e, - exc_info=True, + self.log.exception( + "Failed to fetch More Like This from Solr for document '%s'", query ) raw_results = EmptyResults() @@ -514,11 +504,9 @@ def _process_results( if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) - except Exception as exc: - self.log.error( + except Exception: + self.log.exception( "Error extracting spelling suggestions: %s", - exc, - exc_info=True, extra={"data": {"spellcheck": raw_results.spellcheck}}, ) @@ -747,11 +735,9 @@ def extract_file_contents(self, file_obj, **kwargs): try: return self.conn.extract(file_obj, **kwargs) - except Exception as e: + except Exception: self.log.warning( - "Unable to extract file contents: %s", - e, - exc_info=True, + "Unable to extract file contents", extra={"data": {"file": file_obj}}, ) return None diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index e636148cf..26bac5d87 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -270,16 +270,15 @@ def update(self, index, iterable, commit=True): try: writer.update_document(**doc) - except Exception as e: + except Exception: if not self.silently_fail: raise # We'll log the object identifier but won't include the actual object # to avoid the possibility of that generating encoding errors while # processing the log message: - self.log.error( - "%s while preparing object for update" % e.__class__.__name__, - exc_info=True, + self.log.exception( + "Preparing object for update", extra={"data": {"index": index, "object": get_identifier(obj)}}, ) @@ -298,15 +297,13 @@ def remove(self, obj_or_string, commit=True): try: self.index.delete_by_query(q=self.parser.parse('%s:"%s"' % (ID, whoosh_id))) - except Exception as e: + except Exception: if not self.silently_fail: raise - self.log.error( - "Failed to remove document '%s' from Whoosh: %s", + self.log.exception( + "Failed to remove document '%s' from Whoosh", whoosh_id, - e, - exc_info=True, ) def clear(self, models=None, commit=True): @@ -330,19 +327,17 @@ def clear(self, models=None, commit=True): self.index.delete_by_query( q=self.parser.parse(" OR ".join(models_to_delete)) ) - except Exception as e: + except Exception: if not self.silently_fail: raise if models is not None: - self.log.error( - "Failed to clear Whoosh index of models '%s': %s", + self.log.exception( + "Failed to clear Whoosh index of models '%s'", ",".join(models_to_delete), - e, - exc_info=True, ) else: - self.log.error("Failed to clear Whoosh index: %s", e, exc_info=True) + self.log.exception("Failed to clear Whoosh index") def delete_index(self): # Per the Whoosh mailing list, if wiping out everything from the index, diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py index da50644bc..6d813d6c0 100644 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -144,7 +144,7 @@ def do_update( error_msg += " (pid %(pid)s): %(exc)s" if retries >= max_retries: - LOG.error(error_msg, error_context, exc_info=True) + LOG.exception(error_msg, error_context) raise elif verbosity >= 2: LOG.warning(error_msg, error_context, exc_info=True) diff --git a/haystack/templatetags/more_like_this.py b/haystack/templatetags/more_like_this.py index 2cc22751d..8ec26098a 100644 --- a/haystack/templatetags/more_like_this.py +++ b/haystack/templatetags/more_like_this.py @@ -42,9 +42,11 @@ def render(self, context): sqs = sqs[: self.limit] context[self.varname] = sqs - except Exception as exc: - logging.warning( - "Unhandled exception rendering %r: %s", self, exc, exc_info=True + except Exception: + logging.exception( + "Unhandled exception rendering %r", + self, + level=logging.WARNING, ) return "" From cc8f7d1498b38b4cf3d165dc4b50f6eb6931ff82 Mon Sep 17 00:00:00 2001 From: HAMZA310 Date: Mon, 30 Jan 2023 18:28:40 +0500 Subject: [PATCH 046/162] =?UTF-8?q?docs:=20use=20=E2=80=98stable=E2=80=99?= =?UTF-8?q?=20tag=20in=20Django=20URLs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In URLs that point to pages in django documentation, use 'stable' tag in order to always point to the latest version of the documentation. Previously, hard-coded django versions (e.g. 1.7) were used which are now outdated (throw 404) e.g. this link: https://docs.djangoproject.com/en/1.7/topics/class-based-views/ is no longer valid used in `views_and_forms.html` page. Using 'stable' tag will ensure those links never become outdated. --- docs/changelog.rst | 2 +- docs/python3.rst | 2 +- docs/running_tests.rst | 2 +- docs/spatial.rst | 2 +- docs/views_and_forms.rst | 12 ++++++------ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 00a749710..132326683 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -900,7 +900,7 @@ Other Add python 3.5 to tests - Add python 3.5 to tests. [Marco Badan] - ref: https://docs.djangoproject.com/en/1.9/faq/install/#what-python-version-can-i-use-with-django + ref: https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - SearchQuerySet: don’t trigger backend access in __repr__ [Chris Adams] This can lead to confusing errors or performance issues by diff --git a/docs/python3.rst b/docs/python3.rst index 310ced294..ec5e8874e 100644 --- a/docs/python3.rst +++ b/docs/python3.rst @@ -15,7 +15,7 @@ Virtually all tests pass under both Python 2 & 3, with a small number of expected failures under Python (typically related to ordering, see below). .. _`six`: http://pythonhosted.org/six/ -.. _`Django`: https://docs.djangoproject.com/en/1.5/topics/python3/#str-and-unicode-methods +.. _`Django`: https://docs.djangoproject.com/en/stable/topics/python3/#str-and-unicode-methods Supported Backends diff --git a/docs/running_tests.rst b/docs/running_tests.rst index 76d4daea8..9123ed1ea 100644 --- a/docs/running_tests.rst +++ b/docs/running_tests.rst @@ -67,4 +67,4 @@ If you want to run the geo-django tests you may need to review the cd test_haystack ./run_tests.py elasticsearch_tests -.. _GeoDjango GEOS and GDAL settings: https://docs.djangoproject.com/en/1.7/ref/contrib/gis/install/geolibs/#geos-library-path +.. _GeoDjango GEOS and GDAL settings: https://docs.djangoproject.com/en/stable/ref/contrib/gis/install/geolibs/#geos-library-path diff --git a/docs/spatial.rst b/docs/spatial.rst index 34227fa85..76bd5021f 100644 --- a/docs/spatial.rst +++ b/docs/spatial.rst @@ -14,7 +14,7 @@ close to GeoDjango_ as possible. There are some differences, which we'll highlight throughout this guide. Additionally, while the support isn't as comprehensive as PostGIS (for example), it is still quite useful. -.. _GeoDjango: https://docs.djangoproject.com/en/1.11/ref/contrib/gis/ +.. _GeoDjango: https://docs.djangoproject.com/en/stable/ref/contrib/gis/ Additional Requirements diff --git a/docs/views_and_forms.rst b/docs/views_and_forms.rst index 0edeeeb54..7f518e79b 100644 --- a/docs/views_and_forms.rst +++ b/docs/views_and_forms.rst @@ -11,7 +11,7 @@ Views & Forms which use the standard Django `class-based views`_ which are available in every version of Django which is supported by Haystack. -.. _class-based views: https://docs.djangoproject.com/en/1.7/topics/class-based-views/ +.. _class-based views: https://docs.djangoproject.com/en/stable/topics/class-based-views/ Haystack comes with some default, simple views & forms as well as some django-style views to help you get started and to cover the common cases. @@ -137,7 +137,7 @@ Views which use the standard Django `class-based views`_ which are available in every version of Django which is supported by Haystack. -.. _class-based views: https://docs.djangoproject.com/en/1.7/topics/class-based-views/ +.. _class-based views: https://docs.djangoproject.com/en/stable/topics/class-based-views/ New Django Class Based Views ---------------------------- @@ -145,7 +145,7 @@ New Django Class Based Views .. versionadded:: 2.4.0 The views in ``haystack.generic_views.SearchView`` inherit from Django’s standard -`FormView `_. +`FormView `_. The example views can be customized like any other Django class-based view as demonstrated in this example which filters the search results in ``get_queryset``:: @@ -232,9 +232,9 @@ preprocess the values returned by Haystack, that code would move to ``get_contex | ``get_query()`` | `get_queryset()`_ | +-----------------------+-------------------------------------------+ -.. _get_context_data(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/mixins-simple/#django.views.generic.base.ContextMixin.get_context_data -.. _dispatch(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/base/#django.views.generic.base.View.dispatch -.. _get_queryset(): https://docs.djangoproject.com/en/1.7/ref/class-based-views/mixins-multiple-object/#django.views.generic.list.MultipleObjectMixin.get_queryset +.. _get_context_data(): https://docs.djangoproject.com/en/stable/ref/class-based-views/mixins-simple/#django.views.generic.base.ContextMixin.get_context_data +.. _dispatch(): https://docs.djangoproject.com/en/stable/ref/class-based-views/base/#django.views.generic.base.View.dispatch +.. _get_queryset(): https://docs.djangoproject.com/en/stable/ref/class-based-views/mixins-multiple-object/#django.views.generic.list.MultipleObjectMixin.get_queryset Old-Style Views From f109fdc4d2afc8c765cfb357b9d7d230afcc1713 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Apr 2023 07:18:26 +0000 Subject: [PATCH 047/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/isort: 5.10.1 → 5.12.0](https://github.com/PyCQA/isort/compare/5.10.1...5.12.0) - [github.com/psf/black: 22.3.0 → 23.3.0](https://github.com/psf/black/compare/22.3.0...23.3.0) - [github.com/pre-commit/pre-commit-hooks: v4.2.0 → v4.4.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.2.0...v4.4.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 026e9ec74..fece4be03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,15 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.3.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.4.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] From 07dd454aa22a52eeb000ce37839d3e902928155e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Apr 2023 07:18:48 +0000 Subject: [PATCH 048/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/backends/solr_backend.py | 1 - haystack/management/commands/update_index.py | 1 - haystack/query.py | 1 - haystack/utils/loading.py | 1 - test_haystack/elasticsearch_tests/test_elasticsearch_backend.py | 1 - test_haystack/simple_tests/test_simple_backend.py | 1 - test_haystack/solr_tests/test_solr_management_commands.py | 2 -- .../test_app_using_appconfig/migrations/0001_initial.py | 1 - 8 files changed, 9 deletions(-) diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 12bfd2f3b..405508523 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -197,7 +197,6 @@ def build_search_kwargs( collate=None, **extra_kwargs ): - index = haystack.connections[self.connection_alias].get_unified_index() kwargs = {"fl": "* score", "df": index.document_field} diff --git a/haystack/management/commands/update_index.py b/haystack/management/commands/update_index.py index 6d813d6c0..070332ff8 100644 --- a/haystack/management/commands/update_index.py +++ b/haystack/management/commands/update_index.py @@ -81,7 +81,6 @@ def do_update( max_retries=DEFAULT_MAX_RETRIES, last_max_pk=None, ): - # Get a clone of the QuerySet so that the cache doesn't bloat up # in memory. Useful when reindexing large amounts of data. # the query must be ordered by PK in order to get the max PK in each batch diff --git a/haystack/query.py b/haystack/query.py index 1be64658f..382e5682f 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -172,7 +172,6 @@ def post_process_results(self, results): for result in results: if self._load_all: - model_objects = loaded_objects.get(result.model, {}) # Try to coerce a primary key object that matches the models pk # We have to deal with semi-arbitrary keys being cast from strings (UUID, int, etc) diff --git a/haystack/utils/loading.py b/haystack/utils/loading.py index 216e485a1..d96af7125 100644 --- a/haystack/utils/loading.py +++ b/haystack/utils/loading.py @@ -338,7 +338,6 @@ def get_index_fieldname(self, field): return self._fieldnames.get(field) or field def get_index(self, model_klass): - indexes = self.get_indexes() if model_klass not in indexes: diff --git a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py index 665b00cea..7de53333c 100644 --- a/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py +++ b/test_haystack/elasticsearch_tests/test_elasticsearch_backend.py @@ -229,7 +229,6 @@ def test_kwargs_are_passed_on(self): class ElasticSearchMockUnifiedIndex(UnifiedIndex): - spy_args = None def get_index(self, model_klass): diff --git a/test_haystack/simple_tests/test_simple_backend.py b/test_haystack/simple_tests/test_simple_backend.py index e19662217..3f3df65e8 100644 --- a/test_haystack/simple_tests/test_simple_backend.py +++ b/test_haystack/simple_tests/test_simple_backend.py @@ -206,7 +206,6 @@ def test_more_like_this(self): self.assertEqual(self.backend.more_like_this(self.sample_objs[0])["hits"], 0) def test_score_field_collision(self): - index = connections["simple"].get_unified_index().get_index(ScoreMockModel) sample_objs = ScoreMockModel.objects.all() diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 6c6a537e0..32a3d6608 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -202,7 +202,6 @@ def test_multiprocessing(self): self.assertEqual(self.solr.search("*:*").hits, 0) def test_build_schema_wrong_backend(self): - settings.HAYSTACK_CONNECTIONS["whoosh"] = { "ENGINE": "haystack.backends.whoosh_backend.WhooshEngine", "PATH": mkdtemp(prefix="dummy-path-"), @@ -214,7 +213,6 @@ def test_build_schema_wrong_backend(self): ) def test_build_schema(self): - # Stow. oldhdf = constants.DOCUMENT_FIELD oldui = connections["solr"].get_unified_index() diff --git a/test_haystack/test_app_using_appconfig/migrations/0001_initial.py b/test_haystack/test_app_using_appconfig/migrations/0001_initial.py index 1f9b7051e..309b49009 100644 --- a/test_haystack/test_app_using_appconfig/migrations/0001_initial.py +++ b/test_haystack/test_app_using_appconfig/migrations/0001_initial.py @@ -2,7 +2,6 @@ class Migration(migrations.Migration): - dependencies = [] operations = [ From 137d74dba501648c8f83972d17ec58c8f299a999 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 18 Sep 2022 07:10:31 +0200 Subject: [PATCH 049/162] GitHub Actions: Add Python 3.10 to the testing --- .github/workflows/docs.yml | 4 ++-- .github/workflows/flake8.yml | 6 +++--- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 36 +++++++++++++++++++++++------------ 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 532eaea9a..ec3c04b77 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,9 +6,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install dependencies diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 6889aa5a4..350d2bb04 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -6,12 +6,12 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.9 - name: Install tools run: pip install flake8 flake8-assertive flake8-bugbear flake8-builtins flake8-comprehensions flake8-logging-format - name: Run flake8 - run: flake8 example_project haystack + run: flake8 --ignore=B028 example_project haystack diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 13ae34cee..247e4d09c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,9 +9,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 755527e3e..735eb66ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,25 +4,37 @@ on: [pull_request, push] jobs: test: - - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: - matrix: + fail-fast: false + matrix: # https://docs.djangoproject.com/en/4.1/faq/install django-version: [2.2, 3.1, 3.2] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7] elastic-version: [1.7, 2.4, 5.5, '7.13.1'] include: - django-version: '4.0' - python-version: 3.8 - elastic-version: 5.5 + python-version: '3.8' + elastic-version: '5.5' - django-version: '4.0' - python-version: 3.8 + python-version: '3.8' elastic-version: '7.13.1' - django-version: '4.0' - python-version: 3.9 - elastic-version: 5.5 + python-version: '3.9' + elastic-version: '5.5' + - django-version: '4.0' + python-version: '3.9' + elastic-version: '7.13.1' - django-version: '4.0' - python-version: 3.9 + python-version: '3.10' + elastic-version: '5.5' + - django-version: '4.0' + python-version: '3.10' + elastic-version: '7.13.1' + - django-version: '4.1' + python-version: '3.11' + elastic-version: '5.5' + - django-version: '4.1' + python-version: '3.11' elastic-version: '7.13.1' services: elastic: @@ -41,9 +53,9 @@ jobs: ports: - 9001:9001 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install system dependencies From 6040caea36a36081270506315a871abe978e0cd8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 12 Apr 2023 17:08:09 +0200 Subject: [PATCH 050/162] Drop Python 3.6, etc. --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/flake8.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 40 ++++++++------------------- 5 files changed, 16 insertions(+), 32 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b8a15d08b..7e7e0054d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ec3c04b77..52630ede0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.x - name: Install dependencies run: pip install sphinx - name: Build docs diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 350d2bb04..e05cc9e37 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: 3.x - name: Install tools run: pip install flake8 flake8-assertive flake8-bugbear flake8-builtins flake8-comprehensions flake8-logging-format - name: Run flake8 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 247e4d09c..382bdbc4c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.x - name: Install dependencies run: python -m pip install --upgrade pip setuptools twine wheel - name: Build package diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 735eb66ae..6b8b0f439 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,38 +4,22 @@ on: [pull_request, push] jobs: test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/4.1/faq/install - django-version: [2.2, 3.1, 3.2] - python-version: [3.6, 3.7] - elastic-version: [1.7, 2.4, 5.5, '7.13.1'] + django-version: ["3.2", "4.0", "4.1"] + python-version: ["3.7", "3.8", "3.9", "3.10"] + elastic-version: ["1.7", "2.4", "5.5", "7.13.1"] + exclude: + - django-version: "4.0" + python-version: "3.7" + - django-version: "4.1" + python-version: "3.7" include: - - django-version: '4.0' - python-version: '3.8' - elastic-version: '5.5' - - django-version: '4.0' - python-version: '3.8' - elastic-version: '7.13.1' - - django-version: '4.0' - python-version: '3.9' - elastic-version: '5.5' - - django-version: '4.0' - python-version: '3.9' - elastic-version: '7.13.1' - - django-version: '4.0' - python-version: '3.10' - elastic-version: '5.5' - - django-version: '4.0' - python-version: '3.10' - elastic-version: '7.13.1' - - django-version: '4.1' - python-version: '3.11' - elastic-version: '5.5' - - django-version: '4.1' - python-version: '3.11' - elastic-version: '7.13.1' + - django-version: "4.1" + python-version: "3.11" + elastic-version: "7.13.1" services: elastic: image: elasticsearch:${{ matrix.elastic-version }} From 15fa32763ee9e6f8a36a6c35c3351051f61b493f Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 1 Apr 2023 07:45:11 +0200 Subject: [PATCH 051/162] catch makedirs problem in early state --- haystack/backends/whoosh_backend.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 26bac5d87..f5c701b9b 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -130,7 +130,13 @@ def setup(self): # Make sure the index is there. if self.use_file_storage and not os.path.exists(self.path): - os.makedirs(self.path) + try: + os.makedirs(self.path) + except: + raise IOError( + "The directory of your Whoosh index '%s' (cwd='%s') cannot be created for the current user/group." + % (self.path, os.getcwd()) + ) new_index = True if self.use_file_storage and not os.access(self.path, os.W_OK): From 8585a178fa57e6c07a4cdc01ebf70a3684409fcf Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 16 Apr 2023 16:28:07 +0200 Subject: [PATCH 052/162] Upgrade GitHub Actions flake8 to ruff --- .github/workflows/flake8.yml | 17 ----------------- .github/workflows/ruff.yml | 14 ++++++++++++++ docs/conf.py | 2 -- haystack/backends/simple_backend.py | 2 +- haystack/backends/whoosh_backend.py | 2 +- haystack/templatetags/more_like_this.py | 2 +- pyproject.toml | 20 ++++++++++++++++++++ 7 files changed, 37 insertions(+), 22 deletions(-) delete mode 100644 .github/workflows/flake8.yml create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index e05cc9e37..000000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: flake8 - -on: [pull_request, push] - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install tools - run: pip install flake8 flake8-assertive flake8-bugbear flake8-builtins flake8-comprehensions flake8-logging-format - - name: Run flake8 - run: flake8 --ignore=B028 example_project haystack diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 000000000..ac82ede98 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,14 @@ +# https://beta.ruff.rs +name: ruff +on: + push: + branches: [master] + pull_request: + branches: [master] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: pip install --user ruff + - run: ruff --format=github . diff --git a/docs/conf.py b/docs/conf.py index 3b46fa208..d8239e5a2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,8 +10,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import os -import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/haystack/backends/simple_backend.py b/haystack/backends/simple_backend.py index a3bb59400..a94625281 100644 --- a/haystack/backends/simple_backend.py +++ b/haystack/backends/simple_backend.py @@ -56,7 +56,7 @@ def search(self, query_string, **kwargs): if hasattr(field, "related"): continue - if not field.get_internal_type() in ( + if field.get_internal_type() not in ( "TextField", "CharField", "SlugField", diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index f5c701b9b..5cf7832ec 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -132,7 +132,7 @@ def setup(self): if self.use_file_storage and not os.path.exists(self.path): try: os.makedirs(self.path) - except: + except Exception: raise IOError( "The directory of your Whoosh index '%s' (cwd='%s') cannot be created for the current user/group." % (self.path, os.getcwd()) diff --git a/haystack/templatetags/more_like_this.py b/haystack/templatetags/more_like_this.py index 8ec26098a..3f710e9a0 100644 --- a/haystack/templatetags/more_like_this.py +++ b/haystack/templatetags/more_like_this.py @@ -75,7 +75,7 @@ def more_like_this(parser, token): """ bits = token.split_contents() - if not len(bits) in (4, 6, 8): + if len(bits) not in (4, 6, 8): raise template.TemplateSyntaxError( "'%s' tag requires either 3, 5 or 7 arguments." % bits[0] ) diff --git a/pyproject.toml b/pyproject.toml index 403009f96..c16439b07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,3 +12,23 @@ multi_line_output = 3 [tool.setuptools_scm] fallback_version = "0.0.dev0" write_to = "haystack/version.py" + +[tool.ruff] +exclude = ["test_haystack"] +ignore = ["B018", "B028", "B904", "B905"] +line-length = 162 +select = ["B", "C4", "E", "F", "G", "PLR091", "W"] +show-source = true +target-version = "py37" + +[tool.ruff.isort] +known-first-party = ["haystack", "test_haystack"] + +[tool.ruff.mccabe] +max-complexity = 14 + +[tool.ruff.pylint] +max-args = 20 +max-branches = 39 +max-returns = 8 +max-statements = 91 From c2d754d017bf4e7879f42451cc49a15d6ea68876 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 16 Apr 2023 18:53:13 +0200 Subject: [PATCH 053/162] ci: Do not run the tests if linting fails --- .github/workflows/ruff.yml | 14 -------------- .github/workflows/test.yml | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml deleted file mode 100644 index ac82ede98..000000000 --- a/.github/workflows/ruff.yml +++ /dev/null @@ -1,14 +0,0 @@ -# https://beta.ruff.rs -name: ruff -on: - push: - branches: [master] - pull_request: - branches: [master] -jobs: - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - run: pip install --user ruff - - run: ruff --format=github . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b8b0f439..3199e1a85 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,23 @@ name: Test -on: [pull_request, push] +on: + push: + branches: [master] + pull_request: + branches: [master] jobs: + ruff: # https://beta.ruff.rs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: pip install --user ruff + - run: ruff --format=github . + - run: ruff --format=github --select=ALL . # THIS WILL FAIL!! + test: runs-on: ubuntu-latest + needs: ruff # Do not run the tests if linting fails. strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/4.1/faq/install From 70091e9a343d46c3bd167fc0d42cc3392a5c8bed Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 16 Apr 2023 19:28:30 +0200 Subject: [PATCH 054/162] Fix the broken lint step --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3199e1a85..e5d69b39a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,6 @@ jobs: - uses: actions/checkout@v3 - run: pip install --user ruff - run: ruff --format=github . - - run: ruff --format=github --select=ALL . # THIS WILL FAIL!! test: runs-on: ubuntu-latest From 632208bd006b27ecd8d34018b7bb3a758eeb0b1d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 16 Apr 2023 19:05:28 +0200 Subject: [PATCH 055/162] Add ruff to pre-commit --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fece4be03..dabd8b278 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,11 @@ exclude: ".*/vendor/.*" repos: + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.261 + hooks: + - id: ruff + # args: [ --fix, --exit-non-zero-on-fix ] + - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: @@ -21,6 +27,7 @@ repos: - id: check-json - id: check-merge-conflict - id: check-symlinks + - id: check-toml - id: check-xml - id: check-yaml - id: debug-statements From ffcd9bf8f20ca410e42d950114a99e4c557e726e Mon Sep 17 00:00:00 2001 From: code-review-doctor <72647856+code-review-doctor@users.noreply.github.com> Date: Mon, 21 Feb 2022 23:46:33 +0000 Subject: [PATCH 056/162] Fix issue duplicate-test-names found at https://codereview.doctor --- test_haystack/test_management_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_haystack/test_management_commands.py b/test_haystack/test_management_commands.py index 5d55de3a1..d78203007 100644 --- a/test_haystack/test_management_commands.py +++ b/test_haystack/test_management_commands.py @@ -92,7 +92,7 @@ def test_rebuild_index_nocommit(self, *mocks): @patch("haystack.management.commands.clear_index.Command.handle", return_value="") @patch("haystack.management.commands.update_index.Command.handle", return_value="") - def test_rebuild_index_nocommit(self, update_mock, clear_mock): + def test_rebuild_index_nocommit_two(self, update_mock, clear_mock): """ Confirm that command-line option parsing produces the same results as using call_command() directly, mostly as a sanity check for the logic in rebuild_index which combines the option_lists for its From 70df0ac5bf58b191dfc247a0e70e5f9c1baf44d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 May 2023 05:19:39 +0000 Subject: [PATCH 057/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.261 → v0.0.263](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.261...v0.0.263) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dabd8b278..10e959b25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.261 + rev: v0.0.263 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From b01e63c55dee10cb4ca17a28822188b6f771f8d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 May 2023 06:00:33 +0000 Subject: [PATCH 058/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.263 → v0.0.265](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.263...v0.0.265) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10e959b25..a546d2713 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.263 + rev: v0.0.265 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 924b5ad75b586ce72696d6fc48207e628ecbfcd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 May 2023 07:24:09 +0100 Subject: [PATCH 059/162] [pre-commit.ci] pre-commit autoupdate (#1880) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.265 → v0.0.267](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.265...v0.0.267) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a546d2713..0d75aeb34 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.265 + rev: v0.0.267 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From ce81e09569255cef9c04f522490b9bb8a81b406e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 08:42:39 +0200 Subject: [PATCH 060/162] [pre-commit.ci] pre-commit autoupdate (#1881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.267 → v0.0.270](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.267...v0.0.270) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d75aeb34..4dc591333 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.267 + rev: v0.0.270 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 28ef792522f356d9c9904972a381994adc3425b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Jun 2023 09:10:59 +0200 Subject: [PATCH 061/162] [pre-commit.ci] pre-commit autoupdate (#1882) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.270 → v0.0.272](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.270...v0.0.272) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4dc591333..42d2e666d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.272 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 418815f179b38bbda6f946f394a0c47735f43820 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Jun 2023 08:04:51 +0200 Subject: [PATCH 062/162] [pre-commit.ci] pre-commit autoupdate (#1884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/charliermarsh/ruff-pre-commit: v0.0.272 → v0.0.275](https://github.com/charliermarsh/ruff-pre-commit/compare/v0.0.272...v0.0.275) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42d2e666d..1ebfb646f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.272 + rev: v0.0.275 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 4ef294cd2cdcab71dd30e8f99ad1872670c873b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:55:29 +0200 Subject: [PATCH 063/162] [pre-commit.ci] pre-commit autoupdate (#1885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.275 → v0.0.276](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.275...v0.0.276) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1ebfb646f..fc0a11f17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.275 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.276 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 14392a71f54671c0fc1af9a288562e14cf81986c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 11:00:03 +0200 Subject: [PATCH 064/162] [pre-commit.ci] pre-commit autoupdate (#1886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.276 → v0.0.277](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.276...v0.0.277) - [github.com/psf/black: 23.3.0 → 23.7.0](https://github.com/psf/black/compare/23.3.0...23.7.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc0a11f17..eb00f04ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.276 + rev: v0.0.277 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From b22f066bf839b37c0ac17c5de1baadd74e6f6aef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 19 Jul 2023 11:56:01 +0200 Subject: [PATCH 065/162] [pre-commit.ci] pre-commit autoupdate (#1888) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.277 → v0.0.278](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.277...v0.0.278) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb00f04ce..a31640a76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.277 + rev: v0.0.278 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From b6d0ce96e6db64fc6bb91f2dac03f59f0522cf76 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 08:46:51 +0200 Subject: [PATCH 066/162] [pre-commit.ci] pre-commit autoupdate (#1890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.278 → v0.0.280](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.278...v0.0.280) * max-branches = 40 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a31640a76..762a72c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.278 + rev: v0.0.280 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] diff --git a/pyproject.toml b/pyproject.toml index c16439b07..3a08e1563 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ write_to = "haystack/version.py" exclude = ["test_haystack"] ignore = ["B018", "B028", "B904", "B905"] line-length = 162 -select = ["B", "C4", "E", "F", "G", "PLR091", "W"] +select = ["ASYNC", "B", "C4", "E", "F", "G", "PLR091", "W"] show-source = true target-version = "py37" @@ -29,6 +29,6 @@ max-complexity = 14 [tool.ruff.pylint] max-args = 20 -max-branches = 39 +max-branches = 40 max-returns = 8 max-statements = 91 From f1c7add74e96158c62eb5b16f72f8fa547ffedcc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 12:31:36 +0200 Subject: [PATCH 067/162] [pre-commit.ci] pre-commit autoupdate (#1891) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.280 → v0.0.281](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.280...v0.0.281) * # noqa: F401 --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 2 +- haystack/utils/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 762a72c7a..2d6592e4c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.280 + rev: v0.0.281 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] diff --git a/haystack/utils/__init__.py b/haystack/utils/__init__.py index b0b0d082a..18d939c41 100644 --- a/haystack/utils/__init__.py +++ b/haystack/utils/__init__.py @@ -4,7 +4,7 @@ from django.conf import settings from haystack.constants import DJANGO_CT, DJANGO_ID, ID -from haystack.utils.highlighting import Highlighter # noqa=F401 +from haystack.utils.highlighting import Highlighter # noqa: F401 IDENTIFIER_REGEX = re.compile(r"^[\w\d_]+\.[\w\d_]+\.[\w\d-]+$") From 3a43fc14bdab1cfefb208b74036df27f09730cc1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 07:30:45 -0400 Subject: [PATCH 068/162] [pre-commit.ci] pre-commit autoupdate (#1892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.281 → v0.0.282](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.281...v0.0.282) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d6592e4c..4cba2e6b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.281 + rev: v0.0.282 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From b397f98e515d76fb2a0bca50357ffa585defba42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:32:41 -0400 Subject: [PATCH 069/162] [pre-commit.ci] pre-commit autoupdate (#1893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.282 → v0.0.284](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.282...v0.0.284) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4cba2e6b7..6d0ceb86e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.282 + rev: v0.0.284 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 7fabd6215f9267e9911f0867a2d50537b6a1dbc5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 11:02:14 +0200 Subject: [PATCH 070/162] [pre-commit.ci] pre-commit autoupdate (#1894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.284 → v0.0.285](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.284...v0.0.285) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d0ceb86e..dc6f9b7d6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.284 + rev: v0.0.285 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 1706e821510dbf47d0c08cf868e65fa8ff31348d Mon Sep 17 00:00:00 2001 From: Srivardhan Rathore <69527817+srivardhanrr@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:02:40 +0545 Subject: [PATCH 071/162] Update spatial.rst Fixed the import class for D, from django.contrib.gis.geos to django.contrib.gis.measure. --- docs/spatial.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/spatial.rst b/docs/spatial.rst index 76bd5021f..3f4b8e028 100644 --- a/docs/spatial.rst +++ b/docs/spatial.rst @@ -261,7 +261,8 @@ calculations on your part. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) # Within a two miles. @@ -304,7 +305,8 @@ include these calculated distances on results. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) @@ -322,7 +324,8 @@ key, well-cached hotspots in town but want distances from the user's current position:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) user_loc = Point(-95.23455619812012, 38.97240128290697) @@ -363,7 +366,8 @@ distance information on the results & nothing to sort by. Examples:: from haystack.query import SearchQuerySet - from django.contrib.gis.geos import Point, D + from django.contrib.gis.geos import Point + from django.contrib.gis.measure import D ninth_and_mass = Point(-95.23592948913574, 38.96753407043678) downtown_bottom_left = Point(-95.23947, 38.9637903) From 7bcd980ab997f9eaea40f9f586d2d66e6ccf55b2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:33:38 +0200 Subject: [PATCH 072/162] [pre-commit.ci] pre-commit autoupdate (#1897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.285 → v0.0.286](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.285...v0.0.286) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc6f9b7d6..d62297396 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.285 + rev: v0.0.286 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 27fdcfd06f414380113b2c165108876989778aa3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:49:47 +0200 Subject: [PATCH 073/162] [pre-commit.ci] pre-commit autoupdate (#1899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.286 → v0.0.287](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.286...v0.0.287) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d62297396..0eff92ca7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.286 + rev: v0.0.287 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 92578e334a4a47123a71ae4d73933b637a71f5a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:35:41 +0200 Subject: [PATCH 074/162] [pre-commit.ci] pre-commit autoupdate (#1900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.287 → v0.0.288](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.287...v0.0.288) - [github.com/psf/black: 23.7.0 → 23.9.1](https://github.com/psf/black/compare/23.7.0...23.9.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0eff92ca7..0c9e2deee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.287 + rev: v0.0.288 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 23.9.1 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 87aa1bc5581a0654c04854ab3089587c5e4ea4c7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:51:14 +0200 Subject: [PATCH 075/162] [pre-commit.ci] pre-commit autoupdate (#1901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.288 → v0.0.290](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.288...v0.0.290) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c9e2deee..a82bb0a12 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.288 + rev: v0.0.290 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From fa364df7faae40768687ac6962c8a7a152526b50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 14:13:21 +0200 Subject: [PATCH 076/162] [pre-commit.ci] pre-commit autoupdate (#1904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.290 → v0.0.291](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.290...v0.0.291) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a82bb0a12..34a47f0fa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.290 + rev: v0.0.291 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 660fda863d8ba1dbe8e6e6fb82b11f60d186ffcb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:00:37 +0200 Subject: [PATCH 077/162] [pre-commit.ci] pre-commit autoupdate (#1905) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.291 → v0.0.292](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.291...v0.0.292) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34a47f0fa..a6f9cb84d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.291 + rev: v0.0.292 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5c95186764d57242b181e398f4b1de2a68dfaf91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 23:53:58 +0200 Subject: [PATCH 078/162] [pre-commit.ci] pre-commit autoupdate (#1906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6f9cb84d..425cddd03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] From b9ebb9187a621f72df2996d7bd89c6c55031a860 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 07:06:05 +0200 Subject: [PATCH 079/162] [pre-commit.ci] pre-commit autoupdate (#1907) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.0.292 → v0.1.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.292...v0.1.0) * ruff --output-format=github . --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .github/workflows/test.yml | 4 ++-- .pre-commit-config.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5d69b39a..ca34b1913 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,9 +10,9 @@ jobs: ruff: # https://beta.ruff.rs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: pip install --user ruff - - run: ruff --format=github . + - run: ruff --output-format=github . test: runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 425cddd03..97dc5fabc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 + rev: v0.1.0 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 31a221c0dae33b4b5b7385b8bdcce9b109942164 Mon Sep 17 00:00:00 2001 From: notPlancha Date: Mon, 23 Oct 2023 18:40:50 +0100 Subject: [PATCH 080/162] __unicode__ to __str__ in tutorial --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b902b7894..d3228beea 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -54,7 +54,7 @@ note-taking application. Here is ``myapp/models.py``:: title = models.CharField(max_length=200) body = models.TextField() - def __unicode__(self): + def __str__(self): return self.title Finally, before starting with Haystack, you will want to choose a search From 1811f187a8909e4e40f7917f9e2bfb8095171230 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 21:08:00 +0000 Subject: [PATCH 081/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.0 → v0.1.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.0...v0.1.1) - [github.com/psf/black: 23.9.1 → 23.10.1](https://github.com/psf/black/compare/23.9.1...23.10.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97dc5fabc..d9b76881c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.0 + rev: v0.1.1 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From f3465368e2efb2cfe569a08d8e749542f846a219 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 00:58:12 +0100 Subject: [PATCH 082/162] [pre-commit.ci] pre-commit autoupdate (#1910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.1 → v0.1.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.1...v0.1.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d9b76881c..61f1d4c54 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.1 + rev: v0.1.3 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 56d646512155b12ba17df8357ed59ca22e7d5fd7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:54:05 +0600 Subject: [PATCH 083/162] [pre-commit.ci] pre-commit autoupdate (#1911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.3 → v0.1.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.3...v0.1.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 61f1d4c54..b1cc6c15d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.3 + rev: v0.1.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 014ee191fc0620fedd49eccc8b927bb2726215a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:24:08 +0600 Subject: [PATCH 084/162] [pre-commit.ci] pre-commit autoupdate (#1912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.4 → v0.1.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.4...v0.1.5) - [github.com/psf/black: 23.10.1 → 23.11.0](https://github.com/psf/black/compare/23.10.1...23.11.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b1cc6c15d..57ccbf3c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.4 + rev: v0.1.5 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From eb3f1d54973517309da559876fd6c7cf6b971025 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:29:58 +0545 Subject: [PATCH 085/162] [pre-commit.ci] pre-commit autoupdate (#1913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.5 → v0.1.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.5...v0.1.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57ccbf3c5..38d19bd8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.5 + rev: v0.1.6 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 961378d7100c77d6a6c00bbdece2031e2f0609e7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 6 Dec 2023 16:43:51 +0100 Subject: [PATCH 086/162] Django_v5.0 --- haystack/fields.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/haystack/fields.py b/haystack/fields.py index 0965377ea..44f150be6 100644 --- a/haystack/fields.py +++ b/haystack/fields.py @@ -2,7 +2,11 @@ from inspect import ismethod from django.template import loader -from django.utils import datetime_safe + +try: # datetime_safe was removed in Django 5.0 + from django.utils import datetime_safe +except ImportError: + import datetime as datetime_safe from haystack.exceptions import SearchFieldError from haystack.utils import get_model_ct_tuple From 7afd020ff928ab7c5886c8bd5a11bab8fbc240e7 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 6 Dec 2023 21:59:54 +0100 Subject: [PATCH 087/162] fields.py: Replace datetime_safe with Standard Library datetime --- haystack/fields.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/haystack/fields.py b/haystack/fields.py index 44f150be6..3531bf31b 100644 --- a/haystack/fields.py +++ b/haystack/fields.py @@ -1,13 +1,9 @@ +import datetime import re from inspect import ismethod from django.template import loader -try: # datetime_safe was removed in Django 5.0 - from django.utils import datetime_safe -except ImportError: - import datetime as datetime_safe - from haystack.exceptions import SearchFieldError from haystack.utils import get_model_ct_tuple @@ -399,7 +395,7 @@ def convert(self, value): if match: data = match.groupdict() - return datetime_safe.date( + return datetime.date( int(data["year"]), int(data["month"]), int(data["day"]) ) else: @@ -432,7 +428,7 @@ def convert(self, value): if match: data = match.groupdict() - return datetime_safe.datetime( + return datetime.datetime( int(data["year"]), int(data["month"]), int(data["day"]), From 927fe675de149adf79b6e4c8f188a056e0b9c6f2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 7 Dec 2023 00:32:13 +0100 Subject: [PATCH 088/162] README.rst: Experimental support for Django v5.0 --- README.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 22afa29b1..e573494f2 100644 --- a/README.rst +++ b/README.rst @@ -59,9 +59,19 @@ Requirements Haystack has a relatively easily-met set of requirements. -* Python 3.6+ +* A supported version of Python: https://devguide.python.org/versions/#supported-versions * A supported version of Django: https://www.djangoproject.com/download/#supported-versions Additionally, each backend has its own requirements. You should refer to https://django-haystack.readthedocs.io/en/latest/installing_search_engines.html for more details. + +Experimental support for Django v5.0 +==================================== + +The current release on PyPI_ does not yet support Django v5.0. + +.. _PyPI: https://pypi.org/project/django-haystack/ + +To run on Django v5.0, please install by using: +``pip install git+https://github.com/django-haystack/django-haystack.git`` From 3d59b94810e5fe851bad4f2ac400e5d2803c8514 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 7 Dec 2023 00:49:28 +0100 Subject: [PATCH 089/162] setup.py: Current Python and current Django --- setup.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index eb77d460c..f7022ddd7 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,10 @@ #!/usr/bin/env python from setuptools import setup -install_requires = ["Django>=2.2"] +install_requires = [ + "Django>=3.2", + "setuptools", +] tests_require = [ "pysolr>=3.7.0", @@ -39,18 +42,19 @@ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", "Framework :: Django", - "Framework :: Django :: 2.2", - "Framework :: Django :: 3.1", "Framework :: Django :: 3.2", + "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Utilities", ], zip_safe=False, From 1bf01d64de72d8950da052cf4ebebf31faa9442b Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Thu, 7 Dec 2023 12:02:45 -0500 Subject: [PATCH 090/162] Use GitHub Actions to publish to PyPI --- .github/workflows/pypi-release.yml | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/pypi-release.yml diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 000000000..a7a42a2f0 --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,38 @@ +name: "PyPI releases" + +on: release + +jobs: + build_sdist: + name: Build Python source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + pypi-publish: + name: Upload release to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + needs: + - build_sdist + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pysolr + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + # unpacks default artifact into dist/ + # if `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From 1a10943ee57cc6d5a131a167e819447d0d005750 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Thu, 7 Dec 2023 15:12:16 -0500 Subject: [PATCH 091/162] PyPI: fix environment declaration --- .github/workflows/publish.yml | 22 ---------------------- .github/workflows/pypi-release.yml | 2 +- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 382bdbc4c..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Publish - -on: - release: - types: [published] - -jobs: - publish: - - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install dependencies - run: python -m pip install --upgrade pip setuptools twine wheel - - name: Build package - run: python setup.py sdist bdist_wheel - - name: Publish to PyPI - run: twine upload --non-interactive dist/* diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index a7a42a2f0..0c001ce28 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/pysolr + url: https://pypi.org/p/django-haystack permissions: id-token: write steps: From 5b546c225d096060e4a8359878820140dfec48d3 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Thu, 7 Dec 2023 15:19:08 -0500 Subject: [PATCH 092/162] Add minimal Read the Docs configuration --- .readthedocs.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..eef0a0675 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py From ab84664377ad255da3b61d10bd177fded8a332ef Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 23:04:38 +0100 Subject: [PATCH 093/162] [pre-commit.ci] pre-commit autoupdate (#1923) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.6 → v0.1.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.6...v0.1.7) - [github.com/PyCQA/isort: 5.12.0 → 5.13.1](https://github.com/PyCQA/isort/compare/5.12.0...5.13.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38d19bd8e..3751df452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.6 + rev: v0.1.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.1 hooks: - id: isort - repo: https://github.com/psf/black From b4ec8554a2432980d9ac40e196352f235673967c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 07:30:18 +0100 Subject: [PATCH 094/162] [pre-commit.ci] pre-commit autoupdate (#1925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.7 → v0.1.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.7...v0.1.8) - [github.com/PyCQA/isort: 5.13.1 → 5.13.2](https://github.com/PyCQA/isort/compare/5.13.1...5.13.2) - [github.com/psf/black: 23.11.0 → 23.12.0](https://github.com/psf/black/compare/23.11.0...23.12.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3751df452..1b3679323 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,17 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.7 + rev: v0.1.8 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] - repo: https://github.com/PyCQA/isort - rev: 5.13.1 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 23.12.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 6f20d61214281fab9aa1a12f4fcf39029a0c55e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Dec 2023 23:07:46 +0100 Subject: [PATCH 095/162] [pre-commit.ci] pre-commit autoupdate (#1927) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.8 → v0.1.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.8...v0.1.9) - [github.com/psf/black: 23.12.0 → 23.12.1](https://github.com/psf/black/compare/23.12.0...23.12.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b3679323..d300b81ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.8 + rev: v0.1.9 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 3a566a50e4963bed4fb8853eca60bc894b0b7fc5 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 1 Jan 2024 19:53:28 +0100 Subject: [PATCH 096/162] Fix unittest assert calls for Python 3.12 --- test_haystack/test_managers.py | 4 ++-- test_haystack/test_query.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test_haystack/test_managers.py b/test_haystack/test_managers.py index 3784217cd..cc600752e 100644 --- a/test_haystack/test_managers.py +++ b/test_haystack/test_managers.py @@ -242,11 +242,11 @@ def spelling_suggestion(self): def test_values(self): sqs = self.search_index.objects.auto_query("test").values("id") - self.assert_(isinstance(sqs, ValuesSearchQuerySet)) + self.assertIsInstance(sqs, ValuesSearchQuerySet) def test_valueslist(self): sqs = self.search_index.objects.auto_query("test").values_list("id") - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) class CustomManagerTestCase(TestCase): diff --git a/test_haystack/test_query.py b/test_haystack/test_query.py index ffe35c19a..f7e9a1707 100644 --- a/test_haystack/test_query.py +++ b/test_haystack/test_query.py @@ -442,7 +442,7 @@ def test_len(self): def test_repr(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) - self.assertRegexpMatches( + self.assertRegex( repr(self.msqs), r"^, using=None>$", @@ -967,18 +967,18 @@ def test_or_and(self): class ValuesQuerySetTestCase(SearchQuerySetTestCase): def test_values_sqs(self): sqs = self.msqs.auto_query("test").values("id") - self.assert_(isinstance(sqs, ValuesSearchQuerySet)) + self.assertIsInstance(sqs, ValuesSearchQuerySet) # We'll do a basic test to confirm that slicing works as expected: - self.assert_(isinstance(sqs[0], dict)) - self.assert_(isinstance(sqs[0:5][0], dict)) + self.assertIsInstance(sqs[0], dict) + self.assertIsInstance(sqs[0:5][0], dict) def test_valueslist_sqs(self): sqs = self.msqs.auto_query("test").values_list("id") - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) - self.assert_(isinstance(sqs[0], (list, tuple))) - self.assert_(isinstance(sqs[0:1][0], (list, tuple))) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) + self.assertIsInstance(sqs[0], (list, tuple)) + self.assertIsInstance(sqs[0:1][0], (list, tuple)) self.assertRaises( TypeError, @@ -989,12 +989,12 @@ def test_valueslist_sqs(self): ) flat_sqs = self.msqs.auto_query("test").values_list("id", flat=True) - self.assert_(isinstance(sqs, ValuesListSearchQuerySet)) + self.assertIsInstance(sqs, ValuesListSearchQuerySet) # Note that this will actually be None because a mocked sqs lacks # anything else: - self.assert_(flat_sqs[0] is None) - self.assert_(flat_sqs[0:1][0] is None) + self.assertIsNone(flat_sqs[0]) + self.assertIsNone(flat_sqs[0:1][0]) class EmptySearchQuerySetTestCase(TestCase): From 3267b832ec82731a780c3e4126d802901ac1925a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 4 Jan 2024 17:18:08 +0100 Subject: [PATCH 097/162] Fix Django warnings admin.W411 and models.W042 --- .pre-commit-config.yaml | 2 +- test_haystack/settings.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d300b81ef..b0488e9f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.9 + rev: v0.1.11 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] diff --git a/test_haystack/settings.py b/test_haystack/settings.py index c4234f547..9a78bc5bc 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -8,6 +8,9 @@ "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": "haystack_tests.db"} } +# Use BigAutoField as the default auto field for all models +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", @@ -34,6 +37,7 @@ "APP_DIRS": True, "OPTIONS": { "context_processors": [ + "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", ] From 882fecfd41389a80ba1098adbd7bd8f5c6bb8197 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 5 Dec 2023 13:28:29 +0100 Subject: [PATCH 098/162] GitHub Actions: Update test matrix for Django v5.0 https://pypi.org/project/Django https://www.djangoproject.com/weblog/2023/dec/04/django-50-released > Django 4.1 has reached the end of extended support. The final security release ([4.1.13](https://docs.djangoproject.com/en/stable/releases/4.1.13/)) was issued on November 1st. All Django 4.1 users are encouraged to [upgrade](https://docs.djangoproject.com/en/dev/howto/upgrade-version/) to Django 4.2 or later. https://docs.djangoproject.com/en/5.0/releases/5.0 > Django 5.0 supports Python 3.10, 3.11, and 3.12. https://docs.djangoproject.com/en/5.0/faq/install/#what-python-version-can-i-use-with-django --- .github/workflows/test.yml | 32 ++++++++++++++++---------------- pyproject.toml | 4 ++-- tox.ini | 8 +++----- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca34b1913..24aae21fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,31 +7,31 @@ on: branches: [master] jobs: - ruff: # https://beta.ruff.rs + ruff: # https://docs.astral.sh/ruff runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: pip install --user ruff - - run: ruff --output-format=github . + - run: ruff --output-format=github test: runs-on: ubuntu-latest needs: ruff # Do not run the tests if linting fails. strategy: fail-fast: false - matrix: # https://docs.djangoproject.com/en/4.1/faq/install - django-version: ["3.2", "4.0", "4.1"] - python-version: ["3.7", "3.8", "3.9", "3.10"] - elastic-version: ["1.7", "2.4", "5.5", "7.13.1"] + matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django + django-version: ["3.2", "4.2", "5.0"] + python-version: ["3.8", "3.9"] # , "3.10", "3.11", "3.12"] # Whoosh issues with Py3.10+ + elastic-version: ["7.17.9"] exclude: - - django-version: "4.0" - python-version: "3.7" - - django-version: "4.1" - python-version: "3.7" - include: - - django-version: "4.1" + - django-version: "3.2" python-version: "3.11" - elastic-version: "7.13.1" + - django-version: "3.2" + python-version: "3.12" + - django-version: "5.0" + python-version: "3.8" + - django-version: "5.0" + python-version: "3.9" services: elastic: image: elasticsearch:${{ matrix.elastic-version }} @@ -47,11 +47,11 @@ jobs: solr: image: solr:6 ports: - - 9001:9001 + - 9001:8983 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install system dependencies diff --git a/pyproject.toml b/pyproject.toml index 3a08e1563..b2467d40b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,9 @@ write_to = "haystack/version.py" exclude = ["test_haystack"] ignore = ["B018", "B028", "B904", "B905"] line-length = 162 -select = ["ASYNC", "B", "C4", "E", "F", "G", "PLR091", "W"] +select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] show-source = true -target-version = "py37" +target-version = "py38" [tool.ruff.isort] known-first-party = ["haystack", "test_haystack"] diff --git a/tox.ini b/tox.ini index e2b2e711b..fa7ad5381 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs - py{36,37,38,39,310,py}-django{2.2,3.0,3.1,3.2,4.0}-es{1.x,2.x,5.x,7.x} + py{38,39,310,311,312,py3}-django{3.2,4.2,5.0}-es{1.x,2.x,5.x,7.x} [testenv] @@ -15,11 +15,9 @@ deps = geopy==2.0.0 coverage requests - django2.2: Django>=2.2,<3.0 - django3.0: Django>=3.0,<3.1 - django3.1: Django>=3.1,<3.2 django3.2: Django>=3.2,<3.3 - django4.0: Django>=4.0,<4.1 + django4.2: Django>=4.2,<4.3 + django5.0: Django>=5.0,<5.1 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6 From 233f7c0683ebf7200d70ce2946132c7926f67a9f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 7 Jan 2024 11:05:36 +0100 Subject: [PATCH 099/162] Update tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fa7ad5381..8585d2068 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs - py{38,39,310,311,312,py3}-django{3.2,4.2,5.0}-es{1.x,2.x,5.x,7.x} + py{38,39,310,311,312,py3}-django{3.2,4.2,5.0}-es7.x [testenv] From bc342e0ac2ff151eb5241df777241109bc0579a6 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 14 Jan 2024 16:47:07 +0100 Subject: [PATCH 100/162] pre-commit: format files with prettier (#1936) * pre-commit: format files with prettier * yaml indent_size = 2 --- .editorconfig | 3 + .github/workflows/codeql-analysis.yml | 22 ++++---- .github/workflows/docs.yml | 18 +++--- .github/workflows/pypi-release.yml | 60 ++++++++++---------- .github/workflows/test.yml | 48 ++++++++-------- .pre-commit-config.yaml | 80 ++++++++++++++------------- .readthedocs.yaml | 8 +-- 7 files changed, 124 insertions(+), 115 deletions(-) diff --git a/.editorconfig b/.editorconfig index 87fb28e32..d4649a5fa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,6 +15,9 @@ charset = utf-8 [Makefile] indent_style = tab +[*.{yaml,yml}] +indent_size = 2 + # We don't want to apply our defaults to third-party code or minified bundles: [**/{external,vendor}/**,**.min.{js,css}] indent_style = ignore diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7e7e0054d..e8f8adeba 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,12 +2,12 @@ name: "CodeQL" on: push: - branches: [master, ] + branches: [master] pull_request: # The branches below must be a subset of the branches above branches: [master] schedule: - - cron: '0 6 * * 5' + - cron: "0 6 * * 5" jobs: analyze: @@ -15,14 +15,14 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: python + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 52630ede0..5485eb4c7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,12 +6,12 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.x - - name: Install dependencies - run: pip install sphinx - - name: Build docs - run: cd docs && make html + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + - name: Install dependencies + run: pip install sphinx + - name: Build docs + run: cd docs && make html diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 0c001ce28..05c7dc02e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -3,36 +3,36 @@ name: "PyPI releases" on: release jobs: - build_sdist: - name: Build Python source distribution - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 + build_sdist: + name: Build Python source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 - - name: Build sdist - run: pipx run build --sdist + - name: Build sdist + run: pipx run build --sdist - - uses: actions/upload-artifact@v3 - with: - path: dist/*.tar.gz + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz - pypi-publish: - name: Upload release to PyPI - if: github.event_name == 'release' && github.event.action == 'published' - needs: - - build_sdist - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/django-haystack - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v3 - with: - # unpacks default artifact into dist/ - # if `name: artifact` is omitted, the action will create extra parent dir - name: artifact - path: dist - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + pypi-publish: + name: Upload release to PyPI + if: github.event_name == 'release' && github.event.action == 'published' + needs: + - build_sdist + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/django-haystack + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + # unpacks default artifact into dist/ + # if `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24aae21fb..0dac7558a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,21 +7,21 @@ on: branches: [master] jobs: - ruff: # https://docs.astral.sh/ruff + ruff: # https://docs.astral.sh/ruff runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - run: pip install --user ruff - - run: ruff --output-format=github + - uses: actions/checkout@v4 + - run: pip install --user ruff + - run: ruff --output-format=github test: runs-on: ubuntu-latest - needs: ruff # Do not run the tests if linting fails. + needs: ruff # Do not run the tests if linting fails. strategy: fail-fast: false - matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django + matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django django-version: ["3.2", "4.2", "5.0"] - python-version: ["3.8", "3.9"] # , "3.10", "3.11", "3.12"] # Whoosh issues with Py3.10+ + python-version: ["3.8", "3.9"] # , "3.10", "3.11", "3.12"] # Whoosh issues with Py3.10+ elastic-version: ["7.17.9"] exclude: - django-version: "3.2" @@ -49,20 +49,20 @@ jobs: ports: - 9001:8983 steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: sudo apt install --no-install-recommends -y gdal-bin - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - pip install coverage requests tox tox-gh-actions - pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} - python setup.py clean build install - - name: Run test - run: tox -v - env: - DJANGO: ${{ matrix.django-version }} + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + run: sudo apt install --no-install-recommends -y gdal-bin + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install coverage requests tox tox-gh-actions + pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} + python setup.py clean build install + - name: Run test + run: tox -v + env: + DJANGO: ${{ matrix.django-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0488e9f1..a2683d8e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,40 +1,46 @@ exclude: ".*/vendor/.*" repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.11 - hooks: - - id: ruff - # args: [ --fix, --exit-non-zero-on-fix ] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.13 + hooks: + - id: ruff + # args: [ --fix, --exit-non-zero-on-fix ] - - repo: https://github.com/PyCQA/isort - rev: 5.13.2 - hooks: - - id: isort - - repo: https://github.com/psf/black - rev: 23.12.1 - hooks: - - id: black - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: check-added-large-files - args: ["--maxkb=128"] - - id: check-ast - - id: check-byte-order-marker - - id: check-case-conflict - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-json - - id: check-merge-conflict - - id: check-symlinks - - id: check-toml - - id: check-xml - - id: check-yaml - - id: debug-statements - - id: detect-private-key - - id: end-of-file-fixer - - id: mixed-line-ending - args: ["--fix=lf"] - - id: pretty-format-json - args: ["--autofix", "--no-sort-keys", "--indent=4"] - - id: trailing-whitespace + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/psf/black + rev: 23.12.1 + hooks: + - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files + args: ["--maxkb=128"] + - id: check-ast + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + - id: pretty-format-json + args: ["--autofix", "--no-sort-keys", "--indent=4"] + - id: trailing-whitespace + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + types_or: [json, toml, xml, yaml] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index eef0a0675..134784f59 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,9 +4,9 @@ version: 2 build: - os: ubuntu-22.04 - tools: - python: "3.12" + os: ubuntu-22.04 + tools: + python: "3.12" sphinx: - configuration: docs/conf.py + configuration: docs/conf.py From 3bc1d4e0893e0f1c093a36ee99170c9a2e2e262d Mon Sep 17 00:00:00 2001 From: Georg Date: Sat, 20 Jan 2024 19:28:04 +0100 Subject: [PATCH 101/162] Migrate away from pkg_resources (#1935) * Migrate away from pkg_resources Using pkg_resources as an API is deprecated. Migrate functionality to their importlib and packaging equivalents. Signed-off-by: Georg Pfuetzenreuter * Add packaging to requirements Required for packaging.version after the removal of pkg_resources. Signed-off-by: Georg Pfuetzenreuter --------- Signed-off-by: Georg Pfuetzenreuter --- haystack/__init__.py | 13 +++++++------ setup.py | 1 + test_haystack/solr_tests/test_solr_backend.py | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/haystack/__init__.py b/haystack/__init__.py index 94b8f4674..25448de96 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -1,7 +1,9 @@ +from importlib.metadata import PackageNotFoundError, version + import django from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from pkg_resources import DistributionNotFound, get_distribution, parse_version +from packaging.version import Version from haystack.constants import DEFAULT_ALIAS from haystack.utils import loading @@ -9,12 +11,11 @@ __author__ = "Daniel Lindsley" try: - pkg_distribution = get_distribution("django-haystack") - __version__ = pkg_distribution.version - version_info = pkg_distribution.parsed_version -except DistributionNotFound: + __version__ = version("django-haystack") + version_info = Version(__version__) +except PackageNotFoundError: __version__ = "0.0.dev0" - version_info = parse_version(__version__) + version_info = Version(__version__) if django.VERSION < (3, 2): diff --git a/setup.py b/setup.py index f7022ddd7..70b029272 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ install_requires = [ "Django>=3.2", + "packaging", "setuptools", ] diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index d20347e7e..d8c95d329 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -10,7 +10,7 @@ from django.conf import settings from django.test import TestCase from django.test.utils import override_settings -from pkg_resources import parse_version +from packaging.version import Version from haystack import connections, indexes, reset_search_queries from haystack.exceptions import SkipDocument @@ -1650,7 +1650,7 @@ def test_boost(self): @unittest.skipIf( - parse_version(pysolr.__version__) < parse_version("3.1.1"), + Version(pysolr.__version__) < Version("3.1.1"), "content extraction requires pysolr > 3.1.1", ) class LiveSolrContentExtractionTestCase(TestCase): From c0b1984c24647d2d7f72c890d0c88d2ae13320b3 Mon Sep 17 00:00:00 2001 From: Naggafin Date: Sun, 21 Jan 2024 19:13:29 -0500 Subject: [PATCH 102/162] updated whoosh backend to utilize datetime from the standard library (#1937) Co-authored-by: me --- haystack/backends/whoosh_backend.py | 2 +- test_haystack/whoosh_tests/test_whoosh_backend.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 5cf7832ec..5cec91f45 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -4,10 +4,10 @@ import shutil import threading import warnings +from datetime import date, datetime from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.utils.datetime_safe import date, datetime from django.utils.encoding import force_str from haystack.backends import ( diff --git a/test_haystack/whoosh_tests/test_whoosh_backend.py b/test_haystack/whoosh_tests/test_whoosh_backend.py index fd5f56e14..46fe88271 100644 --- a/test_haystack/whoosh_tests/test_whoosh_backend.py +++ b/test_haystack/whoosh_tests/test_whoosh_backend.py @@ -1,12 +1,11 @@ import os import unittest -from datetime import timedelta +from datetime import date, datetime, timedelta from decimal import Decimal from django.conf import settings from django.test import TestCase from django.test.utils import override_settings -from django.utils.datetime_safe import date, datetime from whoosh.analysis import SpaceSeparatedTokenizer, SubstitutionFilter from whoosh.fields import BOOLEAN, DATETIME, KEYWORD, NUMERIC, TEXT from whoosh.qparser import QueryParser From f28139e126967fdf39c0db1cd813fe553cfee268 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 21:54:36 +0100 Subject: [PATCH 103/162] [pre-commit.ci] pre-commit autoupdate (#1940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.13 → v0.1.14](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.13...v0.1.14) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2683d8e0..b698bf4c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.13 + rev: v0.1.14 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From ff96833967ecfbe4e00b493bfa306d098524b1d8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:35:43 +0100 Subject: [PATCH 104/162] [pre-commit.ci] pre-commit autoupdate (#1941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/psf/black: 23.12.1 → 24.1.1](https://github.com/psf/black/compare/23.12.1...24.1.1) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Keep GitHub Actions up to date with GitHub's Dependabot * https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot * https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .github/dependabot.yml | 13 +++++++++ .pre-commit-config.yaml | 2 +- haystack/backends/elasticsearch_backend.py | 8 +++-- haystack/backends/simple_backend.py | 1 + haystack/backends/solr_backend.py | 29 ++++++++++--------- test_haystack/test_django_config_detection.py | 1 + test_haystack/whoosh_tests/test_forms.py | 1 + 7 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4b5e1c762 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b698bf4c0..1eef7b4b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.1.1 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/haystack/backends/elasticsearch_backend.py b/haystack/backends/elasticsearch_backend.py index 6c708f4f3..e8febf9d3 100644 --- a/haystack/backends/elasticsearch_backend.py +++ b/haystack/backends/elasticsearch_backend.py @@ -677,9 +677,11 @@ def _process_results( if raw_suggest: spelling_suggestion = " ".join( [ - word["text"] - if len(word["options"]) == 0 - else word["options"][0]["text"] + ( + word["text"] + if len(word["options"]) == 0 + else word["options"][0]["text"] + ) for word in raw_suggest ] ) diff --git a/haystack/backends/simple_backend.py b/haystack/backends/simple_backend.py index a94625281..bfef88cb2 100644 --- a/haystack/backends/simple_backend.py +++ b/haystack/backends/simple_backend.py @@ -1,6 +1,7 @@ """ A very basic, ORM-based backend for simple search during tests. """ + from functools import reduce from warnings import warn diff --git a/haystack/backends/solr_backend.py b/haystack/backends/solr_backend.py index 405508523..e077aa302 100644 --- a/haystack/backends/solr_backend.py +++ b/haystack/backends/solr_backend.py @@ -267,9 +267,9 @@ def build_search_kwargs( for facet_field, options in facets.items(): for key, value in options.items(): - kwargs[ - "f.%s.facet.%s" % (facet_field, key) - ] = self.conn._from_python(value) + kwargs["f.%s.facet.%s" % (facet_field, key)] = ( + self.conn._from_python(value) + ) if date_facets is not None: kwargs["facet"] = "on" @@ -277,23 +277,24 @@ def build_search_kwargs( kwargs["facet.%s.other" % self.date_facet_field] = "none" for key, value in date_facets.items(): - kwargs[ - "f.%s.facet.%s.start" % (key, self.date_facet_field) - ] = self.conn._from_python(value.get("start_date")) - kwargs[ - "f.%s.facet.%s.end" % (key, self.date_facet_field) - ] = self.conn._from_python(value.get("end_date")) + kwargs["f.%s.facet.%s.start" % (key, self.date_facet_field)] = ( + self.conn._from_python(value.get("start_date")) + ) + kwargs["f.%s.facet.%s.end" % (key, self.date_facet_field)] = ( + self.conn._from_python(value.get("end_date")) + ) gap_by_string = value.get("gap_by").upper() gap_string = "%d%s" % (value.get("gap_amount"), gap_by_string) if value.get("gap_amount") != 1: gap_string += "S" - kwargs[ - "f.%s.facet.%s.gap" % (key, self.date_facet_field) - ] = "+%s/%s" % ( - gap_string, - gap_by_string, + kwargs["f.%s.facet.%s.gap" % (key, self.date_facet_field)] = ( + "+%s/%s" + % ( + gap_string, + gap_by_string, + ) ) if query_facets is not None: diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py index 31241a48f..0c3827882 100644 --- a/test_haystack/test_django_config_detection.py +++ b/test_haystack/test_django_config_detection.py @@ -1,4 +1,5 @@ """""" + import unittest import django diff --git a/test_haystack/whoosh_tests/test_forms.py b/test_haystack/whoosh_tests/test_forms.py index 204d14f46..64be222fc 100644 --- a/test_haystack/whoosh_tests/test_forms.py +++ b/test_haystack/whoosh_tests/test_forms.py @@ -1,4 +1,5 @@ """Tests for Whoosh spelling suggestions""" + from django.conf import settings from django.http import HttpRequest From 68f84884c1e8d2861d443d25d214a29c59f48404 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:40:16 +0100 Subject: [PATCH 105/162] Bump the github-actions group with 5 updates (#1942) Bumps the github-actions group with 5 updates: | Package | From | To | | --- | --- | --- | | [actions/checkout](https://github.com/actions/checkout) | `3` | `4` | | [github/codeql-action](https://github.com/github/codeql-action) | `1` | `3` | | [actions/setup-python](https://github.com/actions/setup-python) | `4` | `5` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `3` | `4` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `3` | `4` | Updates `actions/checkout` from 3 to 4 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) Updates `github/codeql-action` from 1 to 3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v1...v3) Updates `actions/setup-python` from 4 to 5 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) Updates `actions/upload-artifact` from 3 to 4 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) Updates `actions/download-artifact` from 3 to 4 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/docs.yml | 4 ++-- .github/workflows/pypi-release.yml | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index e8f8adeba..91fea6827 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,13 +16,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: python - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5485eb4c7..edbe9af1a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,9 +6,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.x - name: Install dependencies diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 05c7dc02e..7a158c5be 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,12 +7,12 @@ jobs: name: Build Python source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build sdist run: pipx run build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz @@ -28,7 +28,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: # unpacks default artifact into dist/ # if `name: artifact` is omitted, the action will create extra parent dir From c9606940b6aff0215b9403dbeb1b69f016dc06d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 21:56:50 +0100 Subject: [PATCH 106/162] [pre-commit.ci] pre-commit autoupdate (#1944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.14 → v0.2.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.14...v0.2.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1eef7b4b8..a94a09313 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.14 + rev: v0.2.0 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 2b224806c61d58bcb32538cfde8548c0b25949d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 08:47:18 +0100 Subject: [PATCH 107/162] [pre-commit.ci] pre-commit autoupdate (#1947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.0 → v0.2.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.0...v0.2.1) - [github.com/psf/black: 24.1.1 → 24.2.0](https://github.com/psf/black/compare/24.1.1...24.2.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a94a09313..c7564853a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.2.1 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From d641a1bb7cd4eb834aa9e321e0755f07b8655433 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 05:53:59 +0100 Subject: [PATCH 108/162] [pre-commit.ci] pre-commit autoupdate (#1948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.1 → v0.2.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.1...v0.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7564853a..1f193aed2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.1 + rev: v0.2.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From b4fb1af8238f4200b0955f93e5f5ce4cc42e638e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 08:34:59 +0100 Subject: [PATCH 109/162] [pre-commit.ci] pre-commit autoupdate (#1950) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.2 → v0.3.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.2...v0.3.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f193aed2..75d906cf8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.3.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 77a8306ac7194f48082734b01e7209faeff43f90 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 07:43:15 +0100 Subject: [PATCH 110/162] [pre-commit.ci] pre-commit autoupdate (#1951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.2 → v0.3.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.2...v0.3.3) - [github.com/psf/black: 24.2.0 → 24.3.0](https://github.com/psf/black/compare/24.2.0...24.3.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75d906cf8..709e1e44f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.3.3 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 3511343fe7d9b50612c09466bdefada650ba3b84 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 03:33:21 +0100 Subject: [PATCH 111/162] [pre-commit.ci] pre-commit autoupdate (#1952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.3 → v0.3.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.3...v0.3.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 709e1e44f..dcfdc2dad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.3.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From c045da6cc182c5b6f8e6ba3c7ebe35b98149f9c3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 03:31:22 +0200 Subject: [PATCH 112/162] [pre-commit.ci] pre-commit autoupdate (#1954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.3.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.3.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dcfdc2dad..baa52bb94 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.3.5 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 1b61fd9d0553b157027eba508f08046b2bff310b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 07:37:17 +0200 Subject: [PATCH 113/162] [pre-commit.ci] pre-commit autoupdate (#1955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index baa52bb94..81f355d5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] From 76668c5c4fb44ce77fe7cf6a8f5decd04654fa5d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 05:04:51 +0200 Subject: [PATCH 114/162] [pre-commit.ci] pre-commit autoupdate (#1957) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.5 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.5...v0.3.7) - [github.com/psf/black: 24.3.0 → 24.4.0](https://github.com/psf/black/compare/24.3.0...24.4.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81f355d5a..34b024826 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 + rev: v0.3.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.4.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From ae7bd394b47a5102033b563e15577af58804203f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 07:15:52 +0200 Subject: [PATCH 115/162] [pre-commit.ci] pre-commit autoupdate (#1958) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.7 → v0.4.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.7...v0.4.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34b024826..6423c61f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.7 + rev: v0.4.1 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 0c5f60dc75d78761bc17dd3f9d41ed3d7eb6f43c Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 16:16:55 +0200 Subject: [PATCH 116/162] fix solr under github actions --- .github/workflows/test.yml | 2 ++ .../server/setup-solr-test-server-in-docker.sh | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0dac7558a..7bd4d8a2f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system dependencies run: sudo apt install --no-install-recommends -y gdal-bin + - name: Setup solr test server in Docker + run: bash test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel diff --git a/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh b/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh new file mode 100644 index 000000000..bf2b4fb9d --- /dev/null +++ b/test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh @@ -0,0 +1,15 @@ +# figure out the solr container ID +SOLR_CONTAINER=`docker ps -f ancestor=solr:6 --format '{{.ID}}'` + +LOCAL_CONFDIR=./test_haystack/solr_tests/server/confdir +CONTAINER_CONFDIR=/opt/solr/server/solr/collection1/conf + +# set up a solr core +docker exec $SOLR_CONTAINER ./bin/solr create -c collection1 -p 8983 -n basic_config +# copy the testing schema to the collection and fix permissions +docker cp $LOCAL_CONFDIR/solrconfig.xml $SOLR_CONTAINER:$CONTAINER_CONFDIR/solrconfig.xml +docker cp $LOCAL_CONFDIR/schema.xml $SOLR_CONTAINER:$CONTAINER_CONFDIR/schema.xml +docker exec $SOLR_CONTAINER mv $CONTAINER_CONFDIR/managed-schema $CONTAINER_CONFDIR/managed-schema.old +docker exec -u root $SOLR_CONTAINER chown -R solr:solr /opt/solr/server/solr/collection1 +# reload the solr core +curl "http://localhost:9001/solr/admin/cores?action=RELOAD&core=collection1" From bc1ffd8e838333b736d560b47346b50ae81a236c Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 20:39:42 +0200 Subject: [PATCH 117/162] fix tox gh actions configuration --- tox.ini | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8585d2068..498bb7f00 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,21 @@ [tox] envlist = docs - py{38,39,310,311,312,py3}-django{3.2,4.2,5.0}-es7.x + py{38,39,310,311,312}-django{3.2,4.2,5.0}-es7.x +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 + +[gh-actions:env] +DJANGO = + 3.2: django3.2 + 4.2: django4.2 + 5.0: django5.0 [testenv] commands = From ee8b511ddae4c567d9e8d9b52686a5a6eee39a58 Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:53:10 +0200 Subject: [PATCH 118/162] fix search_help_text --- haystack/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/haystack/admin.py b/haystack/admin.py index feeb1f3f3..590b240d5 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -1,3 +1,4 @@ +from django import VERSION as django_version from django.contrib.admin.options import ModelAdmin, csrf_protect_m from django.contrib.admin.views.main import SEARCH_VAR, ChangeList from django.core.exceptions import PermissionDenied @@ -15,7 +16,10 @@ class SearchChangeList(ChangeList): def __init__(self, **kwargs): self.haystack_connection = kwargs.pop("haystack_connection", DEFAULT_ALIAS) - super().__init__(**kwargs) + super_kwargs = kwargs + if django_version[0] >= 4: + super_kwargs['search_help_text'] = 'Search...' + super().__init__(**super_kwargs) def get_results(self, request): if SEARCH_VAR not in request.GET: From 046e3f55c400462a79f4fdec80181894823220f2 Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:53:23 +0200 Subject: [PATCH 119/162] fix test_rebuild_index_nocommit --- test_haystack/test_management_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_haystack/test_management_commands.py b/test_haystack/test_management_commands.py index d78203007..d82a44aff 100644 --- a/test_haystack/test_management_commands.py +++ b/test_haystack/test_management_commands.py @@ -77,8 +77,8 @@ def test_rebuild_index(self, mock_handle_clear, mock_handle_update): self.assertTrue(mock_handle_clear.called) self.assertTrue(mock_handle_update.called) - @patch("haystack.management.commands.update_index.Command.handle") - @patch("haystack.management.commands.clear_index.Command.handle") + @patch("haystack.management.commands.update_index.Command.handle", return_value='') + @patch("haystack.management.commands.clear_index.Command.handle", return_value='') def test_rebuild_index_nocommit(self, *mocks): call_command("rebuild_index", interactive=False, commit=False) From 543b9dab8511aec66232dbfaff7310526e122afb Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:53:41 +0200 Subject: [PATCH 120/162] fix indexes not resetting in BaseSearchQueryTestCase --- test_haystack/test_query.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_haystack/test_query.py b/test_haystack/test_query.py index f7e9a1707..c66d38427 100644 --- a/test_haystack/test_query.py +++ b/test_haystack/test_query.py @@ -95,6 +95,12 @@ def test_simple_nesting(self): class BaseSearchQueryTestCase(TestCase): fixtures = ["base_data.json", "bulk_data.json"] + @classmethod + def setUpClass(cls): + for connection in connections.all(): + connection.get_unified_index().reset() + super().setUpClass() + def setUp(self): super().setUp() self.bsq = BaseSearchQuery() From dee36ee0f6411edf0b2ed3db1facc92295e1e537 Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:53:53 +0200 Subject: [PATCH 121/162] fix indexes not resetting in test_spatial --- test_haystack/spatial/test_spatial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_haystack/spatial/test_spatial.py b/test_haystack/spatial/test_spatial.py index 8218f9bf8..6d0fbc12a 100644 --- a/test_haystack/spatial/test_spatial.py +++ b/test_haystack/spatial/test_spatial.py @@ -106,6 +106,7 @@ def setUp(self): super().setUp() self.ui = connections[self.using].get_unified_index() + self.ui.reset() self.checkindex = self.ui.get_index(Checkin) self.checkindex.reindex(using=self.using) self.sqs = SearchQuerySet().using(self.using) From 66e5cc2762c6129bea144ef85ba6035ed3c92dd4 Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:54:03 +0200 Subject: [PATCH 122/162] fix newlines in test_solr_backend --- test_haystack/solr_tests/test_solr_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index d8c95d329..cc0ad551a 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -420,7 +420,7 @@ def test_search(self): "results" ] ], - ["Indexed!\n1", "Indexed!\n2", "Indexed!\n3"], + ["Indexed!\n1\n", "Indexed!\n2\n", "Indexed!\n3\n"], ) # full-form highlighting options From 348cc845c8f1f6d3e5061a484a974723613b0efb Mon Sep 17 00:00:00 2001 From: Roman Hudec Date: Sun, 28 Apr 2024 22:54:21 +0200 Subject: [PATCH 123/162] fix better mocking in test_solr_management_commands --- .../test_solr_management_commands.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 32a3d6608..86ae4e850 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -1,5 +1,7 @@ import datetime import os +import shutil +import tempfile from io import StringIO from tempfile import mkdtemp from unittest.mock import patch @@ -218,6 +220,9 @@ def test_build_schema(self): oldui = connections["solr"].get_unified_index() oldurl = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + conf_dir = tempfile.mkdtemp() + with open(os.path.join(conf_dir, 'managed-schema'), 'w+') as fp: + pass try: needle = "Th3S3cr3tK3y" constants.DOCUMENT_FIELD = ( @@ -234,10 +239,6 @@ def test_build_schema(self): rendered_file = StringIO() - script_dir = os.path.realpath(os.path.dirname(__file__)) - conf_dir = os.path.join( - script_dir, "server", "solr", "server", "solr", "mgmnt", "conf" - ) schema_file = os.path.join(conf_dir, "schema.xml") solrconfig_file = os.path.join(conf_dir, "solrconfig.xml") @@ -261,16 +262,19 @@ def test_build_schema(self): os.path.isfile(os.path.join(conf_dir, "managed-schema.old")) ) - call_command("build_solr_schema", using="solr", reload_core=True) + with patch('haystack.management.commands.build_solr_schema.requests.get') as mock_request: + call_command("build_solr_schema", using="solr", reload_core=True) - os.rename(schema_file, "%s.bak" % schema_file) - self.assertRaises( - CommandError, - call_command, - "build_solr_schema", - using="solr", - reload_core=True, - ) + with patch('haystack.management.commands.build_solr_schema.requests.get') as mock_request: + mock_request.return_value.ok = False + + self.assertRaises( + CommandError, + call_command, + "build_solr_schema", + using="solr", + reload_core=True, + ) call_command("build_solr_schema", using="solr", filename=schema_file) with open(schema_file) as s: @@ -280,6 +284,7 @@ def test_build_schema(self): constants.DOCUMENT_FIELD = oldhdf connections["solr"]._index = oldui settings.HAYSTACK_CONNECTIONS["solr"]["URL"] = oldurl + shutil.rmtree(conf_dir, ignore_errors=True) class AppModelManagementCommandTestCase(TestCase): From a1ef180107ec5df0e02882110883edae0e74b849 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 28 Apr 2024 22:25:11 +0000 Subject: [PATCH 124/162] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- haystack/admin.py | 2 +- .../solr_tests/test_solr_management_commands.py | 10 +++++++--- test_haystack/test_management_commands.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/haystack/admin.py b/haystack/admin.py index 590b240d5..3f2fd0c19 100644 --- a/haystack/admin.py +++ b/haystack/admin.py @@ -18,7 +18,7 @@ def __init__(self, **kwargs): self.haystack_connection = kwargs.pop("haystack_connection", DEFAULT_ALIAS) super_kwargs = kwargs if django_version[0] >= 4: - super_kwargs['search_help_text'] = 'Search...' + super_kwargs["search_help_text"] = "Search..." super().__init__(**super_kwargs) def get_results(self, request): diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 86ae4e850..419d21b6d 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -221,7 +221,7 @@ def test_build_schema(self): oldurl = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] conf_dir = tempfile.mkdtemp() - with open(os.path.join(conf_dir, 'managed-schema'), 'w+') as fp: + with open(os.path.join(conf_dir, "managed-schema"), "w+") as fp: pass try: needle = "Th3S3cr3tK3y" @@ -262,10 +262,14 @@ def test_build_schema(self): os.path.isfile(os.path.join(conf_dir, "managed-schema.old")) ) - with patch('haystack.management.commands.build_solr_schema.requests.get') as mock_request: + with patch( + "haystack.management.commands.build_solr_schema.requests.get" + ) as mock_request: call_command("build_solr_schema", using="solr", reload_core=True) - with patch('haystack.management.commands.build_solr_schema.requests.get') as mock_request: + with patch( + "haystack.management.commands.build_solr_schema.requests.get" + ) as mock_request: mock_request.return_value.ok = False self.assertRaises( diff --git a/test_haystack/test_management_commands.py b/test_haystack/test_management_commands.py index d82a44aff..b66faf38f 100644 --- a/test_haystack/test_management_commands.py +++ b/test_haystack/test_management_commands.py @@ -77,8 +77,8 @@ def test_rebuild_index(self, mock_handle_clear, mock_handle_update): self.assertTrue(mock_handle_clear.called) self.assertTrue(mock_handle_update.called) - @patch("haystack.management.commands.update_index.Command.handle", return_value='') - @patch("haystack.management.commands.clear_index.Command.handle", return_value='') + @patch("haystack.management.commands.update_index.Command.handle", return_value="") + @patch("haystack.management.commands.clear_index.Command.handle", return_value="") def test_rebuild_index_nocommit(self, *mocks): call_command("rebuild_index", interactive=False, commit=False) From 8afb7399d09459a9e4aebd208ac76aeacf11f15e Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 29 Apr 2024 22:01:42 +0200 Subject: [PATCH 125/162] PEP 621: Migrate from setup.py and setup.cfg to pyproject.toml --- .github/workflows/test.yml | 6 +-- pyproject.toml | 83 ++++++++++++++++++++++++++++++++++---- setup.cfg | 12 ------ setup.py | 68 ------------------------------- 4 files changed, 79 insertions(+), 90 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bd4d8a2f..b2292f84e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,8 +20,8 @@ jobs: strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - django-version: ["3.2", "4.2", "5.0"] - python-version: ["3.8", "3.9"] # , "3.10", "3.11", "3.12"] # Whoosh issues with Py3.10+ + django-version: ["3.2", "4.2"] # , "5.0"] + python-version: ["3.8", "3.9", "3.10", "3.11"] # , "3.12" # Whoosh issues with Py3.10+ elastic-version: ["7.17.9"] exclude: - django-version: "3.2" @@ -63,7 +63,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install coverage requests tox tox-gh-actions pip install django==${{ matrix.django-version }} elasticsearch==${{ matrix.elastic-version }} - python setup.py clean build install + pip install --editable . - name: Run test run: tox -v env: diff --git a/pyproject.toml b/pyproject.toml index b2467d40b..da82ce895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,87 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" +requires = [ + "setuptools>=61.2", + "setuptools_scm[toml]>=3.4", + "wheel", +] -[tool.black] -line_length=88 +[project] +name = "django-haystack" +description = "Pluggable search for Django." +readme = "README.rst" +authors = [{name = "Daniel Lindsley", email = "daniel@toastdriven.com"}] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Django", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.2", + "Framework :: Django :: 5.0", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Utilities", +] +dynamic = [ + "version", +] +dependencies = [ + "Django>=3.2", + "packaging", +] +[project.optional-dependencies] +elasticsearch = [ + "elasticsearch<8,>=5", +] +testing = [ + "coverage", + "geopy==2", + "pysolr>=3.7", + "python-dateutil", + "requests", + "whoosh<3.0,>=2.5.4", +] +[project.urls] +Documentation = "https://django-haystack.readthedocs.io" +Homepage = "http://haystacksearch.org/" +Source = "https://github.com/django-haystack/django-haystack" -[tool.isort] -known_first_party = ["haystack", "test_haystack"] -profile = "black" -multi_line_output = 3 +[tool.setuptools] +packages = [ + "haystack", + "haystack.backends", + "haystack.management", + "haystack.management.commands", + "haystack.templatetags", + "haystack.utils", +] +include-package-data = false +# test-suite = "test_haystack.run_tests.run_all" # validate-pyproject-toml will complain +zip-safe = false + +[tool.setuptools.package-data] +haystack = [ + "templates/panels/*", + "templates/search_configuration/*", +] [tool.setuptools_scm] fallback_version = "0.0.dev0" write_to = "haystack/version.py" +[tool.isort] +known_first_party = ["haystack", "test_haystack"] +profile = "black" +multi_line_output = 3 + [tool.ruff] exclude = ["test_haystack"] ignore = ["B018", "B028", "B904", "B905"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index bae09868b..000000000 --- a/setup.cfg +++ /dev/null @@ -1,12 +0,0 @@ -[pep8] -line_length=88 -exclude=docs - -[flake8] -line_length=88 -exclude=docs,tests -ignore=E203, E501, W503, D - -[options] -setup_requires = - setuptools_scm diff --git a/setup.py b/setup.py deleted file mode 100644 index 70b029272..000000000 --- a/setup.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -install_requires = [ - "Django>=3.2", - "packaging", - "setuptools", -] - -tests_require = [ - "pysolr>=3.7.0", - "whoosh>=2.5.4,<3.0", - "python-dateutil", - "geopy==2.0.0", - "coverage", - "requests", -] - -setup( - name="django-haystack", - use_scm_version=True, - description="Pluggable search for Django.", - author="Daniel Lindsley", - author_email="daniel@toastdriven.com", - long_description=open("README.rst", "r").read(), - url="http://haystacksearch.org/", - project_urls={ - "Documentation": "https://django-haystack.readthedocs.io", - "Source": "https://github.com/django-haystack/django-haystack", - }, - packages=[ - "haystack", - "haystack.backends", - "haystack.management", - "haystack.management.commands", - "haystack.templatetags", - "haystack.utils", - ], - package_data={ - "haystack": ["templates/panels/*", "templates/search_configuration/*"] - }, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Web Environment", - "Framework :: Django", - "Framework :: Django :: 3.2", - "Framework :: Django :: 4.2", - "Framework :: Django :: 5.0", - "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Utilities", - ], - zip_safe=False, - install_requires=install_requires, - tests_require=tests_require, - extras_require={ - "elasticsearch": ["elasticsearch>=5,<8"], - }, - test_suite="test_haystack.run_tests.run_all", -) From 1201c259b3d3a408816956d8fc9f081eb96e92bb Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 29 Apr 2024 22:04:11 +0200 Subject: [PATCH 126/162] Fixed whoosh test failures with Django 5.0 --- .github/workflows/test.yml | 2 +- test_haystack/whoosh_tests/test_whoosh_backend.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2292f84e..35998d65f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - django-version: ["3.2", "4.2"] # , "5.0"] + django-version: ["3.2", "4.2", "5.0"] python-version: ["3.8", "3.9", "3.10", "3.11"] # , "3.12" # Whoosh issues with Py3.10+ elastic-version: ["7.17.9"] exclude: diff --git a/test_haystack/whoosh_tests/test_whoosh_backend.py b/test_haystack/whoosh_tests/test_whoosh_backend.py index 46fe88271..5de276b5e 100644 --- a/test_haystack/whoosh_tests/test_whoosh_backend.py +++ b/test_haystack/whoosh_tests/test_whoosh_backend.py @@ -114,6 +114,7 @@ def get_model(self): return MockModel +@override_settings(USE_TZ=False) class WhooshSearchBackendTestCase(WhooshTestCase): fixtures = ["bulk_data.json"] From b1f88d41bddcc950eeb7ded5d2620f462e5638a2 Mon Sep 17 00:00:00 2001 From: Tishil Joppan <125431548+tishiljk3@users.noreply.github.com> Date: Tue, 30 Apr 2024 02:30:44 +0530 Subject: [PATCH 127/162] Optimise code by removing unwanted else clause Co-authored-by: rahulrameshan --- haystack/backends/whoosh_backend.py | 3 +-- haystack/generic_views.py | 3 +-- haystack/query.py | 6 ++---- test_haystack/test_indexes.py | 3 +-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 5cec91f45..13d68035c 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -930,8 +930,7 @@ class WhooshSearchQuery(BaseSearchQuery): def _convert_datetime(self, date): if hasattr(date, "hour"): return force_str(date.strftime("%Y%m%d%H%M%S")) - else: - return force_str(date.strftime("%Y%m%d000000")) + return force_str(date.strftime("%Y%m%d000000")) def clean(self, query_fragment): """ diff --git a/haystack/generic_views.py b/haystack/generic_views.py index 2b981a4d1..655ea4f74 100644 --- a/haystack/generic_views.py +++ b/haystack/generic_views.py @@ -128,8 +128,7 @@ def get(self, request, *args, **kwargs): if form.is_valid(): return self.form_valid(form) - else: - return self.form_invalid(form) + return self.form_invalid(form) class FacetedSearchView(FacetedSearchMixin, SearchView): diff --git a/haystack/query.py b/haystack/query.py index 382e5682f..a3cf9490c 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -313,8 +313,7 @@ def __getitem__(self, k): # Cache should be full enough for our needs. if is_slice: return self._result_cache[start:bound] - else: - return self._result_cache[start] + return self._result_cache[start] # Methods that return a SearchQuerySet. def all(self): # noqa A003 @@ -329,8 +328,7 @@ def filter(self, *args, **kwargs): # noqa A003 """Narrows the search based on certain attributes and the default operator.""" if DEFAULT_OPERATOR == "OR": return self.filter_or(*args, **kwargs) - else: - return self.filter_and(*args, **kwargs) + return self.filter_and(*args, **kwargs) def exclude(self, *args, **kwargs): """Narrows the search by ensuring certain attributes are not included.""" diff --git a/test_haystack/test_indexes.py b/test_haystack/test_indexes.py index 19481ea51..6e6ee2d2d 100644 --- a/test_haystack/test_indexes.py +++ b/test_haystack/test_indexes.py @@ -687,8 +687,7 @@ class Meta: def get_index_fieldname(self, f): if f.name == "author": return "author_bar" - else: - return f.name + return f.name class YetAnotherBasicModelSearchIndex(indexes.ModelSearchIndex, indexes.Indexable): From 6264b537d3755f056bc644ea3da23e7624f65edc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 00:53:56 +0200 Subject: [PATCH 128/162] [pre-commit.ci] pre-commit autoupdate (#1964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.1 → v0.4.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.1...v0.4.2) - [github.com/psf/black: 24.4.0 → 24.4.2](https://github.com/psf/black/compare/24.4.0...24.4.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6423c61f0..aff476ef4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.1 + rev: v0.4.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -11,7 +11,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks From 095a4a2cf840f2905fe020875c0867b7b79f10be Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 30 Apr 2024 08:24:55 +0200 Subject: [PATCH 129/162] django-upgrade --target-version=5.0 **/*.py (#1939) * django-upgrade --target-version=5.0 **/*.py * djLint: Use HTTPS for external links. --- .pre-commit-config.yaml | 8 ++++++++ docs/haystack_theme/layout.html | 2 +- example_project/regular_app/models.py | 2 +- haystack/__init__.py | 7 ------- test_haystack/core/admin.py | 4 +--- test_haystack/test_app_using_appconfig/__init__.py | 1 - 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aff476ef4..5f6e6378b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,11 @@ exclude: ".*/vendor/.*" repos: + - repo: https://github.com/adamchainz/django-upgrade + rev: 1.15.0 + hooks: + - id: django-upgrade + args: [--target-version, "5.0"] # Replace with Django version + - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.4.2 hooks: @@ -10,10 +16,12 @@ repos: rev: 5.13.2 hooks: - id: isort + - repo: https://github.com/psf/black rev: 24.4.2 hooks: - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: diff --git a/docs/haystack_theme/layout.html b/docs/haystack_theme/layout.html index b342cb597..2cf423bf3 100644 --- a/docs/haystack_theme/layout.html +++ b/docs/haystack_theme/layout.html @@ -1,7 +1,7 @@ {% extends "basic/layout.html" %} {%- block extrahead %} - + {% endblock %} diff --git a/example_project/regular_app/models.py b/example_project/regular_app/models.py index e1a075e69..854ab2c26 100644 --- a/example_project/regular_app/models.py +++ b/example_project/regular_app/models.py @@ -36,7 +36,7 @@ def full_name(self): class Toy(models.Model): - dog = models.ForeignKey(Dog, related_name="toys") + dog = models.ForeignKey(Dog, on_delete=models.CASCADE, related_name="toys") name = models.CharField(max_length=60) def __str__(self): diff --git a/haystack/__init__.py b/haystack/__init__.py index 25448de96..4f427573d 100644 --- a/haystack/__init__.py +++ b/haystack/__init__.py @@ -1,6 +1,5 @@ from importlib.metadata import PackageNotFoundError, version -import django from django.conf import settings from django.core.exceptions import ImproperlyConfigured from packaging.version import Version @@ -17,12 +16,6 @@ __version__ = "0.0.dev0" version_info = Version(__version__) - -if django.VERSION < (3, 2): - # default_app_config is deprecated since django 3.2. - default_app_config = "haystack.apps.HaystackConfig" - - # Help people clean up from 1.X. if hasattr(settings, "HAYSTACK_SITECONF"): raise ImproperlyConfigured( diff --git a/test_haystack/core/admin.py b/test_haystack/core/admin.py index 3e374bc6b..404dbefbe 100644 --- a/test_haystack/core/admin.py +++ b/test_haystack/core/admin.py @@ -5,10 +5,8 @@ from .models import MockModel +@admin.register(MockModel) class MockModelAdmin(SearchModelAdmin): haystack_connection = "solr" date_hierarchy = "pub_date" list_display = ("author", "pub_date") - - -admin.site.register(MockModel, MockModelAdmin) diff --git a/test_haystack/test_app_using_appconfig/__init__.py b/test_haystack/test_app_using_appconfig/__init__.py index 30a0d2351..e69de29bb 100644 --- a/test_haystack/test_app_using_appconfig/__init__.py +++ b/test_haystack/test_app_using_appconfig/__init__.py @@ -1 +0,0 @@ -default_app_config = "test_app_using_appconfig.apps.SimpleTestAppConfig" From c83a28129f4f5a628a757e992bbd356b63e49ab9 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 30 Apr 2024 11:55:27 +0200 Subject: [PATCH 130/162] Add `setuptools` to fix failing tests on Python 3.12 --- .github/workflows/test.yml | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35998d65f..257b6a42b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django django-version: ["3.2", "4.2", "5.0"] - python-version: ["3.8", "3.9", "3.10", "3.11"] # , "3.12" # Whoosh issues with Py3.10+ + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] elastic-version: ["7.17.9"] exclude: - django-version: "3.2" diff --git a/tox.ini b/tox.ini index 498bb7f00..d5a436091 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ deps = geopy==2.0.0 coverage requests + setuptools; python_version >= "3.12" # Can be removed on pysolr >= v3.10 django3.2: Django>=3.2,<3.3 django4.2: Django>=4.2,<4.3 django5.0: Django>=5.0,<5.1 From a7fcc250d46605e65687c02aa9cb8e6e077e51aa Mon Sep 17 00:00:00 2001 From: JochenGCD Date: Sun, 5 May 2024 18:57:29 +0200 Subject: [PATCH 131/162] use class variables instead of global variables for field mappings (#1965) --- haystack/backends/elasticsearch7_backend.py | 50 ++++++++++----------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/haystack/backends/elasticsearch7_backend.py b/haystack/backends/elasticsearch7_backend.py index 161a9038a..f8d7d767e 100644 --- a/haystack/backends/elasticsearch7_backend.py +++ b/haystack/backends/elasticsearch7_backend.py @@ -27,29 +27,6 @@ ) -DEFAULT_FIELD_MAPPING = { - "type": "text", - "analyzer": "snowball", -} -FIELD_MAPPINGS = { - "edge_ngram": { - "type": "text", - "analyzer": "edgengram_analyzer", - }, - "ngram": { - "type": "text", - "analyzer": "ngram_analyzer", - }, - "date": {"type": "date"}, - "datetime": {"type": "date"}, - "location": {"type": "geo_point"}, - "boolean": {"type": "boolean"}, - "float": {"type": "float"}, - "long": {"type": "long"}, - "integer": {"type": "long"}, -} - - class Elasticsearch7SearchBackend(ElasticsearchSearchBackend): # Settings to add an n-gram & edge n-gram analyzer. DEFAULT_SETTINGS = { @@ -90,6 +67,29 @@ class Elasticsearch7SearchBackend(ElasticsearchSearchBackend): }, } + DEFAULT_FIELD_MAPPING = { + "type": "text", + "analyzer": "snowball", + } + + FIELD_MAPPINGS = { + "edge_ngram": { + "type": "text", + "analyzer": "edgengram_analyzer", + }, + "ngram": { + "type": "text", + "analyzer": "ngram_analyzer", + }, + "date": {"type": "date"}, + "datetime": {"type": "date"}, + "location": {"type": "geo_point"}, + "boolean": {"type": "boolean"}, + "float": {"type": "float"}, + "long": {"type": "long"}, + "integer": {"type": "long"}, + } + def __init__(self, connection_alias, **connection_options): super().__init__(connection_alias, **connection_options) self.content_field_name = None @@ -550,8 +550,8 @@ def build_schema(self, fields): mapping = self._get_common_mapping() for _, field_class in fields.items(): - field_mapping = FIELD_MAPPINGS.get( - field_class.field_type, DEFAULT_FIELD_MAPPING + field_mapping = self.FIELD_MAPPINGS.get( + field_class.field_type, self.DEFAULT_FIELD_MAPPING ).copy() if field_class.boost != 1.0: field_mapping["boost"] = field_class.boost From 0ec5afee14d7ead69d84758111de184edb827520 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 04:40:14 +0200 Subject: [PATCH 132/162] [pre-commit.ci] pre-commit autoupdate (#1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.15.0 → 1.16.0](https://github.com/adamchainz/django-upgrade/compare/1.15.0...1.16.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.2 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.2...v0.4.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f6e6378b..404b84fad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.15.0 + rev: 1.16.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.2 + rev: v0.4.3 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 443f05606af070b6f3e4d87e6da00c1a54841d7b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 22:16:10 +0000 Subject: [PATCH 133/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.16.0 → 1.17.0](https://github.com/adamchainz/django-upgrade/compare/1.16.0...1.17.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.4) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 404b84fad..d8e4e71f2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.16.0 + rev: 1.17.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.3 + rev: v0.4.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 1de3b9b853d840c555ee0a6d1e18fa459a2d77ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 05:19:00 -0400 Subject: [PATCH 134/162] [pre-commit.ci] pre-commit autoupdate (#1969) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.4 → v0.4.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.4...v0.4.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8e4e71f2..43dfa3969 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.4.5 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5074cd5bd454e2dcb3a7110e2a66900da008e248 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:43:11 -0300 Subject: [PATCH 135/162] [pre-commit.ci] pre-commit autoupdate (#1970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.17.0 → 1.18.0](https://github.com/adamchainz/django-upgrade/compare/1.17.0...1.18.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.5 → v0.4.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.5...v0.4.7) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43dfa3969..488a6b6a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.17.0 + rev: 1.18.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.5 + rev: v0.4.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 76ec688d53e0e05f61c72fe088d35e53d6fdc1ff Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Tue, 4 Jun 2024 10:44:10 -0400 Subject: [PATCH 136/162] Basic doc updates --- README.rst | 19 ++++--------------- docs/index.rst | 4 ++-- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index e573494f2..f8d07d548 100644 --- a/README.rst +++ b/README.rst @@ -48,9 +48,8 @@ Documentation ============= * Development version: http://docs.haystacksearch.org/ -* v2.8.X: https://django-haystack.readthedocs.io/en/v2.8.1/ -* v2.7.X: https://django-haystack.readthedocs.io/en/v2.7.0/ -* v2.6.X: https://django-haystack.readthedocs.io/en/v2.6.0/ +* v3.3.0: https://django-haystack.readthedocs.io/en/v3.3.0/ +* v2.8.1: https://django-haystack.readthedocs.io/en/v2.8.1/ See the `changelog `_ @@ -59,19 +58,9 @@ Requirements Haystack has a relatively easily-met set of requirements. -* A supported version of Python: https://devguide.python.org/versions/#supported-versions -* A supported version of Django: https://www.djangoproject.com/download/#supported-versions +* Python 3.8+ +* Django 3-5 Additionally, each backend has its own requirements. You should refer to https://django-haystack.readthedocs.io/en/latest/installing_search_engines.html for more details. - -Experimental support for Django v5.0 -==================================== - -The current release on PyPI_ does not yet support Django v5.0. - -.. _PyPI: https://pypi.org/project/django-haystack/ - -To run on Django v5.0, please install by using: -``pip install git+https://github.com/django-haystack/django-haystack.git`` diff --git a/docs/index.rst b/docs/index.rst index f5267e9c2..7c3a96e10 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -116,8 +116,8 @@ Requirements Haystack has a relatively easily-met set of requirements. -* Python 2.7+ or Python 3.3+ -* A supported version of Django: https://www.djangoproject.com/download/#supported-versions +* Python 3.8+ +* Django 3-5 Additionally, each backend has its own requirements. You should refer to :doc:`installing_search_engines` for more details. From 4d1fce01251f9d1e52ecb3d7fd05c789854d5322 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:50:04 -0400 Subject: [PATCH 137/162] [pre-commit.ci] pre-commit autoupdate (#1973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.7 → v0.4.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.7...v0.4.8) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 488a6b6a8..8a0423407 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.4.8 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5c0738e0f55628095e0da47dd176502dd1b0c077 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:39:39 -0400 Subject: [PATCH 138/162] [pre-commit.ci] pre-commit autoupdate (#1974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.8 → v0.4.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.8...v0.4.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a0423407..2af25d37b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.4.9 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 161a09137d54efbfab03b6a281b911aac15bc250 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 08:01:47 +0200 Subject: [PATCH 139/162] [pre-commit.ci] pre-commit autoupdate (#1975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.9 → v0.4.10](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.9...v0.4.10) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2af25d37b..4972ed0ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 + rev: v0.4.10 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 2fd1360566f1c9c8c40f2e6020a9c4830001e601 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 29 Jun 2024 14:14:42 +0200 Subject: [PATCH 140/162] Update to modern ruff syntax --- .github/workflows/test.yml | 2 +- haystack/views.py | 1 - pyproject.toml | 15 ++++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 257b6a42b..4dec7412f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: pip install --user ruff - - run: ruff --output-format=github + - run: ruff check --output-format=github test: runs-on: ubuntu-latest diff --git a/haystack/views.py b/haystack/views.py index fed1808ea..f203f5e3a 100644 --- a/haystack/views.py +++ b/haystack/views.py @@ -11,7 +11,6 @@ class SearchView: template = "search/search.html" - extra_context = {} query = "" results = EmptySearchQuerySet() request = None diff --git a/pyproject.toml b/pyproject.toml index da82ce895..5962dae5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,20 +83,21 @@ profile = "black" multi_line_output = 3 [tool.ruff] -exclude = ["test_haystack"] -ignore = ["B018", "B028", "B904", "B905"] +extend-exclude = ["test_haystack/*"] line-length = 162 -select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] -show-source = true target-version = "py38" -[tool.ruff.isort] +[tool.ruff.lint] +ignore = ["B018", "B028", "B904", "B905"] +select = ["ASYNC", "B", "C4", "DJ", "E", "F", "G", "PLR091", "W"] + +[tool.ruff.lint.isort] known-first-party = ["haystack", "test_haystack"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 14 -[tool.ruff.pylint] +[tool.ruff.lint.pylint] max-args = 20 max-branches = 40 max-returns = 8 From 1189b8d4ee5b0dea37cca6c0fd0ce381e006244b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 29 Jun 2024 14:35:32 +0200 Subject: [PATCH 141/162] Remove obsolete ElasticSearch2 support and tests --- docs/installing_search_engines.rst | 2 +- docs/tutorial.rst | 20 - haystack/backends/elasticsearch2_backend.py | 384 ---- .../elasticsearch2_tests/__init__.py | 35 - .../elasticsearch2_tests/test_backend.py | 1820 ----------------- .../elasticsearch2_tests/test_inputs.py | 85 - .../elasticsearch2_tests/test_query.py | 247 --- test_haystack/settings.py | 8 +- 8 files changed, 2 insertions(+), 2599 deletions(-) delete mode 100644 haystack/backends/elasticsearch2_backend.py delete mode 100644 test_haystack/elasticsearch2_tests/__init__.py delete mode 100644 test_haystack/elasticsearch2_tests/test_backend.py delete mode 100644 test_haystack/elasticsearch2_tests/test_inputs.py delete mode 100644 test_haystack/elasticsearch2_tests/test_query.py diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 50bd6fb06..8b4157dcb 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -153,7 +153,7 @@ Elasticsearch is similar to Solr — another Java application using Lucene — b focused on ease of deployment and clustering. See https://www.elastic.co/products/elasticsearch for more information. -Haystack currently supports Elasticsearch 1.x, 2.x, 5.x, and 7.x. +Haystack currently supports Elasticsearch 5.x and 7.x. Follow the instructions on https://www.elastic.co/downloads/elasticsearch to download and install Elasticsearch and configure it for your environment. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index d3228beea..d8886c58a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -141,26 +141,6 @@ Example (Solr 6.X):: Elasticsearch ~~~~~~~~~~~~~ -Example (ElasticSearch 1.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - -Example (ElasticSearch 2.x):: - - HAYSTACK_CONNECTIONS = { - 'default': { - 'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine', - 'URL': 'http://127.0.0.1:9200/', - 'INDEX_NAME': 'haystack', - }, - } - Example (ElasticSearch 5.x):: HAYSTACK_CONNECTIONS = { diff --git a/haystack/backends/elasticsearch2_backend.py b/haystack/backends/elasticsearch2_backend.py deleted file mode 100644 index ce744107f..000000000 --- a/haystack/backends/elasticsearch2_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -import datetime -import warnings - -from django.conf import settings - -from haystack.backends import BaseEngine -from haystack.backends.elasticsearch_backend import ( - ElasticsearchSearchBackend, - ElasticsearchSearchQuery, -) -from haystack.constants import DJANGO_CT -from haystack.exceptions import MissingDependency -from haystack.utils import get_identifier, get_model_ct - -try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch.helpers import bulk, scan - - warnings.warn( - "ElasticSearch 2.x support deprecated, will be removed in 4.0", - DeprecationWarning, - ) -except ImportError: - raise MissingDependency( - "The 'elasticsearch2' backend requires the \ - installation of 'elasticsearch>=2.0.0,<3.0.0'. \ - Please refer to the documentation." - ) - - -class Elasticsearch2SearchBackend(ElasticsearchSearchBackend): - def __init__(self, connection_alias, **connection_options): - super().__init__(connection_alias, **connection_options) - self.content_field_name = None - - def clear(self, models=None, commit=True): - """ - Clears the backend of all documents/objects for a collection of models. - - :param models: List or tuple of models to clear. - :param commit: Not used. - """ - if models is not None: - assert isinstance(models, (list, tuple)) - - try: - if models is None: - self.conn.indices.delete(index=self.index_name, ignore=404) - self.setup_complete = False - self.existing_mapping = {} - self.content_field_name = None - else: - models_to_delete = [] - - for model in models: - models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model))) - - # Delete using scroll API - query = { - "query": {"query_string": {"query": " OR ".join(models_to_delete)}} - } - generator = scan( - self.conn, - query=query, - index=self.index_name, - **self._get_doc_type_option(), - ) - actions = ( - {"_op_type": "delete", "_id": doc["_id"]} for doc in generator - ) - bulk( - self.conn, - actions=actions, - index=self.index_name, - **self._get_doc_type_option(), - ) - self.conn.indices.refresh(index=self.index_name) - - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - if models is not None: - self.log.exception( - "Failed to clear Elasticsearch index of models '%s'", - ",".join(models_to_delete), - ) - else: - self.log.exception("Failed to clear Elasticsearch index") - - def build_search_kwargs( - self, - query_string, - sort_by=None, - start_offset=0, - end_offset=None, - fields="", - highlight=False, - facets=None, - date_facets=None, - query_facets=None, - narrow_queries=None, - spelling_query=None, - within=None, - dwithin=None, - distance_point=None, - models=None, - limit_to_registered_models=None, - result_class=None, - ): - kwargs = super().build_search_kwargs( - query_string, - sort_by, - start_offset, - end_offset, - fields, - highlight, - spelling_query=spelling_query, - within=within, - dwithin=dwithin, - distance_point=distance_point, - models=models, - limit_to_registered_models=limit_to_registered_models, - result_class=result_class, - ) - - filters = [] - if start_offset is not None: - kwargs["from"] = start_offset - - if end_offset is not None: - kwargs["size"] = end_offset - start_offset - - if narrow_queries is None: - narrow_queries = set() - - if facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, extra_options in facets.items(): - facet_options = { - "meta": {"_type": "terms"}, - "terms": {"field": facet_fieldname}, - } - if "order" in extra_options: - facet_options["meta"]["order"] = extra_options.pop("order") - # Special cases for options applied at the facet level (not the terms level). - if extra_options.pop("global_scope", False): - # Renamed "global_scope" since "global" is a python keyword. - facet_options["global"] = True - if "facet_filter" in extra_options: - facet_options["facet_filter"] = extra_options.pop("facet_filter") - facet_options["terms"].update(extra_options) - kwargs["aggs"][facet_fieldname] = facet_options - - if date_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in date_facets.items(): - # Need to detect on gap_by & only add amount if it's more than one. - interval = value.get("gap_by").lower() - - # Need to detect on amount (can't be applied on months or years). - if value.get("gap_amount", 1) != 1 and interval not in ( - "month", - "year", - ): - # Just the first character is valid for use. - interval = "%s%s" % (value["gap_amount"], interval[:1]) - - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "date_histogram"}, - "date_histogram": {"field": facet_fieldname, "interval": interval}, - "aggs": { - facet_fieldname: { - "date_range": { - "field": facet_fieldname, - "ranges": [ - { - "from": self._from_python( - value.get("start_date") - ), - "to": self._from_python(value.get("end_date")), - } - ], - } - } - }, - } - - if query_facets is not None: - kwargs.setdefault("aggs", {}) - - for facet_fieldname, value in query_facets: - kwargs["aggs"][facet_fieldname] = { - "meta": {"_type": "query"}, - "filter": {"query_string": {"query": value}}, - } - - for q in narrow_queries: - filters.append({"query_string": {"query": q}}) - - # if we want to filter, change the query type to filteres - if filters: - kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}} - filtered = kwargs["query"]["filtered"] - if "filter" in filtered: - if "bool" in filtered["filter"].keys(): - another_filters = kwargs["query"]["filtered"]["filter"]["bool"][ - "must" - ] - else: - another_filters = [kwargs["query"]["filtered"]["filter"]] - else: - another_filters = filters - - if len(another_filters) == 1: - kwargs["query"]["filtered"]["filter"] = another_filters[0] - else: - kwargs["query"]["filtered"]["filter"] = { - "bool": {"must": another_filters} - } - - return kwargs - - def more_like_this( - self, - model_instance, - additional_query_string=None, - start_offset=0, - end_offset=None, - models=None, - limit_to_registered_models=None, - result_class=None, - **kwargs - ): - from haystack import connections - - if not self.setup_complete: - self.setup() - - # Deferred models will have a different class ("RealClass_Deferred_fieldname") - # which won't be in our registry: - model_klass = model_instance._meta.concrete_model - - index = ( - connections[self.connection_alias] - .get_unified_index() - .get_index(model_klass) - ) - field_name = index.get_content_field() - params = {} - - if start_offset is not None: - params["from_"] = start_offset - - if end_offset is not None: - params["size"] = end_offset - start_offset - - doc_id = get_identifier(model_instance) - - try: - # More like this Query - # https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html - mlt_query = { - "query": { - "more_like_this": { - "fields": [field_name], - "like": [{"_id": doc_id}], - } - } - } - - narrow_queries = [] - - if additional_query_string and additional_query_string != "*:*": - additional_filter = { - "query": {"query_string": {"query": additional_query_string}} - } - narrow_queries.append(additional_filter) - - if limit_to_registered_models is None: - limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - - if models and len(models): - model_choices = sorted(get_model_ct(model) for model in models) - elif limit_to_registered_models: - # Using narrow queries, limit the results to only models handled - # with the current routers. - model_choices = self.build_models_list() - else: - model_choices = [] - - if len(model_choices) > 0: - model_filter = {"terms": {DJANGO_CT: model_choices}} - narrow_queries.append(model_filter) - - if len(narrow_queries) > 0: - mlt_query = { - "query": { - "filtered": { - "query": mlt_query["query"], - "filter": {"bool": {"must": list(narrow_queries)}}, - } - } - } - - raw_results = self.conn.search( - body=mlt_query, - index=self.index_name, - _source=True, - **self._get_doc_type_option(), - **params, - ) - except elasticsearch.TransportError: - if not self.silently_fail: - raise - - self.log.exception( - "Failed to fetch More Like This from Elasticsearch for document '%s'", - doc_id, - ) - raw_results = {} - - return self._process_results(raw_results, result_class=result_class) - - def _process_results( - self, - raw_results, - highlight=False, - result_class=None, - distance_point=None, - geo_sort=False, - ): - results = super()._process_results( - raw_results, highlight, result_class, distance_point, geo_sort - ) - facets = {} - if "aggregations" in raw_results: - facets = {"fields": {}, "dates": {}, "queries": {}} - - for facet_fieldname, facet_info in raw_results["aggregations"].items(): - facet_type = facet_info["meta"]["_type"] - if facet_type == "terms": - facets["fields"][facet_fieldname] = [ - (individual["key"], individual["doc_count"]) - for individual in facet_info["buckets"] - ] - if "order" in facet_info["meta"]: - if facet_info["meta"]["order"] == "reverse_count": - srt = sorted( - facets["fields"][facet_fieldname], key=lambda x: x[1] - ) - facets["fields"][facet_fieldname] = srt - elif facet_type == "date_histogram": - # Elasticsearch provides UTC timestamps with an extra three - # decimals of precision, which datetime barfs on. - facets["dates"][facet_fieldname] = [ - ( - datetime.datetime.utcfromtimestamp( - individual["key"] / 1000 - ), - individual["doc_count"], - ) - for individual in facet_info["buckets"] - ] - elif facet_type == "query": - facets["queries"][facet_fieldname] = facet_info["doc_count"] - results["facets"] = facets - return results - - -class Elasticsearch2SearchQuery(ElasticsearchSearchQuery): - pass - - -class Elasticsearch2SearchEngine(BaseEngine): - backend = Elasticsearch2SearchBackend - query = Elasticsearch2SearchQuery diff --git a/test_haystack/elasticsearch2_tests/__init__.py b/test_haystack/elasticsearch2_tests/__init__.py deleted file mode 100644 index 38fa24fbc..000000000 --- a/test_haystack/elasticsearch2_tests/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import unittest - -from django.conf import settings - -from haystack.utils import log as logging - - -def load_tests(loader, standard_tests, pattern): - log = logging.getLogger("haystack") - try: - import elasticsearch - - if not ((2, 0, 0) <= elasticsearch.__version__ < (3, 0, 0)): - raise ImportError - from elasticsearch import Elasticsearch, exceptions - except ImportError: - log.error( - "Skipping ElasticSearch 2 tests: 'elasticsearch>=2.0.0,<3.0.0' not installed." - ) - raise unittest.SkipTest("'elasticsearch>=2.0.0,<3.0.0' not installed.") - - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - es = Elasticsearch(url) - try: - es.info() - except exceptions.ConnectionError as e: - log.error("elasticsearch not running on %r" % url, exc_info=True) - raise unittest.SkipTest("elasticsearch not running on %r" % url, e) - - package_tests = loader.discover( - start_dir=os.path.dirname(__file__), pattern=pattern - ) - standard_tests.addTests(package_tests) - return standard_tests diff --git a/test_haystack/elasticsearch2_tests/test_backend.py b/test_haystack/elasticsearch2_tests/test_backend.py deleted file mode 100644 index 0ec9608b0..000000000 --- a/test_haystack/elasticsearch2_tests/test_backend.py +++ /dev/null @@ -1,1820 +0,0 @@ -import datetime -import logging as std_logging -import operator -import pickle -import unittest -from decimal import Decimal - -import elasticsearch -from django.apps import apps -from django.conf import settings -from django.test import TestCase -from django.test.utils import override_settings - -from haystack import connections, indexes, reset_search_queries -from haystack.exceptions import SkipDocument -from haystack.inputs import AutoQuery -from haystack.models import SearchResult -from haystack.query import SQ, RelatedSearchQuerySet, SearchQuerySet -from haystack.utils import log as logging -from haystack.utils.loading import UnifiedIndex - -from ..core.models import AFourthMockModel, AnotherMockModel, ASixthMockModel, MockModel -from ..mocks import MockSearchResult - - -def clear_elasticsearch_index(): - # Wipe it clean. - raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - try: - raw_es.indices.delete( - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - raw_es.indices.refresh() - except elasticsearch.TransportError: - pass - - # Since we've just completely deleted the index, we'll reset setup_complete so the next access will - # correctly define the mappings: - connections["elasticsearch"].get_backend().setup_complete = False - - -class Elasticsearch2MockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2MockSearchIndexWithSkipDocument(Elasticsearch2MockSearchIndex): - def prepare_text(self, obj): - if obj.author == "daniel3": - raise SkipDocument - return "Indexed!\n%s" % obj.id - - -class Elasticsearch2MockSpellingIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author", faceted=True) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - def prepare_text(self, obj): - return obj.foo - - -class Elasticsearch2MaintainTypeMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, use_template=True) - month = indexes.CharField(indexed=False) - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def prepare_month(self, obj): - return "%02d" % obj.pub_date.month - - def get_model(self): - return MockModel - - -class Elasticsearch2MockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return MockModel - - -class Elasticsearch2AnotherMockModelSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AnotherMockModel - - def prepare_text(self, obj): - return "You might be searching for the user %s" % obj.author - - -class Elasticsearch2BoostMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField( - document=True, - use_template=True, - template_name="search/indexes/core/mockmodel_template.txt", - ) - author = indexes.CharField(model_attr="author", weight=2.0) - editor = indexes.CharField(model_attr="editor") - pub_date = indexes.DateTimeField(model_attr="pub_date") - - def get_model(self): - return AFourthMockModel - - def prepare(self, obj): - data = super().prepare(obj) - - if obj.pk == 4: - data["boost"] = 5.0 - - return data - - -class Elasticsearch2FacetingMockSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True) - author = indexes.CharField(model_attr="author", faceted=True) - editor = indexes.CharField(model_attr="editor", faceted=True) - pub_date = indexes.DateField(model_attr="pub_date", faceted=True) - facet_field = indexes.FacetCharField(model_attr="author") - - def prepare_text(self, obj): - return "%s %s" % (obj.author, obj.editor) - - def get_model(self): - return AFourthMockModel - - -class Elasticsearch2RoundTripSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(document=True, default="") - name = indexes.CharField() - is_active = indexes.BooleanField() - post_count = indexes.IntegerField() - average_rating = indexes.FloatField() - price = indexes.DecimalField() - pub_date = indexes.DateField() - created = indexes.DateTimeField() - tags = indexes.MultiValueField() - sites = indexes.MultiValueField() - - def get_model(self): - return MockModel - - def prepare(self, obj): - prepped = super().prepare(obj) - prepped.update( - { - "text": "This is some example text.", - "name": "Mister Pants", - "is_active": True, - "post_count": 25, - "average_rating": 3.6, - "price": Decimal("24.99"), - "pub_date": datetime.date(2009, 11, 21), - "created": datetime.datetime(2009, 11, 21, 21, 31, 00), - "tags": ["staff", "outdoor", "activist", "scientist"], - "sites": [3, 5, 1], - } - ) - return prepped - - -class Elasticsearch2ComplexFacetsMockSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(document=True, default="") - name = indexes.CharField(faceted=True) - is_active = indexes.BooleanField(faceted=True) - post_count = indexes.IntegerField() - post_count_i = indexes.FacetIntegerField(facet_for="post_count") - average_rating = indexes.FloatField(faceted=True) - pub_date = indexes.DateField(faceted=True) - created = indexes.DateTimeField(faceted=True) - sites = indexes.MultiValueField(faceted=True) - - def get_model(self): - return MockModel - - -class Elasticsearch2AutocompleteMockModelSearchIndex( - indexes.SearchIndex, indexes.Indexable -): - text = indexes.CharField(model_attr="foo", document=True) - name = indexes.CharField(model_attr="author") - pub_date = indexes.DateTimeField(model_attr="pub_date") - text_auto = indexes.EdgeNgramField(model_attr="foo") - name_auto = indexes.EdgeNgramField(model_attr="author") - - def get_model(self): - return MockModel - - -class Elasticsearch2SpatialSearchIndex(indexes.SearchIndex, indexes.Indexable): - text = indexes.CharField(model_attr="name", document=True) - location = indexes.LocationField() - - def prepare_location(self, obj): - return "%s,%s" % (obj.lat, obj.lon) - - def get_model(self): - return ASixthMockModel - - -class TestSettings(TestCase): - def test_kwargs_are_passed_on(self): - from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend - - backend = ElasticsearchSearchBackend( - "alias", - **{ - "URL": settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"], - "INDEX_NAME": "testing", - "KWARGS": {"max_retries": 42}, - } - ) - - self.assertEqual(backend.conn.transport.max_retries, 42) - - -class Elasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.smmidni = Elasticsearch2MockSearchIndexWithSkipDocument() - self.smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - self.sb.silently_fail = True - - def raw_search(self, query): - try: - return self.raw_es.search( - q="*:*", - index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"], - ) - except elasticsearch.TransportError: - return {} - - def test_non_silent(self): - bad_sb = connections["elasticsearch"].backend( - "bad", - URL="http://omg.wtf.bbq:1000/", - INDEX_NAME="whatver", - SILENTLY_FAIL=False, - TIMEOUT=1, - ) - - try: - bad_sb.update(self.smmi, self.sample_objs) - self.fail() - except: - pass - - try: - bad_sb.remove("core.mockmodel.1") - self.fail() - except: - pass - - try: - bad_sb.clear() - self.fail() - except: - pass - - try: - bad_sb.search("foo") - self.fail() - except: - pass - - def test_update_no_documents(self): - url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - index_name = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=True - ) - self.assertEqual(sb.update(self.smmi, []), None) - - sb = connections["elasticsearch"].backend( - "elasticsearch", URL=url, INDEX_NAME=index_name, SILENTLY_FAIL=False - ) - try: - sb.update(self.smmi, []) - self.fail() - except: - pass - - def test_update(self): - self.sb.update(self.smmi, self.sample_objs) - - # Check what Elasticsearch thinks is there. - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=lambda x: x["id"], - ), - [ - { - "django_id": "1", - "django_ct": "core.mockmodel", - "name": "daniel1", - "name_exact": "daniel1", - "text": "Indexed!\n1\n", - "pub_date": "2009-02-24T00:00:00", - "id": "core.mockmodel.1", - }, - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_update_with_SkipDocument_raised(self): - self.sb.update(self.smmidni, self.sample_objs) - - # Check what Elasticsearch thinks is there. - res = self.raw_search("*:*")["hits"] - self.assertEqual(res["total"], 2) - self.assertListEqual( - sorted([x["_source"]["id"] for x in res["hits"]]), - ["core.mockmodel.1", "core.mockmodel.2"], - ) - - def test_remove(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 2) - self.assertEqual( - sorted( - [res["_source"] for res in self.raw_search("*:*")["hits"]["hits"]], - key=operator.itemgetter("django_id"), - ), - [ - { - "django_id": "2", - "django_ct": "core.mockmodel", - "name": "daniel2", - "name_exact": "daniel2", - "text": "Indexed!\n2\n", - "pub_date": "2009-02-23T00:00:00", - "id": "core.mockmodel.2", - }, - { - "django_id": "3", - "django_ct": "core.mockmodel", - "name": "daniel3", - "name_exact": "daniel3", - "text": "Indexed!\n3\n", - "pub_date": "2009-02-22T00:00:00", - "id": "core.mockmodel.3", - }, - ], - ) - - def test_remove_succeeds_on_404(self): - self.sb.silently_fail = False - self.sb.remove("core.mockmodel.421") - - def test_clear(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear() - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 3) - - self.sb.clear([AnotherMockModel, MockModel]) - self.assertEqual(self.raw_search("*:*").get("hits", {}).get("total", 0), 0) - - def test_search(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - set([result.pk for result in self.sb.search("*:*")["results"]]), - {"2", "1", "3"}, - ) - - self.assertEqual(self.sb.search("", highlight=True), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("Index", highlight=True)["hits"], 3) - self.assertEqual( - sorted( - [ - result.highlighted[0] - for result in self.sb.search("Index", highlight=True)["results"] - ] - ), - [ - "Indexed!\n1\n", - "Indexed!\n2\n", - "Indexed!\n3\n", - ], - ) - - self.assertEqual(self.sb.search("Indx")["hits"], 0) - self.assertEqual(self.sb.search("indaxed")["spelling_suggestion"], "indexed") - self.assertEqual( - self.sb.search("arf", spelling_query="indexyd")["spelling_suggestion"], - "indexed", - ) - - self.assertEqual( - self.sb.search("", facets={"name": {}}), {"hits": 0, "results": []} - ) - results = self.sb.search("Index", facets={"name": {}}) - self.assertEqual(results["hits"], 3) - self.assertSetEqual( - set(results["facets"]["fields"]["name"]), - {("daniel3", 1), ("daniel2", 1), ("daniel1", 1)}, - ) - - self.assertEqual( - self.sb.search( - "", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ), - {"hits": 0, "results": []}, - ) - results = self.sb.search( - "Index", - date_facets={ - "pub_date": { - "start_date": datetime.date(2008, 1, 1), - "end_date": datetime.date(2009, 4, 1), - "gap_by": "month", - "gap_amount": 1, - } - }, - ) - self.assertEqual(results["hits"], 3) - self.assertEqual( - results["facets"]["dates"]["pub_date"], - [(datetime.datetime(2009, 2, 1, 0, 0), 3)], - ) - - self.assertEqual( - self.sb.search("", query_facets=[("name", "[* TO e]")]), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", query_facets=[("name", "[* TO e]")]) - self.assertEqual(results["hits"], 3) - self.assertEqual(results["facets"]["queries"], {"name": 3}) - - self.assertEqual( - self.sb.search("", narrow_queries={"name:daniel1"}), - {"hits": 0, "results": []}, - ) - results = self.sb.search("Index", narrow_queries={"name:daniel1"}) - self.assertEqual(results["hits"], 1) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sb.search("index", result_class=MockSearchResult)["results"][0], - MockSearchResult, - ) - ) - - # Check the use of ``limit_to_registered_models``. - self.assertEqual( - self.sb.search("", limit_to_registered_models=False), - {"hits": 0, "results": []}, - ) - self.assertEqual( - self.sb.search("*:*", limit_to_registered_models=False)["hits"], 3 - ) - self.assertEqual( - sorted( - [ - result.pk - for result in self.sb.search( - "*:*", limit_to_registered_models=False - )["results"] - ] - ), - ["1", "2", "3"], - ) - - # Stow. - old_limit_to_registered_models = getattr( - settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True - ) - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = False - - self.assertEqual(self.sb.search(""), {"hits": 0, "results": []}) - self.assertEqual(self.sb.search("*:*")["hits"], 3) - self.assertEqual( - sorted([result.pk for result in self.sb.search("*:*")["results"]]), - ["1", "2", "3"], - ) - - # Restore. - settings.HAYSTACK_LIMIT_TO_REGISTERED_MODELS = old_limit_to_registered_models - - def test_spatial_search_parameters(self): - from django.contrib.gis.geos import Point - - p1 = Point(1.23, 4.56) - kwargs = self.sb.build_search_kwargs( - "*:*", - distance_point={"field": "location", "point": p1}, - sort_by=(("distance", "desc"),), - ) - - self.assertIn("sort", kwargs) - self.assertEqual(1, len(kwargs["sort"])) - geo_d = kwargs["sort"][0]["_geo_distance"] - - # ElasticSearch supports the GeoJSON-style lng, lat pairs so unlike Solr the values should be - # in the same order as we used to create the Point(): - # http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-geo-distance-filter.html#_lat_lon_as_array_4 - - self.assertDictEqual( - geo_d, {"location": [1.23, 4.56], "unit": "km", "order": "desc"} - ) - - def test_more_like_this(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 3) - - # A functional MLT example with enough data to work is below. Rely on - # this to ensure the API is correct enough. - self.assertEqual(self.sb.more_like_this(self.sample_objs[0])["hits"], 0) - self.assertEqual( - [ - result.pk - for result in self.sb.more_like_this(self.sample_objs[0])["results"] - ], - [], - ) - - def test_build_schema(self): - old_ui = connections["elasticsearch"].get_unified_index() - - (content_field_name, mapping) = self.sb.build_schema(old_ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 4 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - }, - ) - - ui = UnifiedIndex() - ui.build(indexes=[Elasticsearch2ComplexFacetsMockSearchIndex()]) - (content_field_name, mapping) = self.sb.build_schema(ui.all_searchfields()) - self.assertEqual(content_field_name, "text") - self.assertEqual(len(mapping), 15 + 2) # +2 management fields - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name": {"type": "string", "analyzer": "snowball"}, - "is_active_exact": {"type": "boolean"}, - "created": {"type": "date"}, - "post_count": {"type": "long"}, - "created_exact": {"type": "date"}, - "sites_exact": {"index": "not_analyzed", "type": "string"}, - "is_active": {"type": "boolean"}, - "sites": {"type": "string", "analyzer": "snowball"}, - "post_count_i": {"type": "long"}, - "average_rating": {"type": "float"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date_exact": {"type": "date"}, - "name_exact": {"index": "not_analyzed", "type": "string"}, - "pub_date": {"type": "date"}, - "average_rating_exact": {"type": "float"}, - }, - ) - - def test_verify_type(self): - old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - smtmmi = Elasticsearch2MaintainTypeMockSearchIndex() - ui.build(indexes=[smtmmi]) - connections["elasticsearch"]._index = ui - sb = connections["elasticsearch"].get_backend() - sb.update(smtmmi, self.sample_objs) - - self.assertEqual(sb.search("*:*")["hits"], 3) - self.assertEqual( - [result.month for result in sb.search("*:*")["results"]], ["02", "02", "02"] - ) - connections["elasticsearch"]._index = old_ui - - -class CaptureHandler(std_logging.Handler): - logs_seen = [] - - def emit(self, record): - CaptureHandler.logs_seen.append(record) - - -class FailedElasticsearch2SearchBackendTestCase(TestCase): - def setUp(self): - self.sample_objs = [] - - for i in range(1, 4): - mock = MockModel() - mock.id = i - mock.author = "daniel%s" % i - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - # Stow. - # Point the backend at a URL that doesn't exist so we can watch the - # sparks fly. - self.old_es_url = settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = ( - "%s/foo/" % self.old_es_url - ) - self.cap = CaptureHandler() - logging.getLogger("haystack").addHandler(self.cap) - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(config.stream) - - # Setup the rest of the bits. - self.old_ui = connections["elasticsearch"].get_unified_index() - ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = ui - self.sb = connections["elasticsearch"].get_backend() - - def tearDown(self): - # Restore. - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] = self.old_es_url - connections["elasticsearch"]._index = self.old_ui - config = apps.get_app_config("haystack") - logging.getLogger("haystack").removeHandler(self.cap) - logging.getLogger("haystack").addHandler(config.stream) - - @unittest.expectedFailure - def test_all_cases(self): - # Prior to the addition of the try/except bits, these would all fail miserably. - self.assertEqual(len(CaptureHandler.logs_seen), 0) - - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(len(CaptureHandler.logs_seen), 1) - - self.sb.remove(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 2) - - self.sb.search("search") - self.assertEqual(len(CaptureHandler.logs_seen), 3) - - self.sb.more_like_this(self.sample_objs[0]) - self.assertEqual(len(CaptureHandler.logs_seen), 4) - - self.sb.clear([MockModel]) - self.assertEqual(len(CaptureHandler.logs_seen), 5) - - self.sb.clear() - self.assertEqual(len(CaptureHandler.logs_seen), 6) - - -class LiveElasticsearch2SearchQueryTestCase(TestCase): - fixtures = ["base_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - self.sq = connections["elasticsearch"].get_query() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_log_query(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=False): - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - with self.settings(DEBUG=True): - # Redefine it to clear out the cached results. - self.sq = connections["elasticsearch"].query(using="elasticsearch") - self.sq.add_filter(SQ(name="bar")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - - # And again, for good measure. - self.sq = connections["elasticsearch"].query("elasticsearch") - self.sq.add_filter(SQ(name="bar")) - self.sq.add_filter(SQ(text="moof")) - len(self.sq.get_results()) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - self.assertEqual( - connections["elasticsearch"].queries[0]["query_string"], "name:(bar)" - ) - self.assertEqual( - connections["elasticsearch"].queries[1]["query_string"], - "(name:(bar) AND text:(moof))", - ) - - -lssqstc_all_loaded = None - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SearchQuerySetTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - self.rsqs = RelatedSearchQuerySet("elasticsearch") - - # Ugly but not constantly reindexing saves us almost 50% runtime. - global lssqstc_all_loaded - - if lssqstc_all_loaded is None: - lssqstc_all_loaded = True - - # Wipe it clean. - clear_elasticsearch_index() - - # Force indexing of the content. - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_load_all(self): - sqs = self.sqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - results = sorted([int(result.pk) for result in sqs]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_values_slicing(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - - # TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends - - # The values will come back as strings because Hasytack doesn't assume PKs are integers. - # We'll prepare this set once since we're going to query the same results in multiple ways: - expected_pks = [str(i) for i in [3, 2, 4, 5, 6, 7, 8, 9, 10, 11]] - - results = self.sqs.all().order_by("pub_date").values("pk") - self.assertListEqual([i["pk"] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk") - self.assertListEqual([i[0] for i in results[1:11]], expected_pks) - - results = self.sqs.all().order_by("pub_date").values_list("pk", flat=True) - self.assertListEqual(results[1:11], expected_pks) - - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_count(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.sqs.all() - self.assertEqual(sqs.count(), 23) - self.assertEqual(sqs.count(), 23) - self.assertEqual(len(sqs), 23) - self.assertEqual(sqs.count(), 23) - # Should only execute one query to count the length of the result set. - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_manual_iter(self): - results = self.sqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = set([int(result.pk) for result in results._manual_iter()]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.sqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.sqs._cache_is_full(), False) - results = self.sqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test___and__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 & sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) AND (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar") - sqs = sqs3 & sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 3) - self.assertEqual( - sqs.query.build_query(), - "(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))", - ) - - def test___or__(self): - sqs1 = self.sqs.filter(content="foo") - sqs2 = self.sqs.filter(content="bar") - sqs = sqs1 | sqs2 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual(sqs.query.build_query(), "((foo) OR (bar))") - - # Now for something more complex... - sqs3 = self.sqs.exclude(title="moof").filter( - SQ(content="foo") | SQ(content="baz") - ) - sqs4 = self.sqs.filter(content="bar").models(MockModel) - sqs = sqs3 | sqs4 - - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.query_filter), 2) - self.assertEqual( - sqs.query.build_query(), - "((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))", - ) - - def test_auto_query(self): - # Ensure bits in exact matches get escaped properly as well. - # This will break horrifically if escaping isn't working. - sqs = self.sqs.auto_query('"pants:rule"') - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual( - repr(sqs.query.query_filter), '' - ) - self.assertEqual(sqs.query.build_query(), '("pants\\:rule")') - self.assertEqual(len(sqs), 0) - - # Regressions - - def test_regression_proper_start_offsets(self): - sqs = self.sqs.filter(text="index") - self.assertNotEqual(sqs.count(), 0) - - id_counts = {} - - for item in sqs: - if item.id in id_counts: - id_counts[item.id] += 1 - else: - id_counts[item.id] = 1 - - for key, value in id_counts.items(): - if value > 1: - self.fail( - "Result with id '%s' seen more than once in the results." % key - ) - - def test_regression_raw_search_breaks_slicing(self): - sqs = self.sqs.raw_search("text:index") - page_1 = [result.pk for result in sqs[0:10]] - page_2 = [result.pk for result in sqs[10:20]] - - for pk in page_2: - if pk in page_1: - self.fail( - "Result with id '%s' seen more than once in the results." % pk - ) - - # RelatedSearchQuerySet Tests - - def test_related_load_all(self): - sqs = self.rsqs.order_by("pub_date").load_all() - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertTrue(len(sqs) > 0) - self.assertEqual( - sqs[2].object.foo, - "In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.", - ) - - def test_related_load_all_queryset(self): - sqs = self.rsqs.load_all().order_by("pub_date") - self.assertEqual(len(sqs._load_all_querysets), 0) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual(sorted([obj.object.id for obj in sqs]), list(range(2, 24))) - - sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs._load_all_querysets), 1) - self.assertEqual( - set([obj.object.id for obj in sqs]), - {12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20}, - ) - self.assertEqual(set([obj.object.id for obj in sqs[10:20]]), {21, 22, 23}) - - def test_related_iter(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - sqs = self.rsqs.all() - results = set([int(result.pk) for result in sqs]) - self.assertEqual( - results, - { - 2, - 7, - 12, - 17, - 1, - 6, - 11, - 16, - 23, - 5, - 10, - 15, - 22, - 4, - 9, - 14, - 19, - 21, - 3, - 8, - 13, - 18, - 20, - }, - ) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_slice(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - [int(result.pk) for result in results[1:11]], - [3, 2, 4, 5, 6, 7, 8, 9, 10, 11], - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual(int(results[21].pk), 22) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all().order_by("pub_date") - self.assertEqual( - set([int(result.pk) for result in results[20:30]]), {21, 22, 23} - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - - def test_related_manual_iter(self): - results = self.rsqs.all() - - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = sorted([int(result.pk) for result in results._manual_iter()]) - self.assertEqual(results, list(range(1, 24))) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_related_fill_cache(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results = self.rsqs.all() - self.assertEqual(len(results._result_cache), 0) - self.assertEqual(len(connections["elasticsearch"].queries), 0) - results._fill_cache(0, 10) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 10 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 1) - results._fill_cache(10, 20) - self.assertEqual( - len([result for result in results._result_cache if result is not None]), 20 - ) - self.assertEqual(len(connections["elasticsearch"].queries), 2) - - def test_related_cache_is_full(self): - reset_search_queries() - self.assertEqual(len(connections["elasticsearch"].queries), 0) - self.assertEqual(self.rsqs._cache_is_full(), False) - results = self.rsqs.all() - fire_the_iterator_and_fill_cache = [result for result in results] - self.assertEqual(results._cache_is_full(), True) - self.assertEqual(len(connections["elasticsearch"].queries), 3) - - def test_quotes_regression(self): - sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E") - # Should not have empty terms. - self.assertEqual(sqs.query.build_query(), "(44\xb048'40''N 20\xb028'32''E)") - # Should not cause Elasticsearch to 500. - self.assertEqual(sqs.count(), 0) - - sqs = self.sqs.auto_query("blazing") - self.assertEqual(sqs.query.build_query(), "(blazing)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("blazing saddles") - self.assertEqual(sqs.query.build_query(), "(blazing saddles)") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles') - self.assertEqual(sqs.query.build_query(), '(\\"blazing saddles)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles"') - self.assertEqual(sqs.query.build_query(), '("blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing \'saddles"') - self.assertEqual(sqs.query.build_query(), '(mel "blazing \'saddles")') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" ')") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query("mel \"blazing ''saddles\"'\"") - self.assertEqual(sqs.query.build_query(), "(mel \"blazing ''saddles\" '\\\")") - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('"blazing saddles" mel brooks') - self.assertEqual(sqs.query.build_query(), '("blazing saddles" mel brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" brooks)') - self.assertEqual(sqs.count(), 0) - sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') - self.assertEqual(sqs.query.build_query(), '(mel "blazing saddles" \\"brooks)') - self.assertEqual(sqs.count(), 0) - - def test_query_generation(self): - sqs = self.sqs.filter( - SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")) - ) - self.assertEqual( - sqs.query.build_query(), "((hello world) OR title:(hello world))" - ) - - def test_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - sqs = self.sqs.all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - # Custom class. - sqs = self.sqs.result_class(MockSearchResult).all() - self.assertTrue(isinstance(sqs[0], MockSearchResult)) - - # Reset to default. - sqs = self.sqs.result_class(None).all() - self.assertTrue(isinstance(sqs[0], SearchResult)) - - -@override_settings(DEBUG=True) -class LiveElasticsearch2SpellingTestCase(TestCase): - """Used to test actual implementation details of the SearchQuerySet.""" - - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockSpellingIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_spelling(self): - self.assertEqual( - self.sqs.auto_query("structurd").spelling_suggestion(), "structured" - ) - self.assertEqual(self.sqs.spelling_suggestion("structurd"), "structured") - self.assertEqual( - self.sqs.auto_query("srchindex instanc").spelling_suggestion(), - "searchindex instance", - ) - self.assertEqual( - self.sqs.spelling_suggestion("srchindex instanc"), "searchindex instance" - ) - - -class LiveElasticsearch2MoreLikeThisTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_more_like_this(self): - mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1)) - results = [result.pk for result in mlt] - self.assertEqual(mlt.count(), 11) - self.assertEqual( - set(results), {"10", "5", "2", "21", "4", "6", "23", "9", "14"} - ) - self.assertEqual(len(results), 10) - - alt_mlt = self.sqs.filter(name="daniel3").more_like_this( - MockModel.objects.get(pk=2) - ) - results = [result.pk for result in alt_mlt] - self.assertEqual(alt_mlt.count(), 9) - self.assertEqual( - set(results), {"2", "16", "3", "19", "4", "17", "10", "22", "23"} - ) - self.assertEqual(len(results), 9) - - alt_mlt_with_models = self.sqs.models(MockModel).more_like_this( - MockModel.objects.get(pk=1) - ) - results = [result.pk for result in alt_mlt_with_models] - self.assertEqual(alt_mlt_with_models.count(), 10) - self.assertEqual( - set(results), {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"} - ) - self.assertEqual(len(results), 10) - - if hasattr(MockModel.objects, "defer"): - # Make sure MLT works with deferred bits. - qs = MockModel.objects.defer("foo") - self.assertEqual(qs.query.deferred_loading[1], True) - deferred = self.sqs.models(MockModel).more_like_this(qs.get(pk=1)) - self.assertEqual(deferred.count(), 10) - self.assertEqual( - {result.pk for result in deferred}, - {"10", "5", "21", "2", "4", "6", "23", "9", "14", "16"}, - ) - self.assertEqual(len([result.pk for result in deferred]), 10) - - # Ensure that swapping the ``result_class`` works. - self.assertTrue( - isinstance( - self.sqs.result_class(MockSearchResult).more_like_this( - MockModel.objects.get(pk=1) - )[0], - MockSearchResult, - ) - ) - - -class LiveElasticsearch2AutocompleteTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2AutocompleteMockModelSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - # Wipe it clean. - clear_elasticsearch_index() - - # Reboot the schema. - self.sb = connections["elasticsearch"].get_backend() - self.sb.setup() - - self.smmi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_build_schema(self): - self.sb = connections["elasticsearch"].get_backend() - content_name, mapping = self.sb.build_schema(self.ui.all_searchfields()) - self.assertEqual( - mapping, - { - "django_id": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "django_ct": { - "index": "not_analyzed", - "type": "string", - "include_in_all": False, - }, - "name_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - "text": {"type": "string", "analyzer": "snowball"}, - "pub_date": {"type": "date"}, - "name": {"type": "string", "analyzer": "snowball"}, - "text_auto": {"type": "string", "analyzer": "edgengram_analyzer"}, - }, - ) - - def test_autocomplete(self): - autocomplete = self.sqs.autocomplete(text_auto="mod") - self.assertEqual(autocomplete.count(), 16) - self.assertEqual( - set([result.pk for result in autocomplete]), - { - "1", - "12", - "6", - "14", - "7", - "4", - "23", - "17", - "13", - "18", - "20", - "22", - "19", - "15", - "10", - "2", - }, - ) - self.assertTrue("mod" in autocomplete[0].text.lower()) - self.assertTrue("mod" in autocomplete[1].text.lower()) - self.assertTrue("mod" in autocomplete[2].text.lower()) - self.assertTrue("mod" in autocomplete[3].text.lower()) - self.assertTrue("mod" in autocomplete[4].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete]), 16) - - # Test multiple words. - autocomplete_2 = self.sqs.autocomplete(text_auto="your mod") - self.assertEqual(autocomplete_2.count(), 13) - self.assertEqual( - set([result.pk for result in autocomplete_2]), - {"1", "6", "2", "14", "12", "13", "10", "19", "4", "20", "23", "22", "15"}, - ) - map_results = {result.pk: result for result in autocomplete_2} - self.assertTrue("your" in map_results["1"].text.lower()) - self.assertTrue("mod" in map_results["1"].text.lower()) - self.assertTrue("your" in map_results["6"].text.lower()) - self.assertTrue("mod" in map_results["6"].text.lower()) - self.assertTrue("your" in map_results["2"].text.lower()) - self.assertEqual(len([result.pk for result in autocomplete_2]), 13) - - # Test multiple fields. - autocomplete_3 = self.sqs.autocomplete(text_auto="Django", name_auto="dan") - self.assertEqual(autocomplete_3.count(), 4) - self.assertEqual( - set([result.pk for result in autocomplete_3]), {"12", "1", "22", "14"} - ) - self.assertEqual(len([result.pk for result in autocomplete_3]), 4) - - # Test numbers in phrases - autocomplete_4 = self.sqs.autocomplete(text_auto="Jen 867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - # Test numbers alone - autocomplete_4 = self.sqs.autocomplete(text_auto="867") - self.assertEqual(autocomplete_4.count(), 1) - self.assertEqual(set([result.pk for result in autocomplete_4]), {"20"}) - - -class LiveElasticsearch2RoundTripTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.srtsi = Elasticsearch2RoundTripSearchIndex() - self.ui.build(indexes=[self.srtsi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sqs = SearchQuerySet("elasticsearch") - - # Fake indexing. - mock = MockModel() - mock.id = 1 - self.sb.update(self.srtsi, [mock]) - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_round_trip(self): - results = self.sqs.filter(id="core.mockmodel.1") - - # Sanity check. - self.assertEqual(results.count(), 1) - - # Check the individual fields. - result = results[0] - self.assertEqual(result.id, "core.mockmodel.1") - self.assertEqual(result.text, "This is some example text.") - self.assertEqual(result.name, "Mister Pants") - self.assertEqual(result.is_active, True) - self.assertEqual(result.post_count, 25) - self.assertEqual(result.average_rating, 3.6) - self.assertEqual(result.price, "24.99") - self.assertEqual(result.pub_date, datetime.date(2009, 11, 21)) - self.assertEqual(result.created, datetime.datetime(2009, 11, 21, 21, 31, 00)) - self.assertEqual(result.tags, ["staff", "outdoor", "activist", "scientist"]) - self.assertEqual(result.sites, [3, 5, 1]) - - -class LiveElasticsearch2PickleTestCase(TestCase): - fixtures = ["bulk_data.json"] - - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2MockModelSearchIndex() - self.sammi = Elasticsearch2AnotherMockModelSearchIndex() - self.ui.build(indexes=[self.smmi, self.sammi]) - connections["elasticsearch"]._index = self.ui - - self.sqs = SearchQuerySet("elasticsearch") - - self.smmi.update(using="elasticsearch") - self.sammi.update(using="elasticsearch") - - def tearDown(self): - # Restore. - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_pickling(self): - results = self.sqs.all() - - for res in results: - # Make sure the cache is full. - pass - - in_a_pickle = pickle.dumps(results) - like_a_cuke = pickle.loads(in_a_pickle) - self.assertEqual(len(like_a_cuke), len(results)) - self.assertEqual(like_a_cuke[0].id, results[0].id) - - -class Elasticsearch2BoostBackendTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2BoostMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - self.sample_objs = [] - - for i in range(1, 5): - mock = AFourthMockModel() - mock.id = i - - if i % 2: - mock.author = "daniel" - mock.editor = "david" - else: - mock.author = "david" - mock.editor = "daniel" - - mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def raw_search(self, query): - return self.raw_es.search( - q="*:*", index=settings.HAYSTACK_CONNECTIONS["elasticsearch"]["INDEX_NAME"] - ) - - def test_boost(self): - self.sb.update(self.smmi, self.sample_objs) - self.assertEqual(self.raw_search("*:*")["hits"]["total"], 4) - - results = SearchQuerySet(using="elasticsearch").filter( - SQ(author="daniel") | SQ(editor="daniel") - ) - - self.assertEqual( - set([result.id for result in results]), - { - "core.afourthmockmodel.4", - "core.afourthmockmodel.3", - "core.afourthmockmodel.1", - "core.afourthmockmodel.2", - }, - ) - - def test__to_python(self): - self.assertEqual(self.sb._to_python("abc"), "abc") - self.assertEqual(self.sb._to_python("1"), 1) - self.assertEqual(self.sb._to_python("2653"), 2653) - self.assertEqual(self.sb._to_python("25.5"), 25.5) - self.assertEqual(self.sb._to_python("[1, 2, 3]"), [1, 2, 3]) - self.assertEqual( - self.sb._to_python('{"a": 1, "b": 2, "c": 3}'), {"a": 1, "c": 3, "b": 2} - ) - self.assertEqual( - self.sb._to_python("2009-05-09T16:14:00"), - datetime.datetime(2009, 5, 9, 16, 14), - ) - self.assertEqual( - self.sb._to_python("2009-05-09T00:00:00"), - datetime.datetime(2009, 5, 9, 0, 0), - ) - self.assertEqual(self.sb._to_python(None), None) - - -class RecreateIndexTestCase(TestCase): - def setUp(self): - self.raw_es = elasticsearch.Elasticsearch( - settings.HAYSTACK_CONNECTIONS["elasticsearch"]["URL"] - ) - - def test_recreate_index(self): - clear_elasticsearch_index() - - sb = connections["elasticsearch"].get_backend() - sb.silently_fail = True - sb.setup() - - original_mapping = self.raw_es.indices.get_mapping(index=sb.index_name) - - sb.clear() - sb.setup() - - try: - updated_mapping = self.raw_es.indices.get_mapping(sb.index_name) - except elasticsearch.NotFoundError: - self.fail("There is no mapping after recreating the index") - - self.assertEqual( - original_mapping, - updated_mapping, - "Mapping after recreating the index differs from the original one", - ) - - -class Elasticsearch2FacetingTestCase(TestCase): - def setUp(self): - super().setUp() - - # Wipe it clean. - clear_elasticsearch_index() - - # Stow. - self.old_ui = connections["elasticsearch"].get_unified_index() - self.ui = UnifiedIndex() - self.smmi = Elasticsearch2FacetingMockSearchIndex() - self.ui.build(indexes=[self.smmi]) - connections["elasticsearch"]._index = self.ui - self.sb = connections["elasticsearch"].get_backend() - - # Force the backend to rebuild the mapping each time. - self.sb.existing_mapping = {} - self.sb.setup() - - self.sample_objs = [] - - for i in range(1, 10): - mock = AFourthMockModel() - mock.id = i - if i > 5: - mock.editor = "George Taylor" - else: - mock.editor = "Perry White" - if i % 2: - mock.author = "Daniel Lindsley" - else: - mock.author = "Dan Watson" - mock.pub_date = datetime.date(2013, 9, (i % 4) + 1) - self.sample_objs.append(mock) - - def tearDown(self): - connections["elasticsearch"]._index = self.old_ui - super().tearDown() - - def test_facet(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 5), ("Dan Watson", 4)] - ) - self.assertEqual( - counts["fields"]["editor"], [("Perry White", 5), ("George Taylor", 4)] - ) - counts = ( - SearchQuerySet("elasticsearch") - .filter(content="white") - .facet("facet_field", order="reverse_count") - .facet_counts() - ) - self.assertEqual( - counts["fields"]["facet_field"], [("Dan Watson", 2), ("Daniel Lindsley", 3)] - ) - - def test_multiple_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .narrow('editor_exact:"Perry White"') - .narrow('author_exact:"Daniel Lindsley"') - .facet("author") - .facet_counts() - ) - self.assertEqual(counts["fields"]["author"], [("Daniel Lindsley", 3)]) - - def test_narrow(self): - self.sb.update(self.smmi, self.sample_objs) - counts = ( - SearchQuerySet("elasticsearch") - .facet("author") - .facet("editor") - .narrow('editor_exact:"Perry White"') - .facet_counts() - ) - self.assertEqual( - counts["fields"]["author"], [("Daniel Lindsley", 3), ("Dan Watson", 2)] - ) - self.assertEqual(counts["fields"]["editor"], [("Perry White", 5)]) - - def test_date_facet(self): - self.sb.update(self.smmi, self.sample_objs) - start = datetime.date(2013, 9, 1) - end = datetime.date(2013, 9, 30) - # Facet by day - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="day") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], - [ - (datetime.datetime(2013, 9, 1), 2), - (datetime.datetime(2013, 9, 2), 3), - (datetime.datetime(2013, 9, 3), 2), - (datetime.datetime(2013, 9, 4), 2), - ], - ) - # By month - counts = ( - SearchQuerySet("elasticsearch") - .date_facet("pub_date", start_date=start, end_date=end, gap_by="month") - .facet_counts() - ) - self.assertEqual( - counts["dates"]["pub_date"], [(datetime.datetime(2013, 9, 1), 9)] - ) diff --git a/test_haystack/elasticsearch2_tests/test_inputs.py b/test_haystack/elasticsearch2_tests/test_inputs.py deleted file mode 100644 index af9f8f332..000000000 --- a/test_haystack/elasticsearch2_tests/test_inputs.py +++ /dev/null @@ -1,85 +0,0 @@ -from django.test import TestCase - -from haystack import connections, inputs - - -class Elasticsearch2InputTestCase(TestCase): - def setUp(self): - super().setUp() - self.query_obj = connections["elasticsearch"].get_query() - - def test_raw_init(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {}) - self.assertEqual(raw.post_process, False) - - raw = inputs.Raw("hello OR there, :you", test="really") - self.assertEqual(raw.query_string, "hello OR there, :you") - self.assertEqual(raw.kwargs, {"test": "really"}) - self.assertEqual(raw.post_process, False) - - def test_raw_prepare(self): - raw = inputs.Raw("hello OR there, :you") - self.assertEqual(raw.prepare(self.query_obj), "hello OR there, :you") - - def test_clean_init(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.query_string, "hello OR there, :you") - self.assertEqual(clean.post_process, True) - - def test_clean_prepare(self): - clean = inputs.Clean("hello OR there, :you") - self.assertEqual(clean.prepare(self.query_obj), "hello or there, \\:you") - - def test_exact_init(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.query_string, "hello OR there, :you") - self.assertEqual(exact.post_process, True) - - def test_exact_prepare(self): - exact = inputs.Exact("hello OR there, :you") - self.assertEqual(exact.prepare(self.query_obj), '"hello OR there, :you"') - - exact = inputs.Exact("hello OR there, :you", clean=True) - self.assertEqual(exact.prepare(self.query_obj), '"hello or there, \\:you"') - - def test_not_init(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.query_string, "hello OR there, :you") - self.assertEqual(not_it.post_process, True) - - def test_not_prepare(self): - not_it = inputs.Not("hello OR there, :you") - self.assertEqual(not_it.prepare(self.query_obj), "NOT (hello or there, \\:you)") - - def test_autoquery_init(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual(autoquery.query_string, 'panic -don\'t "froody dude"') - self.assertEqual(autoquery.post_process, False) - - def test_autoquery_prepare(self): - autoquery = inputs.AutoQuery('panic -don\'t "froody dude"') - self.assertEqual( - autoquery.prepare(self.query_obj), 'panic NOT don\'t "froody dude"' - ) - - def test_altparser_init(self): - altparser = inputs.AltParser("dismax") - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "") - self.assertEqual(altparser.kwargs, {}) - self.assertEqual(altparser.post_process, False) - - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual(altparser.parser_name, "dismax") - self.assertEqual(altparser.query_string, "douglas adams") - self.assertEqual(altparser.kwargs, {"mm": 1, "qf": "author"}) - self.assertEqual(altparser.post_process, False) - - def test_altparser_prepare(self): - altparser = inputs.AltParser("dismax", "douglas adams", qf="author", mm=1) - self.assertEqual( - altparser.prepare(self.query_obj), - """{!dismax mm=1 qf=author v='douglas adams'}""", - ) diff --git a/test_haystack/elasticsearch2_tests/test_query.py b/test_haystack/elasticsearch2_tests/test_query.py deleted file mode 100644 index 5a0111d5b..000000000 --- a/test_haystack/elasticsearch2_tests/test_query.py +++ /dev/null @@ -1,247 +0,0 @@ -import datetime - -import elasticsearch -from django.contrib.gis.measure import D -from django.test import TestCase - -from haystack import connections -from haystack.inputs import Exact -from haystack.models import SearchResult -from haystack.query import SQ, SearchQuerySet - -from ..core.models import AnotherMockModel, MockModel - - -class Elasticsearch2SearchQueryTestCase(TestCase): - def setUp(self): - super().setUp() - self.sq = connections["elasticsearch"].get_query() - - def test_build_query_all(self): - self.assertEqual(self.sq.build_query(), "*:*") - - def test_build_query_single_word(self): - self.sq.add_filter(SQ(content="hello")) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_build_query_boolean(self): - self.sq.add_filter(SQ(content=True)) - self.assertEqual(self.sq.build_query(), "(True)") - - def test_regression_slash_search(self): - self.sq.add_filter(SQ(content="hello/")) - self.assertEqual(self.sq.build_query(), "(hello\\/)") - - def test_build_query_datetime(self): - self.sq.add_filter(SQ(content=datetime.datetime(2009, 5, 8, 11, 28))) - self.assertEqual(self.sq.build_query(), "(2009-05-08T11:28:00)") - - def test_build_query_multiple_words_and(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_filter(SQ(content="world")) - self.assertEqual(self.sq.build_query(), "((hello) AND (world))") - - def test_build_query_multiple_words_not(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) AND NOT ((world)))") - - def test_build_query_multiple_words_or(self): - self.sq.add_filter(~SQ(content="hello")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.assertEqual(self.sq.build_query(), "(NOT ((hello)) OR (hello))") - - def test_build_query_multiple_words_mixed(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(content="hello"), use_or=True) - self.sq.add_filter(~SQ(content="world")) - self.assertEqual( - self.sq.build_query(), "(((why) OR (hello)) AND NOT ((world)))" - ) - - def test_build_query_phrase(self): - self.sq.add_filter(SQ(content="hello world")) - self.assertEqual(self.sq.build_query(), "(hello AND world)") - - self.sq.add_filter(SQ(content__exact="hello world")) - self.assertEqual( - self.sq.build_query(), '((hello AND world) AND ("hello world"))' - ) - - def test_build_query_boost(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_boost("world", 5) - self.assertEqual(self.sq.build_query(), "(hello) world^5") - - def test_build_query_multiple_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=Exact("2009-02-10 01:59:00"))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=Exact("2009-02-12 12:13:00"))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10 01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12 12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_multiple_filter_types_with_datetimes(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__lte=datetime.datetime(2009, 2, 10, 1, 59, 0))) - self.sq.add_filter(SQ(author__gt="daniel")) - self.sq.add_filter(SQ(created__lt=datetime.datetime(2009, 2, 12, 12, 13, 0))) - self.sq.add_filter(SQ(title__gte="B")) - self.sq.add_filter(SQ(id__in=[1, 2, 3])) - self.sq.add_filter(SQ(rating__range=[3, 5])) - self.assertEqual( - self.sq.build_query(), - '((why) AND pub_date:([* TO "2009-02-10T01:59:00"]) AND author:({"daniel" TO *}) AND created:({* TO "2009-02-12T12:13:00"}) AND title:(["B" TO *]) AND id:("1" OR "2" OR "3") AND rating:(["3" TO "5"]))', - ) - - def test_build_query_in_filter_multiple_words(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=["A Famous Paper", "An Infamous Article"])) - self.assertEqual( - self.sq.build_query(), - '((why) AND title:("A Famous Paper" OR "An Infamous Article"))', - ) - - def test_build_query_in_filter_datetime(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(pub_date__in=[datetime.datetime(2009, 7, 6, 1, 56, 21)])) - self.assertEqual( - self.sq.build_query(), '((why) AND pub_date:("2009-07-06T01:56:21"))' - ) - - def test_build_query_in_with_set(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in={"A Famous Paper", "An Infamous Article"})) - self.assertTrue("((why) AND title:(" in self.sq.build_query()) - self.assertTrue('"A Famous Paper"' in self.sq.build_query()) - self.assertTrue('"An Infamous Article"' in self.sq.build_query()) - - def test_build_query_wildcard_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__startswith="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack*))") - - def test_build_query_fuzzy_filter_types(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__fuzzy="haystack")) - self.assertEqual(self.sq.build_query(), "((why) AND title:(haystack~))") - - def test_clean(self): - self.assertEqual(self.sq.clean("hello world"), "hello world") - self.assertEqual(self.sq.clean("hello AND world"), "hello and world") - self.assertEqual( - self.sq.clean( - r'hello AND OR NOT TO + - && || ! ( ) { } [ ] ^ " ~ * ? : \ / world' - ), - 'hello and or not to \\+ \\- \\&& \\|| \\! \\( \\) \\{ \\} \\[ \\] \\^ \\" \\~ \\* \\? \\: \\\\ \\/ world', - ) - self.assertEqual( - self.sq.clean("so please NOTe i am in a bAND and bORed"), - "so please NOTe i am in a bAND and bORed", - ) - - def test_build_query_with_models(self): - self.sq.add_filter(SQ(content="hello")) - self.sq.add_model(MockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - self.sq.add_model(AnotherMockModel) - self.assertEqual(self.sq.build_query(), "(hello)") - - def test_set_result_class(self): - # Assert that we're defaulting to ``SearchResult``. - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - # Custom class. - class IttyBittyResult: - pass - - self.sq.set_result_class(IttyBittyResult) - self.assertTrue(issubclass(self.sq.result_class, IttyBittyResult)) - - # Reset to default. - self.sq.set_result_class(None) - self.assertTrue(issubclass(self.sq.result_class, SearchResult)) - - def test_in_filter_values_list(self): - self.sq.add_filter(SQ(content="why")) - self.sq.add_filter(SQ(title__in=[1, 2, 3])) - self.assertEqual(self.sq.build_query(), '((why) AND title:("1" OR "2" OR "3"))') - - def test_narrow_sq(self): - sqs = SearchQuerySet(using="elasticsearch").narrow(SQ(foo="moof")) - self.assertTrue(isinstance(sqs, SearchQuerySet)) - self.assertEqual(len(sqs.query.narrow_queries), 1) - self.assertEqual(sqs.query.narrow_queries.pop(), "foo:(moof)") - - -class Elasticsearch2SearchQuerySpatialBeforeReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (0, 9, 9) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions < 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - {"distance": 0.5, "location_field": {"lat": 2.3456789, "lon": 1.2345678}}, - ) - - -class Elasticsearch2SearchQuerySpatialAfterReleaseTestCase(TestCase): - def setUp(self): - super().setUp() - self.backend = connections["elasticsearch"].get_backend() - self._elasticsearch_version = elasticsearch.VERSION - elasticsearch.VERSION = (1, 0, 0) - - def tearDown(self): - elasticsearch.VERSION = self._elasticsearch_version - - def test_build_query_with_dwithin_range(self): - """ - Test build_search_kwargs with dwithin range for Elasticsearch versions >= 1.0.0 - """ - from django.contrib.gis.geos import Point - - search_kwargs = self.backend.build_search_kwargs( - "where", - dwithin={ - "field": "location_field", - "point": Point(1.2345678, 2.3456789), - "distance": D(m=500), - }, - ) - self.assertEqual( - search_kwargs["query"]["filtered"]["filter"]["bool"]["must"][1][ - "geo_distance" - ], - { - "distance": "0.500000km", - "location_field": {"lat": 2.3456789, "lon": 1.2345678}, - }, - ) diff --git a/test_haystack/settings.py b/test_haystack/settings.py index 9a78bc5bc..7c658836a 100644 --- a/test_haystack/settings.py +++ b/test_haystack/settings.py @@ -95,13 +95,7 @@ try: import elasticsearch - if (2,) <= elasticsearch.__version__ <= (3,): - HAYSTACK_CONNECTIONS["elasticsearch"].update( - { - "ENGINE": "haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine" - } - ) - elif (5,) <= elasticsearch.__version__ <= (6,): + if (5,) <= elasticsearch.__version__ <= (6,): HAYSTACK_CONNECTIONS["elasticsearch"].update( { "ENGINE": "haystack.backends.elasticsearch5_backend.Elasticsearch5SearchEngine" From a200e257f0f1c6c7bb7ba3ab08313ad6ff479f3e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 07:10:14 +0200 Subject: [PATCH 142/162] [pre-commit.ci] pre-commit autoupdate (#1979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.18.0 → 1.19.0](https://github.com/adamchainz/django-upgrade/compare/1.18.0...1.19.0) - [github.com/astral-sh/ruff-pre-commit: v0.4.10 → v0.5.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.10...v0.5.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4972ed0ea..434f03a11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.18.0 + rev: 1.19.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.10 + rev: v0.5.0 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From aeefbd8bf9e48577716c6e3997bf4d846990cff9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 02:11:56 +0200 Subject: [PATCH 143/162] [pre-commit.ci] pre-commit autoupdate (#1981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.0 → v0.5.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.0...v0.5.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 434f03a11..6378eb2b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.5.1 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 4bb8bcf353159cb3fb5e2dcf2d1634d93fbb7048 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 9 Jul 2024 18:57:46 +0200 Subject: [PATCH 144/162] pre-commit no longer supports the prettier file formater https://github.com/pre-commit/mirrors-prettier --- .pre-commit-config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6378eb2b6..d314b465f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,9 +46,3 @@ repos: - id: pretty-format-json args: ["--autofix", "--no-sort-keys", "--indent=4"] - id: trailing-whitespace - - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 - hooks: - - id: prettier - types_or: [json, toml, xml, yaml] From 28a539e6a7587d87a92de80c4a287feb35f6ac85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 07:13:27 +0200 Subject: [PATCH 145/162] [pre-commit.ci] pre-commit autoupdate (#1985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.1 → v0.5.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.1...v0.5.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d314b465f..a7ff528ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.1 + rev: v0.5.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 98c8d737bfc5f6ce865abb89e15b9d0c5b3899cc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:10:49 +0200 Subject: [PATCH 146/162] [pre-commit.ci] pre-commit autoupdate (#1987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.19.0 → 1.20.0](https://github.com/adamchainz/django-upgrade/compare/1.19.0...1.20.0) - [github.com/astral-sh/ruff-pre-commit: v0.5.2 → v0.5.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.2...v0.5.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7ff528ea..c828c0afa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.19.0 + rev: 1.20.0 hooks: - id: django-upgrade args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.2 + rev: v0.5.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 33f3e8fd9423b9facb270384720b2efd3ff710d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 01:39:01 +0200 Subject: [PATCH 147/162] [pre-commit.ci] pre-commit autoupdate (#1988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.4 → v0.5.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.4...v0.5.5) * test.yml: sudo apt update --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .github/workflows/test.yml | 4 +++- .pre-commit-config.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4dec7412f..95bc31008 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install system dependencies - run: sudo apt install --no-install-recommends -y gdal-bin + run: | + sudo apt update + sudo apt install --no-install-recommends -y gdal-bin - name: Setup solr test server in Docker run: bash test_haystack/solr_tests/server/setup-solr-test-server-in-docker.sh - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c828c0afa..84fb6c0c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.5.5 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 5993d986e55ce3a367114a8231c57a0a3661e618 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 06:29:54 +0200 Subject: [PATCH 148/162] [pre-commit.ci] pre-commit autoupdate (#1989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.5 → v0.5.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.5...v0.5.6) - [github.com/psf/black: 24.4.2 → 24.8.0](https://github.com/psf/black/compare/24.4.2...24.8.0) * .pre-commit-config.yaml: ci: autoupdate_schedule: monthly --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 84fb6c0c6..7f04b2f94 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,5 @@ +ci: + autoupdate_schedule: monthly exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade @@ -7,7 +9,7 @@ repos: args: [--target-version, "5.0"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.5.6 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] @@ -18,7 +20,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black From 7f492268d906de488cc1a72fb8ea89de4decd5c3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 7 Aug 2024 17:23:54 +0200 Subject: [PATCH 149/162] Add Django v5.1 to the testing --- .github/workflows/test.yml | 6 +++++- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 + tox.ini | 4 +++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95bc31008..a520d80d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django - django-version: ["3.2", "4.2", "5.0"] + django-version: ["3.2", "4.2", "5.0", "5.1"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] elastic-version: ["7.17.9"] exclude: @@ -32,6 +32,10 @@ jobs: python-version: "3.8" - django-version: "5.0" python-version: "3.9" + - django-version: "5.1" + python-version: "3.8" + - django-version: "5.1" + python-version: "3.9" services: elastic: image: elasticsearch:${{ matrix.elastic-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f04b2f94..2ff0fcf4e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: rev: 1.20.0 hooks: - id: django-upgrade - args: [--target-version, "5.0"] # Replace with Django version + args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.6 diff --git a/pyproject.toml b/pyproject.toml index 5962dae5b..57c95a8e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ classifiers = [ "Framework :: Django :: 3.2", "Framework :: Django :: 4.2", "Framework :: Django :: 5.0", + "Framework :: Django :: 5.1", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", diff --git a/tox.ini b/tox.ini index d5a436091..7868aec7d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = docs - py{38,39,310,311,312}-django{3.2,4.2,5.0}-es7.x + py{38,39,310,311,312}-django{3.2,4.2,5.0,5.1}-es7.x [gh-actions] python = @@ -16,6 +16,7 @@ DJANGO = 3.2: django3.2 4.2: django4.2 5.0: django5.0 + 5.1: django5.1 [testenv] commands = @@ -32,6 +33,7 @@ deps = django3.2: Django>=3.2,<3.3 django4.2: Django>=4.2,<4.3 django5.0: Django>=5.0,<5.1 + django5.1: Django>=5.1,<5.2 es1.x: elasticsearch>=1,<2 es2.x: elasticsearch>=2,<3 es5.x: elasticsearch>=5,<6 From 068507e6627f96ebd4f3cbe1b789e7b35e590c77 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:00:49 +0200 Subject: [PATCH 150/162] [pre-commit.ci] pre-commit autoupdate (#1995) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.6 → v0.6.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.6...v0.6.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ff0fcf4e..65d32e8c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.3 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 10c15ec12c6d71a815b8843c34ab9dd4d01fddb3 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 14 Oct 2024 12:44:15 +0200 Subject: [PATCH 151/162] GitHub Actions: Add Python 3.13 to the testing (#1997) * GitHub Actions: Add Python 3.13 to the testing * elastic-version: ["7.17.12"] --- .github/workflows/test.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a520d80d4..7349dd71d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,19 +21,21 @@ jobs: fail-fast: false matrix: # https://docs.djangoproject.com/en/stable/faq/install/#what-python-version-can-i-use-with-django django-version: ["3.2", "4.2", "5.0", "5.1"] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - elastic-version: ["7.17.9"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + elastic-version: ["7.17.12"] exclude: - django-version: "3.2" python-version: "3.11" - django-version: "3.2" python-version: "3.12" - - django-version: "5.0" - python-version: "3.8" + - django-version: "3.2" + python-version: "3.13" + - django-version: "4.2" + python-version: "3.13" - django-version: "5.0" python-version: "3.9" - - django-version: "5.1" - python-version: "3.8" + - django-version: "5.0" + python-version: "3.13" - django-version: "5.1" python-version: "3.9" services: From 887836c5e20fdfcd29124beb697ed7dfcc079fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Martano?= Date: Mon, 28 Oct 2024 11:48:14 -0300 Subject: [PATCH 152/162] Fix typo. --- docs/installing_search_engines.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installing_search_engines.rst b/docs/installing_search_engines.rst index 8b4157dcb..d2556298b 100644 --- a/docs/installing_search_engines.rst +++ b/docs/installing_search_engines.rst @@ -28,7 +28,7 @@ but not useful for haystack, and we'll need to configure solr to use a static (classic) schema. Haystack can generate a viable schema.xml and solrconfig.xml for you from your application and reload the core for you (once Haystack is installed and setup). To do this run: -``./manage.py build_solr_schema --configure-directory= +``./manage.py build_solr_schema --configure-directory= --reload-core``. In this example CoreConfigDir is something like ``../solr-6.5.0/server/solr/tester/conf``, and ``--reload-core`` is what triggers reloading of the core. Please refer to ``build_solr_schema`` From 9a7e091df60c8c2f9b4b24763e7fcee8d9a26a5e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:26:35 +0100 Subject: [PATCH 153/162] [pre-commit.ci] pre-commit autoupdate (#2001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/adamchainz/django-upgrade: 1.20.0 → 1.23.1](https://github.com/adamchainz/django-upgrade/compare/1.20.0...1.23.1) - [github.com/astral-sh/ruff-pre-commit: v0.6.3 → v0.9.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.3...v0.9.9) - [github.com/PyCQA/isort: 5.13.2 → 6.0.1](https://github.com/PyCQA/isort/compare/5.13.2...6.0.1) - [github.com/psf/black: 24.8.0 → 25.1.0](https://github.com/psf/black/compare/24.8.0...25.1.0) - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 10 +++++----- haystack/exceptions.py | 1 + test_haystack/test_django_config_detection.py | 4 ---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65d32e8c1..d2c6368f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,29 +3,29 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.20.0 + rev: 1.23.1 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.9.9 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 6.0.1 hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 25.1.0 hooks: - id: black - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files args: ["--maxkb=128"] diff --git a/haystack/exceptions.py b/haystack/exceptions.py index 5c2c4b9a3..95d0bb92a 100644 --- a/haystack/exceptions.py +++ b/haystack/exceptions.py @@ -48,6 +48,7 @@ class SpatialError(HaystackError): class StatsError(HaystackError): "Raised when incorrect arguments have been provided for stats" + pass diff --git a/test_haystack/test_django_config_detection.py b/test_haystack/test_django_config_detection.py index 0c3827882..f4808f68c 100644 --- a/test_haystack/test_django_config_detection.py +++ b/test_haystack/test_django_config_detection.py @@ -16,10 +16,6 @@ def testDefaultAppConfigIsDefined_whenDjangoVersionIsLessThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertTrue(has_default_appconfig_attr) - @unittest.skipIf( - django.VERSION < (3, 2), - "default_app_config should be used in versions prior to django 3.2.", - ) def testDefaultAppConfigIsDefined_whenDjangoVersionIsMoreThan3_2(self): has_default_appconfig_attr = hasattr(haystack, "default_app_config") self.assertFalse(has_default_appconfig_attr) From 9f970a861d3e53af1f1c425da23e4e516674d442 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 27 Apr 2025 23:01:28 +0200 Subject: [PATCH 154/162] [pre-commit.ci] pre-commit autoupdate (#2007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/adamchainz/django-upgrade: 1.23.1 → 1.24.0](https://github.com/adamchainz/django-upgrade/compare/1.23.1...1.24.0) - [github.com/astral-sh/ruff-pre-commit: v0.9.9 → v0.11.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.9...v0.11.4) * facet_types.update(dict.fromkeys(facets, "fields")) --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Christian Clauss --- .pre-commit-config.yaml | 4 ++-- haystack/backends/whoosh_backend.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2c6368f8..f088cd191 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.23.1 + rev: 1.24.0 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.9 + rev: v0.11.4 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] diff --git a/haystack/backends/whoosh_backend.py b/haystack/backends/whoosh_backend.py index 13d68035c..f63ce100a 100644 --- a/haystack/backends/whoosh_backend.py +++ b/haystack/backends/whoosh_backend.py @@ -462,7 +462,7 @@ def search( group_by += [ FieldFacet(facet, allow_overlap=True, maptype=Count) for facet in facets ] - facet_types.update({facet: "fields" for facet in facets}) + facet_types.update(dict.fromkeys(facets, "fields")) if date_facets is not None: From 7be88384752865756a3618c078473ad497e8f44a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 20:35:26 +0000 Subject: [PATCH 155/162] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.11.4 → v0.11.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.4...v0.11.8) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f088cd191..29df9666c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.4 + rev: v0.11.8 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 7d139b4937b821cd12e055d3cb0f20a69a62919b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:59:58 +0200 Subject: [PATCH 156/162] [pre-commit.ci] pre-commit autoupdate (#2010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/adamchainz/django-upgrade: 1.24.0 → 1.25.0](https://github.com/adamchainz/django-upgrade/compare/1.24.0...1.25.0) - [github.com/astral-sh/ruff-pre-commit: v0.11.8 → v0.11.12](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.8...v0.11.12) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29df9666c..9b3c2d6e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,13 +3,13 @@ ci: exclude: ".*/vendor/.*" repos: - repo: https://github.com/adamchainz/django-upgrade - rev: 1.24.0 + rev: 1.25.0 hooks: - id: django-upgrade args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.11.12 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From d04a5841d72228961ee84715b9a9562c80fabcb2 Mon Sep 17 00:00:00 2001 From: Craig de Stigter Date: Wed, 4 Jun 2025 15:36:13 +1200 Subject: [PATCH 157/162] Fix RelatedSearchQueryset.load_all() truncating results Fixes #2011 --- haystack/query.py | 6 +++--- test_haystack/solr_tests/test_solr_backend.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/haystack/query.py b/haystack/query.py index a3cf9490c..0e49486dc 100644 --- a/haystack/query.py +++ b/haystack/query.py @@ -194,9 +194,9 @@ def post_process_results(self, results): # No objects were returned -- possible due to SQS nesting such as # XYZ.objects.filter(id__gt=10) where the amount ignored are # exactly equal to the ITERATOR_LOAD_PER_QUERY - del self._result_cache[: len(results)] - self._ignored_result_count += len(results) - break + del self._result_cache[:1] + self._ignored_result_count += 1 + continue to_cache.append(result) diff --git a/test_haystack/solr_tests/test_solr_backend.py b/test_haystack/solr_tests/test_solr_backend.py index cc0ad551a..cab7b88b1 100644 --- a/test_haystack/solr_tests/test_solr_backend.py +++ b/test_haystack/solr_tests/test_solr_backend.py @@ -1220,6 +1220,21 @@ def test_related_load_all_queryset(self): self.assertEqual([obj.object.id for obj in sqs], list(range(11, 24))) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) + def test_related_load_all_with_empty_model_results(self): + another_index = SolrAnotherMockModelSearchIndex() + another_index.update("solr") + self.ui.build(indexes=[self.smmi, another_index]) + + sqs = self.rsqs.order_by("id") + assert len(list(sqs)) == 25 + sqs = sqs.all().load_all_queryset( + AnotherMockModel, AnotherMockModel.objects.none() + ) + sqs = sqs.load_all() + # two AnotherMockModel objects are skipped, so only 23 results now + # (but those results are still present and weren't skipped) + assert len(list(sqs)) == 23 + def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections["solr"].queries), 0) From f3abe0edc57f0999b67ec43e63aa1b6dfd8835a0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 23:19:05 +0200 Subject: [PATCH 158/162] [pre-commit.ci] pre-commit autoupdate (#2014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.11.12 → v0.12.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.12...v0.12.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b3c2d6e1..42930e402 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.12 + rev: v0.12.2 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 8330862da58abe03ae778f6fa112740442d16b92 Mon Sep 17 00:00:00 2001 From: Dhaval Gojiya <53856555+DhavalGojiya@users.noreply.github.com> Date: Thu, 10 Jul 2025 22:35:02 +0530 Subject: [PATCH 159/162] FIXED: Handle trailing slash in Solr index URL for core reload. (#1968) - When running `python manage.py build_solr_schema --reload_core=True`, it is crucial to correctly extract the Solr core name from the URL defined in the settings. - The existing implementation failed if the URL ended with a trailing slash, resulting in an empty core name due to the final slash being considered as a separator. Added test cases: - `test_build_solr_schema_reload_core_with_trailing_slash` - `test_build_solr_schema_reload_core_without_trailing_slash` These ensure that the core reload logic works correctly regardless of whether the Solr URL has a trailing slash. --- .../management/commands/build_solr_schema.py | 6 ++- .../test_solr_management_commands.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/haystack/management/commands/build_solr_schema.py b/haystack/management/commands/build_solr_schema.py index 21fd4c86b..0ff6215d1 100644 --- a/haystack/management/commands/build_solr_schema.py +++ b/haystack/management/commands/build_solr_schema.py @@ -111,7 +111,11 @@ def handle(self, **options): ) if reload_core: - core = settings.HAYSTACK_CONNECTIONS[using]["URL"].rsplit("/", 1)[-1] + core = ( + settings.HAYSTACK_CONNECTIONS[using]["URL"] + .rstrip("/") + .rsplit("/", 1)[-1] + ) if "ADMIN_URL" not in settings.HAYSTACK_CONNECTIONS[using]: raise ImproperlyConfigured( diff --git a/test_haystack/solr_tests/test_solr_management_commands.py b/test_haystack/solr_tests/test_solr_management_commands.py index 419d21b6d..73ad57c74 100644 --- a/test_haystack/solr_tests/test_solr_management_commands.py +++ b/test_haystack/solr_tests/test_solr_management_commands.py @@ -290,6 +290,48 @@ def test_build_schema(self): settings.HAYSTACK_CONNECTIONS["solr"]["URL"] = oldurl shutil.rmtree(conf_dir, ignore_errors=True) + def test_build_solr_schema_reload_core_without_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL does not have a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Remove trailing slash if present + updated_url = ( + current_url.rstrip("/") if current_url.endswith("/") else current_url + ) + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + + def test_build_solr_schema_reload_core_with_trailing_slash(self): + """Ensure `build_solr_schema` works when the Solr core URL has a trailing slash.""" + + # Get the current Solr URL from settings + current_url = settings.HAYSTACK_CONNECTIONS["solr"]["URL"] + + # Add a trailing slash if not present + updated_url = current_url if current_url.endswith("/") else current_url + "/" + + # Patch only the `URL` key inside `settings.HAYSTACK_CONNECTIONS["solr"]` + with patch.dict(settings.HAYSTACK_CONNECTIONS["solr"], {"URL": updated_url}): + out = StringIO() # Capture output + call_command( + "build_solr_schema", using="solr", reload_core=True, stdout=out + ) + output = out.getvalue() + self.assertIn( + "Trying to reload core named", output + ) # Verify core reload message + class AppModelManagementCommandTestCase(TestCase): fixtures = ["base_data", "bulk_data.json"] From 63f95058f4d17dacb3a84f60fadf8345e9722353 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 06:58:53 +0200 Subject: [PATCH 160/162] [pre-commit.ci] pre-commit autoupdate (#2017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.12.2 → v0.12.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.12.2...v0.12.7) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42930e402..27a1e0665 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: [--target-version, "5.1"] # Replace with Django version - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.2 + rev: v0.12.7 hooks: - id: ruff # args: [ --fix, --exit-non-zero-on-fix ] From 04dcb8ad0bf494f5dd0a012af934f96d82e80f5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:46:23 +0200 Subject: [PATCH 161/162] Bump the github-actions group with 2 updates (#2018) Bumps the github-actions group with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/checkout` from 4 to 5 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) Updates `actions/download-artifact` from 4 to 5 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/download-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pypi-release.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 91fea6827..664a4dca9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index edbe9af1a..7fb7221a0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 7a158c5be..e1bd2ac86 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,7 +7,7 @@ jobs: name: Build Python source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build sdist run: pipx run build --sdist @@ -28,7 +28,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: # unpacks default artifact into dist/ # if `name: artifact` is omitted, the action will create extra parent dir diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7349dd71d..43f75ecf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: ruff: # https://docs.astral.sh/ruff runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: pip install --user ruff - run: ruff check --output-format=github @@ -55,7 +55,7 @@ jobs: ports: - 9001:8983 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 98b7a8a8bc08f4f8aec44bac9230fa1a06dc46eb Mon Sep 17 00:00:00 2001 From: Leif Date: Mon, 18 Aug 2025 12:34:02 -0400 Subject: [PATCH 162/162] Update license field to use proper SPDX identifier (#2016) * Update license field to use proper SPDX identifier This changes the license field to be a valid [SPDX identifier](https://spdx.org/licenses) aligning with [PEP 639](https://peps.python.org/pep-0639/#project-source-metadata). This populates the `license_expression` field in the PyPI API which is used by downstream tools including deps.dev * Update pyproject.toml * Remove superceded license classifier --------- Co-authored-by: Chris Adams --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 57c95a8e4..d112a5b00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ name = "django-haystack" description = "Pluggable search for Django." readme = "README.rst" authors = [{name = "Daniel Lindsley", email = "daniel@toastdriven.com"}] +license = "BSD-3-Clause" +license-files = ["LICENSE"] classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", @@ -20,7 +22,6 @@ classifiers = [ "Framework :: Django :: 5.0", "Framework :: Django :: 5.1", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3",