diff --git a/CHANGES.txt b/CHANGES.txt index 4f58c8d2..d04a31a2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,16 @@ Unreleased ========== +2023/07/17 0.33.0 +================= + +- SQLAlchemy: Rename leftover occurrences of ``Object``. The new symbol to represent + CrateDB's ``OBJECT`` column type is now ``ObjectType``. + +- SQLAlchemy DQL: Use CrateDB's native ``ILIKE`` operator instead of using SA's + generic implementation ``lower() LIKE lower()``. Thanks, @hlcianfagna. + + 2023/07/06 0.32.0 ================= diff --git a/docs/by-example/sqlalchemy/working-with-types.rst b/docs/by-example/sqlalchemy/working-with-types.rst index bcddf8f8..169acede 100644 --- a/docs/by-example/sqlalchemy/working-with-types.rst +++ b/docs/by-example/sqlalchemy/working-with-types.rst @@ -7,7 +7,7 @@ SQLAlchemy: Working with special CrateDB types This section of the documentation shows how to work with special data types from the CrateDB SQLAlchemy dialect. Currently, these are: -- Container types ``Object`` and ``ObjectArray``. +- Container types ``ObjectType`` and ``ObjectArray``. - Geospatial types ``Geopoint`` and ``Geoshape``. @@ -33,7 +33,7 @@ Import the relevant symbols: ... except ImportError: ... from sqlalchemy.ext.declarative import declarative_base >>> from uuid import uuid4 - >>> from crate.client.sqlalchemy.types import Object, ObjectArray + >>> from crate.client.sqlalchemy.types import ObjectType, ObjectArray >>> from crate.client.sqlalchemy.types import Geopoint, Geoshape Establish a connection to the database, see also :ref:`sa:engines_toplevel` @@ -53,9 +53,9 @@ Introduction to container types In a document oriented database, it is a common pattern to store objects within a single field. For such cases, the CrateDB SQLAlchemy dialect provides the -``Object`` and ``ObjectArray`` types. +``ObjectType`` and ``ObjectArray`` types. -The ``Object`` type effectively implements a dictionary- or map-like type. The +The ``ObjectType`` type effectively implements a dictionary- or map-like type. The ``ObjectArray`` type maps to a Python list of dictionaries. For exercising those features, let's define a schema using SQLAlchemy's @@ -69,15 +69,15 @@ For exercising those features, let's define a schema using SQLAlchemy's ... id = sa.Column(sa.String, primary_key=True, default=gen_key) ... name = sa.Column(sa.String) ... quote = sa.Column(sa.String) - ... details = sa.Column(Object) + ... details = sa.Column(ObjectType) ... more_details = sa.Column(ObjectArray) In CrateDB's SQL dialect, those container types map to :ref:`crate-reference:type-object` and :ref:`crate-reference:type-array`. -``Object`` -========== +``ObjectType`` +============== Let's add two records which have additional items within the ``details`` field. Note that item keys have not been defined in the DDL schema, effectively @@ -113,7 +113,7 @@ A subsequent select query will see all the records: [('Arthur Dent', 'male'), ('Tricia McMillan', 'female')] It is also possible to just select a part of the document, even inside the -``Object`` type: +``ObjectType`` type: >>> sorted(session.query(Character.details['gender']).all()) [('female',), ('male',)] @@ -129,7 +129,7 @@ Update dictionary ----------------- The SQLAlchemy CrateDB dialect supports change tracking deep down the nested -levels of a ``Object`` type field. For example, the following query will only +levels of a ``ObjectType`` type field. For example, the following query will only update the ``gender`` key. The ``species`` key which is on the same level will be left untouched. @@ -170,7 +170,7 @@ Refresh and query "characters" table: ``ObjectArray`` =============== -Note that opposed to the ``Object`` type, the ``ObjectArray`` type isn't smart +Note that opposed to the ``ObjectType`` type, the ``ObjectArray`` type isn't smart and doesn't have intelligent change tracking. Therefore, the generated ``UPDATE`` statement will affect the whole list: diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index 2c1a7471..c3d0c7af 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -201,7 +201,7 @@ system `: ... name = sa.Column(sa.String, crate_index=False) ... name_normalized = sa.Column(sa.String, sa.Computed("lower(name)")) ... quote = sa.Column(sa.String, nullable=False) - ... details = sa.Column(types.Object) + ... details = sa.Column(types.ObjectType) ... more_details = sa.Column(types.ObjectArray) ... name_ft = sa.Column(sa.String) ... quote_ft = sa.Column(sa.String) @@ -224,7 +224,7 @@ In this example, we: - Disable indexing of the ``name`` column using ``crate_index=False`` - Define a computed column ``name_normalized`` (based on ``name``) that translates into a generated column -- Use the `Object`_ extension type for the ``details`` column +- Use the `ObjectType`_ extension type for the ``details`` column - Use the `ObjectArray`_ extension type for the ``more_details`` column - Set up the ``name_ft`` and ``quote_ft`` fulltext indexes, but exclude them from the mapping (so SQLAlchemy doesn't try to update them as if they were columns) @@ -314,9 +314,10 @@ dialect provides. The appendix has a full :ref:`data types reference `. .. _object: +.. _objecttype: -``Object`` -.......... +``ObjectType`` +.............. Objects are a common, and useful, data type when using CrateDB, so the CrateDB SQLAlchemy dialect provides a custom ``Object`` type extension for working with @@ -355,7 +356,7 @@ insert two records: .. NOTE:: - Behind the scenes, if you update an ``Object`` property and ``commit`` that + Behind the scenes, if you update an ``ObjectType`` property, and ``commit`` that change, the :ref:`UPDATE ` statement sent to CrateDB will only include the data necessary to update the changed sub-columns. @@ -365,7 +366,7 @@ insert two records: ``ObjectArray`` ............... -In addition to the `Object`_ type, the CrateDB SQLAlchemy dialect also provides +In addition to the `ObjectType`_ type, the CrateDB SQLAlchemy dialect also provides an ``ObjectArray`` type, which is structured as a :class:`py:list` of :class:`dictionaries `. @@ -386,7 +387,7 @@ The resulting object will look like this: .. CAUTION:: - Behind the scenes, if you update an ``ObjectArray`` and ``commit`` that + Behind the scenes, if you update an ``ObjectArray``, and ``commit`` that change, the :ref:`UPDATE ` statement sent to CrateDB will include all of the ``ObjectArray`` data. @@ -468,12 +469,12 @@ Here's what a regular select might look like: [('Arthur Dent', 'male'), ('Tricia McMillan', 'female')] You can also select a portion of each record, and this even works inside -`Object`_ columns: +`ObjectType`_ columns: >>> sorted(session.query(Character.details['gender']).all()) [('female',), ('male',)] -You can also filter on attributes inside the `Object`_ column: +You can also filter on attributes inside the `ObjectType`_ column: >>> query = session.query(Character.name) >>> query.filter(Character.details['gender'] == 'male').all() diff --git a/src/crate/client/__init__.py b/src/crate/client/__init__.py index 604331c2..bf1c1648 100644 --- a/src/crate/client/__init__.py +++ b/src/crate/client/__init__.py @@ -29,7 +29,7 @@ # version string read from setup.py using a regex. Take care not to break the # regex! -__version__ = "0.32.0" +__version__ = "0.33.0" apilevel = "2.0" threadsafety = 2 diff --git a/src/crate/client/sqlalchemy/compiler.py b/src/crate/client/sqlalchemy/compiler.py index 3965c9e1..3ae7a7cb 100644 --- a/src/crate/client/sqlalchemy/compiler.py +++ b/src/crate/client/sqlalchemy/compiler.py @@ -244,6 +244,49 @@ def visit_any(self, element, **kw): self.process(element.right, **kw) ) + def visit_ilike_case_insensitive_operand(self, element, **kw): + """ + Use native `ILIKE` operator, like PostgreSQL's `PGCompiler`. + """ + if self.dialect.has_ilike_operator(): + return element.element._compiler_dispatch(self, **kw) + else: + return super().visit_ilike_case_insensitive_operand(element, **kw) + + def visit_ilike_op_binary(self, binary, operator, **kw): + """ + Use native `ILIKE` operator, like PostgreSQL's `PGCompiler`. + + Do not implement the `ESCAPE` functionality, because it is not + supported by CrateDB. + """ + if binary.modifiers.get("escape", None) is not None: + raise NotImplementedError("Unsupported feature: ESCAPE is not supported") + if self.dialect.has_ilike_operator(): + return "%s ILIKE %s" % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw), + ) + else: + return super().visit_ilike_op_binary(binary, operator, **kw) + + def visit_not_ilike_op_binary(self, binary, operator, **kw): + """ + Use native `ILIKE` operator, like PostgreSQL's `PGCompiler`. + + Do not implement the `ESCAPE` functionality, because it is not + supported by CrateDB. + """ + if binary.modifiers.get("escape", None) is not None: + raise NotImplementedError("Unsupported feature: ESCAPE is not supported") + if self.dialect.has_ilike_operator(): + return "%s NOT ILIKE %s" % ( + self.process(binary.left, **kw), + self.process(binary.right, **kw), + ) + else: + return super().visit_not_ilike_op_binary(binary, operator, **kw) + def limit_clause(self, select, **kw): """ Generate OFFSET / LIMIT clause, PostgreSQL-compatible. diff --git a/src/crate/client/sqlalchemy/dialect.py b/src/crate/client/sqlalchemy/dialect.py index e992d41a..3f1197df 100644 --- a/src/crate/client/sqlalchemy/dialect.py +++ b/src/crate/client/sqlalchemy/dialect.py @@ -33,7 +33,7 @@ ) from crate.client.exceptions import TimezoneUnawareException from .sa_version import SA_VERSION, SA_1_4, SA_2_0 -from .types import Object, ObjectArray +from .types import ObjectType, ObjectArray TYPES_MAP = { "boolean": sqltypes.Boolean, @@ -41,7 +41,7 @@ "smallint": sqltypes.SmallInteger, "timestamp": sqltypes.TIMESTAMP, "timestamp with time zone": sqltypes.TIMESTAMP, - "object": Object, + "object": ObjectType, "integer": sqltypes.Integer, "long": sqltypes.NUMERIC, "bigint": sqltypes.NUMERIC, @@ -356,6 +356,13 @@ def _create_column_info(self, row): def _resolve_type(self, type_): return TYPES_MAP.get(type_, sqltypes.UserDefinedType) + def has_ilike_operator(self): + """ + Only CrateDB 4.1.0 and higher implements the `ILIKE` operator. + """ + server_version_info = self.server_version_info + return server_version_info is not None and server_version_info >= (4, 1, 0) + class DateTrunc(functions.GenericFunction): name = "date_trunc" diff --git a/src/crate/client/sqlalchemy/tests/__init__.py b/src/crate/client/sqlalchemy/tests/__init__.py index 3c032ebb..6102cb5a 100644 --- a/src/crate/client/sqlalchemy/tests/__init__.py +++ b/src/crate/client/sqlalchemy/tests/__init__.py @@ -2,6 +2,7 @@ from ..compat.api13 import monkeypatch_amend_select_sa14, monkeypatch_add_connectionfairy_driver_connection from ..sa_version import SA_1_4, SA_VERSION +from ...test_util import ParametrizedTestCase # `sql.select()` of SQLAlchemy 1.3 uses old calling semantics, # but the test cases already need the modern ones. @@ -32,6 +33,9 @@ def test_suite_unit(): tests.addTest(makeSuite(SqlAlchemyDictTypeTest)) tests.addTest(makeSuite(SqlAlchemyDateAndDateTimeTest)) tests.addTest(makeSuite(SqlAlchemyCompilerTest)) + tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": None})) + tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": (4, 0, 12)})) + tests.addTest(ParametrizedTestCase.parametrize(SqlAlchemyCompilerTest, param={"server_version_info": (4, 1, 10)})) tests.addTest(makeSuite(SqlAlchemyUpdateTest)) tests.addTest(makeSuite(SqlAlchemyMatchTest)) tests.addTest(makeSuite(SqlAlchemyCreateTableTest)) diff --git a/src/crate/client/sqlalchemy/tests/compiler_test.py b/src/crate/client/sqlalchemy/tests/compiler_test.py index 17612232..5d5cc89e 100644 --- a/src/crate/client/sqlalchemy/tests/compiler_test.py +++ b/src/crate/client/sqlalchemy/tests/compiler_test.py @@ -18,8 +18,8 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. - -from unittest import mock, TestCase, skipIf +from textwrap import dedent +from unittest import mock, skipIf from crate.client.sqlalchemy.compiler import crate_before_execute @@ -28,12 +28,16 @@ from crate.client.sqlalchemy.sa_version import SA_VERSION, SA_1_4, SA_2_0 from crate.client.sqlalchemy.types import ObjectType +from crate.client.test_util import ParametrizedTestCase -class SqlAlchemyCompilerTest(TestCase): +class SqlAlchemyCompilerTest(ParametrizedTestCase): def setUp(self): self.crate_engine = sa.create_engine('crate://') + if isinstance(self.param, dict) and "server_version_info" in self.param: + server_version_info = self.param["server_version_info"] + self.crate_engine.dialect.server_version_info = server_version_info self.sqlite_engine = sa.create_engine('sqlite://') self.metadata = sa.MetaData() self.mytable = sa.Table('mytable', self.metadata, @@ -71,6 +75,65 @@ def test_bulk_update_on_builtin_type(self): self.assertFalse(hasattr(clauseelement, '_crate_specific')) + def test_select_with_ilike_no_escape(self): + """ + Verify the compiler uses CrateDB's native `ILIKE` method. + """ + selectable = self.mytable.select().where(self.mytable.c.name.ilike("%foo%")) + statement = str(selectable.compile(bind=self.crate_engine)) + if self.crate_engine.dialect.has_ilike_operator(): + self.assertEqual(statement, dedent(""" + SELECT mytable.name, mytable.data + FROM mytable + WHERE mytable.name ILIKE ? + """).strip()) # noqa: W291 + else: + self.assertEqual(statement, dedent(""" + SELECT mytable.name, mytable.data + FROM mytable + WHERE lower(mytable.name) LIKE lower(?) + """).strip()) # noqa: W291 + + def test_select_with_not_ilike_no_escape(self): + """ + Verify the compiler uses CrateDB's native `ILIKE` method. + """ + selectable = self.mytable.select().where(self.mytable.c.name.notilike("%foo%")) + statement = str(selectable.compile(bind=self.crate_engine)) + if SA_VERSION < SA_1_4 or not self.crate_engine.dialect.has_ilike_operator(): + self.assertEqual(statement, dedent(""" + SELECT mytable.name, mytable.data + FROM mytable + WHERE lower(mytable.name) NOT LIKE lower(?) + """).strip()) # noqa: W291 + else: + self.assertEqual(statement, dedent(""" + SELECT mytable.name, mytable.data + FROM mytable + WHERE mytable.name NOT ILIKE ? + """).strip()) # noqa: W291 + + def test_select_with_ilike_and_escape(self): + """ + Verify the compiler fails when using CrateDB's native `ILIKE` method together with `ESCAPE`. + """ + + selectable = self.mytable.select().where(self.mytable.c.name.ilike("%foo%", escape='\\')) + with self.assertRaises(NotImplementedError) as cmex: + selectable.compile(bind=self.crate_engine) + self.assertEqual(str(cmex.exception), "Unsupported feature: ESCAPE is not supported") + + @skipIf(SA_VERSION < SA_1_4, "SQLAlchemy 1.3 and earlier do not support native `NOT ILIKE` compilation") + def test_select_with_not_ilike_and_escape(self): + """ + Verify the compiler fails when using CrateDB's native `ILIKE` method together with `ESCAPE`. + """ + + selectable = self.mytable.select().where(self.mytable.c.name.notilike("%foo%", escape='\\')) + with self.assertRaises(NotImplementedError) as cmex: + selectable.compile(bind=self.crate_engine) + self.assertEqual(str(cmex.exception), "Unsupported feature: ESCAPE is not supported") + def test_select_with_offset(self): """ Verify the `CrateCompiler.limit_clause` method, with offset. diff --git a/src/crate/client/sqlalchemy/tests/create_table_test.py b/src/crate/client/sqlalchemy/tests/create_table_test.py index b7fb9b87..4c6072aa 100644 --- a/src/crate/client/sqlalchemy/tests/create_table_test.py +++ b/src/crate/client/sqlalchemy/tests/create_table_test.py @@ -25,7 +25,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base -from crate.client.sqlalchemy.types import Object, ObjectArray, Geopoint +from crate.client.sqlalchemy.types import ObjectType, ObjectArray, Geopoint from crate.client.cursor import Cursor from unittest import TestCase @@ -76,7 +76,7 @@ def test_column_obj(self): class DummyTable(self.Base): __tablename__ = 'dummy' pk = sa.Column(sa.String, primary_key=True) - obj_col = sa.Column(Object) + obj_col = sa.Column(ObjectType) self.Base.metadata.create_all(bind=self.engine) fake_cursor.execute.assert_called_with( ('\nCREATE TABLE dummy (\n\tpk STRING NOT NULL, \n\tobj_col OBJECT, ' diff --git a/src/crate/client/sqlalchemy/tests/dialect_test.py b/src/crate/client/sqlalchemy/tests/dialect_test.py index 6e1581d7..bdcfc838 100644 --- a/src/crate/client/sqlalchemy/tests/dialect_test.py +++ b/src/crate/client/sqlalchemy/tests/dialect_test.py @@ -28,7 +28,7 @@ from crate.client.cursor import Cursor from crate.client.sqlalchemy import SA_VERSION from crate.client.sqlalchemy.sa_version import SA_1_4, SA_2_0 -from crate.client.sqlalchemy.types import Object +from crate.client.sqlalchemy.types import ObjectType from sqlalchemy import inspect from sqlalchemy.orm import Session try: @@ -67,7 +67,7 @@ class Character(self.base): name = sa.Column(sa.String, primary_key=True) age = sa.Column(sa.Integer, primary_key=True) - obj = sa.Column(Object) + obj = sa.Column(ObjectType) ts = sa.Column(sa.DateTime, onupdate=datetime.utcnow) self.session = Session(bind=self.engine) diff --git a/src/crate/client/sqlalchemy/tests/query_caching.py b/src/crate/client/sqlalchemy/tests/query_caching.py index 037d6423..43e28a44 100644 --- a/src/crate/client/sqlalchemy/tests/query_caching.py +++ b/src/crate/client/sqlalchemy/tests/query_caching.py @@ -34,7 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base -from crate.client.sqlalchemy.types import Object, ObjectArray +from crate.client.sqlalchemy.types import ObjectType, ObjectArray class SqlAlchemyQueryCompilationCaching(TestCase): @@ -55,7 +55,7 @@ class Character(Base): __tablename__ = 'characters' name = sa.Column(sa.String, primary_key=True) age = sa.Column(sa.Integer) - data = sa.Column(Object) + data = sa.Column(ObjectType) data_list = sa.Column(ObjectArray) return Character diff --git a/src/crate/client/sqlalchemy/tests/update_test.py b/src/crate/client/sqlalchemy/tests/update_test.py index 00aeef0a..a2d5462b 100644 --- a/src/crate/client/sqlalchemy/tests/update_test.py +++ b/src/crate/client/sqlalchemy/tests/update_test.py @@ -23,7 +23,7 @@ from unittest import TestCase from unittest.mock import patch, MagicMock -from crate.client.sqlalchemy.types import Object +from crate.client.sqlalchemy.types import ObjectType import sqlalchemy as sa from sqlalchemy.orm import Session @@ -52,7 +52,7 @@ class Character(self.base): name = sa.Column(sa.String, primary_key=True) age = sa.Column(sa.Integer) - obj = sa.Column(Object) + obj = sa.Column(ObjectType) ts = sa.Column(sa.DateTime, onupdate=datetime.utcnow) self.character = Character diff --git a/src/crate/client/test_util.py b/src/crate/client/test_util.py index 90379a79..823a44e3 100644 --- a/src/crate/client/test_util.py +++ b/src/crate/client/test_util.py @@ -18,6 +18,7 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. +import unittest class ClientMocked(object): @@ -42,3 +43,27 @@ def set_next_server_infos(self, server, server_name, version): def close(self): pass + + +class ParametrizedTestCase(unittest.TestCase): + """ + TestCase classes that want to be parametrized should + inherit from this class. + + https://eli.thegreenplace.net/2011/08/02/python-unit-testing-parametrized-test-cases + """ + def __init__(self, methodName="runTest", param=None): + super(ParametrizedTestCase, self).__init__(methodName) + self.param = param + + @staticmethod + def parametrize(testcase_klass, param=None): + """ Create a suite containing all tests taken from the given + subclass, passing them the parameter 'param'. + """ + testloader = unittest.TestLoader() + testnames = testloader.getTestCaseNames(testcase_klass) + suite = unittest.TestSuite() + for name in testnames: + suite.addTest(testcase_klass(name, param=param)) + return suite