Add tests for backends and fix bugs that find by them. Remove redundant code. Correct database features.

Author: jayvynl

CHANGELOG.md

@@ -1,3 +1,11 @@
+0.2.1 (2022-10-30)
+---
+
+- Add tests for backends.
+- Remove redundant code.
+- Correct database features.
+- Fix bugs that find by new tests.
+
 0.2.0 (2022-10-26)
 ---
 

README.md

@@ -1,7 +1,7 @@
 Django ClickHouse Database Backend
 ===
 
-[中文文档](README_cn.md)
+[中文文档](https://github.com/jayvynl/django-clickhouse-backend/blob/main/README_cn.md)
 
 Django clickhouse backend is a [django database backend](https://docs.djangoproject.com/en/4.1/ref/databases/) for 
 [clickhouse](https://clickhouse.com/docs/en/home/) database. This project allows using django ORM to interact with 
@@ -26,7 +26,7 @@ Get started
 ### Installation
 
 ```shell
-pip install git+https://github.com/jayvynl/django-clickhouse-backend
+pip install django-clickhouse-backend
 ```
 
 or

clickhouse_backend/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

clickhouse_backend/backend/base.py

@@ -14,6 +14,7 @@
 from .introspection import DatabaseIntrospection  # NOQA
 from .operations import DatabaseOperations  # NOQA
 from .schema import DatabaseSchemaEditor  # NOQA
+from django.utils.functional import cached_property
 
 
 class DatabaseWrapper(BaseDatabaseWrapper):
@@ -58,8 +59,8 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         'lte': '<= %s',
         'startswith': 'LIKE %s',
         'endswith': 'LIKE %s',
-        'istartswith': 'LIKE UPPER(%s)',
-        'iendswith': 'LIKE UPPER(%s)',
+        'istartswith': 'ILIKE %s',
+        'iendswith': 'ILIKE %s',
     }
 
     # The patterns below are used to generate SQL pattern lookup clauses when
@@ -73,11 +74,11 @@ class DatabaseWrapper(BaseDatabaseWrapper):
     pattern_esc = r"replaceAll(replaceAll(replaceAll({}, '\\', '\\\\'), '%', '\\%'), '_', '\\_')"
     pattern_ops = {
         'contains': "LIKE '%%' || {} || '%%'",
-        'icontains': "LIKE '%%' || UPPER({}) || '%%'",
+        'icontains': "ILIKE '%%' || {} || '%%'",
         'startswith': "LIKE {} || '%%'",
-        'istartswith': "LIKE UPPER({}) || '%%'",
+        'istartswith': "ILIKE {} || '%%'",
         'endswith': "LIKE '%%' || {}",
-        'iendswith': "LIKE '%%' || UPPER({})",
+        'iendswith': "ILIKE '%%' || {}",
     }
 
     Database = Database
@@ -151,17 +152,24 @@ def _set_autocommit(self, autocommit):
 
     def is_usable(self):
         try:
-            # Use a psycopg cursor directly, bypassing Django's utilities.
+            # Use a clickhouse_driver cursor directly, bypassing Django's utilities.
             with self.connection.cursor() as cursor:
                 cursor.execute('SELECT 1')
         except Database.Error:
             return False
         else:
             return True
 
-    def make_debug_cursor(self, cursor):
-        return CursorDebugWrapper(cursor, self)
-
-
-class CursorDebugWrapper(BaseCursorDebugWrapper):
-    pass
+    @cached_property
+    def ch_version(self):
+        with self.temporary_connection() as cursor:
+            cursor.execute("SELECT version()")
+            row = cursor.fetchone()
+        return row[0]
+
+    def get_database_version(self):
+        """
+        Return a tuple of the database's version.
+        E.g. for ch_version "22.9.3.18", return (22, 9, 3, 18).
+        """
+        return tuple(map(int, self.ch_version.split(".")))

clickhouse_backend/backend/creation.py

@@ -10,41 +10,37 @@ class DatabaseCreation(BaseDatabaseCreation):
     def _quote_name(self, name):
         return self.connection.ops.quote_name(name)
 
-    def _get_database_create_suffix(self, engine=None):
-        suffix = ""
-        if engine:
-            suffix += "ENGINE = {}".format(engine)
-        return suffix
-
     def sql_table_creation_suffix(self):
-        test_settings = self.connection.settings_dict['TEST']
-        return self._get_database_create_suffix(
-            engine=test_settings.get('ENGINE'),
-        )
+        test_settings = self.connection.settings_dict["TEST"]
+        engine = test_settings.get("ENGINE")
+        if engine:
+            return "ENGINE = %s" % engine
+        return ""
 
     def _database_exists(self, cursor, database_name):
-        cursor.execute('SELECT 1 FROM system.databases WHERE name = %s', [strip_quotes(database_name)])
+        cursor.execute(
+            "SELECT 1 FROM system.databases WHERE name = %s",
+            [strip_quotes(database_name)]
+        )
         return cursor.fetchone() is not None
 
     def create_test_db(self, verbosity=1, autoclobber=False, serialize=True, keepdb=False):
         super().create_test_db(verbosity, autoclobber, serialize, keepdb)
-        self.connection.settings_dict.setdefault(
-            'fake_transaction',
-            self.connection.settings_dict['TEST'].get('fake_transaction', False)
-        )
-        self.connection.fake_transaction = self.connection.settings_dict['fake_transaction']
+        test_settings = self.connection.settings_dict["TEST"]
+        if "fake_transaction" in test_settings:
+            self.connection.fake_transaction = test_settings["fake_transaction"]
 
     def _execute_create_test_db(self, cursor, parameters, keepdb=False):
         try:
-            if keepdb and self._database_exists(cursor, parameters['dbname']):
+            if keepdb and self._database_exists(cursor, parameters["dbname"]):
                 # If the database should be kept and it already exists, don't
                 # try to create a new one.
                 return
             super()._execute_create_test_db(cursor, parameters, keepdb)
         except Exception as e:
-            if not e.args or getattr(e.args[0], 'code', '') != ErrorCodes.DATABASE_ALREADY_EXISTS:
+            if not e.args or getattr(e.args[0], "code", "") != ErrorCodes.DATABASE_ALREADY_EXISTS:
                 # All errors except "database already exists" cancel tests.
-                self.log('Got an error creating the test database: %s' % e)
+                self.log("Got an error creating the test database: %s" % e)
                 sys.exit(2)
             elif not keepdb:
                 # If the database should be kept, ignore "database already

clickhouse_backend/backend/features.py

@@ -4,6 +4,8 @@
 
 
 class DatabaseFeatures(BaseDatabaseFeatures):
+    # TODO: figure out compatible clickhouse version.
+    minimum_database_version = None
     # Use this class attribute control whether using fake transaction.
     # Fake transaction is used in test, prevent other database such as postgresql
     # from flush at the end of each testcase. Only use this feature when you are
@@ -17,33 +19,31 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     # https://clickhouse.com/docs/en/sql-reference/data-types/string/
     # allows_group_by_lob = True
 
-    # In clickhouse_backend MergeTree table family, PK have different meaning as in RDBMS
-    # PK is used for efficient range scans, pk can be duplicated, no auto incr pk.
-    # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#primary-keys-and-indexes-in-queries
-    allows_group_by_pk = False
-    allows_group_by_selected_pks = False
-
     # There is no unique constraint in clickhouse_backend.
     # supports_nullable_unique_constraints = True
     # supports_partially_nullable_unique_constraints = True
-    supports_deferrable_unique_constraints = True
+    # supports_deferrable_unique_constraints = False
 
-    # Transaction is not supported, and limited transaction is under development.
+    # Clickhouse only supports limited transaction.
     # https://clickhouse.com/docs/en/sql-reference/ansi/
     # https://github.com/ClickHouse/ClickHouse/issues/32513
+    # https://clickhouse.com/docs/en/guides/developer/transactional
     @cached_property
     def uses_savepoints(self):
         return self.fake_transaction
+
     can_release_savepoints = False
 
     # Is there a true datatype for uuid?
     has_native_uuid_field = True
 
     # Clickhouse use re2 syntax which does not support backreference.
+    # https://clickhouse.com/docs/en/sql-reference/functions/string-search-functions#matchhaystack-pattern
+    # https://github.com/google/re2/wiki/Syntax
     supports_regex_backreferencing = False
 
     # Can date/datetime lookups be performed using a string?
-    supports_date_lookup_using_string = False
+    supports_date_lookup_using_string = True
 
     # Confirm support for introspected foreign keys
     # Every database can do this reliably, except MySQL,
@@ -57,32 +57,27 @@ def uses_savepoints(self):
         'BinaryField': 'BinaryField',
         'BooleanField': 'BooleanField',
         'CharField': 'CharField',
-        'DurationField': 'DurationField',
         'GenericIPAddressField': 'GenericIPAddressField',
         'IntegerField': 'IntegerField',
         'PositiveBigIntegerField': 'PositiveBigIntegerField',
         'PositiveIntegerField': 'PositiveIntegerField',
         'PositiveSmallIntegerField': 'PositiveSmallIntegerField',
         'SmallIntegerField': 'SmallIntegerField',
-        'TimeField': 'TimeField',
     }
 
     # https://clickhouse.com/docs/en/sql-reference/statements/alter/index/
     # Index manipulation is supported only for tables with *MergeTree* engine (including replicated variants).
     supports_index_column_ordering = False
 
     # Does the backend support introspection of materialized views?
-    can_introspect_materialized_views = False
+    can_introspect_materialized_views = True
 
     # Support for the DISTINCT ON clause
     can_distinct_on_fields = True
 
     # Does the backend prevent running SQL queries in broken transactions?
     atomic_transactions = False
 
-    # Does it support operations requiring references rename in a transaction?
-    supports_atomic_references_rename = False
-
     # Can we issue more than one ALTER COLUMN clause in an ALTER TABLE?
     supports_combined_alters = True
 
@@ -93,14 +88,11 @@ def uses_savepoints(self):
     supports_column_check_constraints = False
     supports_table_check_constraints = True
     # Does the backend support introspection of CHECK constraints?
-    can_introspect_check_constraints = False
+    can_introspect_check_constraints = True
 
     # What kind of error does the backend throw when accessing closed cursor?
     closed_cursor_error_class = InterfaceError
 
-    # Does 'a' LIKE 'A' match?
-    has_case_insensitive_like = False
-
     # https://clickhouse.com/docs/en/sql-reference/statements/insert-into/#constraints
     supports_ignore_conflicts = False
 
@@ -119,11 +111,6 @@ def uses_savepoints(self):
     # SQL template override for tests.aggregation.tests.NowUTC
     test_now_utc_template = 'now64()'
 
-    @cached_property
-    def supports_explaining_query_execution(self):
-        """Does this backend support explaining query execution?"""
-        return True
-
     @cached_property
     def supports_transactions(self):
         return self.fake_transaction

clickhouse_backend/backend/introspection.py

@@ -17,7 +17,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
     # Maps type codes to Django Field types.
     data_types_reverse = {
         # 'String': 'BinaryField',
-        # The best type for String is BinaryField, but sometime you may need TextField.
+        # The best type for String is BinaryField, but sometimes you may need TextField.
         'String': 'TextField',
         'Int64': 'BigIntegerField',
         'Int16': 'SmallIntegerField',
@@ -27,7 +27,7 @@ class DatabaseIntrospection(BaseDatabaseIntrospection):
         'UInt32': 'PositiveIntegerField',
         'Float32': 'FloatField',
         'Float64': 'FloatField',
-        'IPv4': 'IPAddressField',
+        'IPv4': 'GenericIPAddressField',
         'IPv6': 'GenericIPAddressField',
         'Date': 'DateField',
         'Date32': 'DateField',
@@ -61,23 +61,18 @@ def get_table_list(self, cursor):
 
     def get_table_description(self, cursor, table_name):
         """
-        Return a description of the table with the DB-API cursor.description
-        interface.
+        Return a description of the table.
         """
-        # Query the pg_catalog tables as cursor.description does not reliably
-        # return the nullable property and information_schema.columns does not
-        # contain details of materialized views.
+        # Query the INFORMATION_SCHEMA.COLUMNS table.
         cursor.execute("""
             SELECT column_name, data_type, NULL, character_maximum_length,
             coalesce(numeric_precision, datetime_precision),
-            numeric_scale, is_nullable, column_default
+            numeric_scale, is_nullable, column_default, NULL
             FROM INFORMATION_SCHEMA.COLUMNS
             WHERE table_catalog = currentDatabase() AND table_name = %s
         """, [table_name])
         return [
-            FieldInfo(
-                *line, None
-            )
+            FieldInfo(*line)
             for line in cursor.fetchall()
         ]
 
@@ -118,7 +113,10 @@ def get_constraints(self, cursor, table_name):
         return constraints
 
     @cached_property
-    def settings(self):
+    def settings(self) -> set:
+        """
+        Get all available settings.
+        """
         with self.connection.cursor() as cursor:
             cursor.execute("SELECT name from system.settings")
             rows = cursor.fetchall()

clickhouse_backend/backend/operations.py

@@ -2,6 +2,7 @@
 
 from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
+
 from clickhouse_backend import compat
 
 
@@ -211,10 +212,6 @@ def lookup_cast(self, lookup_type, internal_type=None):
             else:
                 lookup = "CAST(%s, 'Nullable(String)')"
 
-        # Use UPPER(x) for case-insensitive lookups; it"s faster.
-        if lookup_type in ("iexact", "icontains", "istartswith", "iendswith"):
-            lookup = "UPPER(%s)" % lookup
-
         return lookup
 
     def max_name_length(self):
@@ -352,6 +349,13 @@ def savepoint_rollback_sql(self, sid):
             return "SELECT 1"
         return super().savepoint_rollback_sql(sid)
 
+    def last_executed_query(self, cursor, sql, params):
+        if sql.startswith("INSERT INTO"):
+            return "%s %s" % (sql, ", ".join(map(str, params)))
+        if params:
+            return sql % params
+        return sql
+
     def settings_sql(self, **settings):
         result = []
         params = []