Feat JSONField (#21)

jayvynl · web-flow · commit 566188a400db · 2023-06-29T23:09:37.000+08:00
#17 JSON field support
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 - Fix [bug when save django model instance](https://github.com/jayvynl/django-clickhouse-backend/issues/9).
 - Support [clickhouse-driver 0.2.6](https://github.com/mymarilyn/clickhouse-driver), drop support for python3.6.
 - Support [Django 4.2](https://docs.djangoproject.com).
+- Support [clickhouse JSON type](https://clickhouse.com/docs/en/sql-reference/data-types/json).
 
 ### 1.0.2 (2023-02-28)
 - Fix test db name when NAME not provided in DATABASES setting.
diff --git a/clickhouse_backend/backend/base.py b/clickhouse_backend/backend/base.py
@@ -28,6 +28,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         "BigAutoField": "Int64",
         "IPAddressField": "IPv4",
         "GenericIPAddressField": "IPv6",
+        "JSONField": "JSON",
         "BinaryField": "String",
         "CharField": "FixedString(%(max_length)s)",
         "DateField": "Date32",
diff --git a/clickhouse_backend/backend/features.py b/clickhouse_backend/backend/features.py
@@ -100,7 +100,23 @@ def uses_savepoints(self):
     supports_partial_indexes = False
 
     # Does the backend support JSONField?
-    supports_json_field = False
+    supports_json_field = True
+    # Can the backend introspect a JSONField?
+    can_introspect_json_field = True
+    # Does the backend support primitives in JSONField?
+    supports_primitives_in_json_field = False
+    # Is there a true datatype for JSON?
+    has_native_json_field = True
+    # Does the backend use PostgreSQL-style JSON operators like '->'?
+    has_json_operators = False
+    # Does the backend support __contains and __contained_by lookups for
+    # a JSONField?
+    supports_json_field_contains = False
+    # Does value__d__contains={'f': 'g'} (without a list around the dict) match
+    # {'d': [{'f': 'g'}]}?
+    json_key_contains_list_matching_requires_list = False
+    # Does the backend support JSONObject() database function?
+    has_json_object_function = False
 
     # Does the backend support column collations?
     supports_collation_on_charfield = False
diff --git a/clickhouse_backend/backend/introspection.py b/clickhouse_backend/backend/introspection.py
@@ -39,6 +39,8 @@ def get_field_type(self, data_type, description):
             return "TupleField"
         elif data_type.startswith("Map"):
             return "MapField"
+        elif data_type == "Object('json')":
+            return "JSONField"
 
         return f"{data_type}Field"  # Int8
 
diff --git a/clickhouse_backend/backend/operations.py b/clickhouse_backend/backend/operations.py
@@ -1,9 +1,9 @@
-from django.conf import settings
 from django.db.backends.base.operations import BaseDatabaseOperations
 
 from clickhouse_backend import compat
-from clickhouse_backend.utils import get_timezone
+from clickhouse_backend.driver import JSON
 from clickhouse_backend.driver.client import insert_pattern
+from clickhouse_backend.utils import get_timezone
 
 
 class DatabaseOperations(BaseDatabaseOperations):
@@ -294,6 +294,9 @@ def adapt_datetimefield_value(self, value):
     def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
         return value
 
+    def adapt_json_value(self, value, encoder):
+        return JSON(value)
+
     def explain_query_prefix(self, format=None, **options):
         # bypass normal explain prefix insert in compiler.as_sql
         return ""
diff --git a/clickhouse_backend/driver/__init__.py b/clickhouse_backend/driver/__init__.py
@@ -9,7 +9,7 @@
 
 from .connection import Connection
 # Binary is compatible for django's BinaryField.
-from .types import Binary  # NOQA
+from .types import Binary, JSON  # NOQA
 
 
 def connect(dsn=None, host=None,
diff --git a/clickhouse_backend/driver/escape.py b/clickhouse_backend/driver/escape.py
@@ -61,7 +61,7 @@ def escape_param(item, context, for_server=False):
         return "[%s]" % ', '.join(str(escape_param(x, context, for_server=for_server)) for x in item)
 
     elif isinstance(item, tuple):
-        return "(%s)" % ', '.join(str(escape_param(x, context, for_server=for_server)) for x in item)
+        return "tuple(%s)" % ', '.join(str(escape_param(x, context, for_server=for_server)) for x in item)
 
     elif isinstance(item, Enum):
         return escape_param(item.value, context, for_server=for_server)
@@ -72,6 +72,15 @@ def escape_param(item, context, for_server=False):
     elif isinstance(item, types.Binary):
         return escape_binary(item, context)
 
+    elif isinstance(item, types.JSON):
+        value = item.value
+        if isinstance(value, list):
+            return escape_param([types.JSON(v) for v in value], context, for_server=for_server)
+        elif isinstance(value, dict):
+            return escape_param(tuple(types.JSON(v) for v in value.values()), context, for_server=for_server)
+        else:
+            return escape_param(value, context, for_server=for_server)
+
     else:
         return item
 
diff --git a/clickhouse_backend/driver/types.py b/clickhouse_backend/driver/types.py
@@ -1 +1,6 @@
 Binary = bytes
+
+
+class JSON:
+    def __init__(self, value):
+        self.value = value
diff --git a/clickhouse_backend/models/__init__.py b/clickhouse_backend/models/__init__.py
@@ -1,3 +1,4 @@
+from clickhouse_backend.patch import patch_all
 from .base import ClickhouseModel
 from .engines import *
 from .engines import __all__ as engines_all  # NOQA
@@ -15,3 +16,4 @@
     *fucntions_all,
     *indexes_all,
 ]
+patch_all()
diff --git a/clickhouse_backend/models/base.py b/clickhouse_backend/models/base.py
@@ -1,8 +1,6 @@
 from django.db import models
 from django.db.migrations import state
-from django.db.models import functions
 from django.db.models import options
-from django.db.models.manager import BaseManager
 
 from .query import QuerySet
 from .sql import Query
@@ -16,15 +14,6 @@
 state.DEFAULT_NAMES = options.DEFAULT_NAMES
 
 
-def as_clickhouse(self, compiler, connection, **extra_context):
-    return functions.Random.as_sql(
-        self, compiler, connection, function="rand64", **extra_context
-    )
-
-
-functions.Random.as_clickhouse = as_clickhouse
-
-
 class ClickhouseManager(models.Manager):
     _queryset_class = QuerySet
 
diff --git a/clickhouse_backend/models/fields/__init__.py b/clickhouse_backend/models/fields/__init__.py
@@ -13,6 +13,7 @@
 from .array import *
 from .base import FieldMixin
 from .integer import *
+from .json import *
 from .map import *
 from .tuple import *
 
@@ -30,7 +31,7 @@
     "DateField", "Date32Field", "DateTimeField", "DateTime64Field",
     "Enum8Field", "Enum16Field", "EnumField",
     "IPv4Field", "IPv6Field", "GenericIPAddressField",
-    "ArrayField", "TupleField", "MapField",
+    "ArrayField", "TupleField", "MapField", "JSONField",
 ]
 
 
diff --git a/clickhouse_backend/models/fields/base.py b/clickhouse_backend/models/fields/base.py
@@ -6,9 +6,8 @@ class FieldMixin:
     """All clickhouse field should inherit this mixin.
 
     1. Remove unsupported arguments: unique, db_index, unique_for_date,
-       unique_for_month, unique_for_year, db_tablespace.
+       unique_for_month, unique_for_year, db_tablespace, db_collation.
     2. Return shortened name in deconstruct method.
-    3. Add low_cardinality attribute, corresponding to clickhouse LowCardinality Data Type.
 
     low_cardinality argument is added separately in every specific field that support LowCardinality.
     If added in this mixin, then PyCharm will not supply argument hints.
diff --git a/clickhouse_backend/models/fields/json.py b/clickhouse_backend/models/fields/json.py
@@ -0,0 +1,37 @@
+from django.db.models.fields import json
+from django.db.models.fields import Field
+
+from clickhouse_backend.driver import JSON
+from .base import FieldMixin
+
+__all__ = ["JSONField"]
+
+
+class JSONField(FieldMixin, json.JSONField):
+    nullable_allowed = False
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        if path.startswith("clickhouse_backend.models.json"):
+            path = path.replace("clickhouse_backend.models.json", "clickhouse_backend.models")
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expression, connection):
+        return value
+
+    def get_prep_value(self, value):
+        # django 4.1 and below dumps value as json string.
+        return Field.get_prep_value(self, value)
+
+    def get_db_prep_value(self, value, connection, prepared=False):
+        value = super().get_db_prep_value(value, connection, prepared)
+        # django 4.1 and below does not call connection.ops.adapt_json_value
+        if not isinstance(value, JSON):
+            value = JSON(value)
+        return value
+
+    def get_db_prep_save(self, value, connection):
+        value = super().get_db_prep_save(value, connection)
+        if isinstance(value, JSON):
+            value = value.value
+        return value
diff --git a/clickhouse_backend/models/fields/map.py b/clickhouse_backend/models/fields/map.py
@@ -163,6 +163,8 @@ def get_db_prep_value(self, value, connection, prepared=False):
         return value
 
     def get_db_prep_save(self, value, connection):
+        if hasattr(value, "as_sql"):
+            return value
         if isinstance(value, collections.abc.Mapping):
             return {
                 self.key_field.get_db_prep_save(k, connection):
diff --git a/clickhouse_backend/models/fields/tuple.py b/clickhouse_backend/models/fields/tuple.py
@@ -230,6 +230,9 @@ def _from_db_value(self, value, expression, connection):
             return value
         self._validate_length(value)
         values = []
+        # when set allow_experimental_object_type=1 at session level, value will be a dict.
+        if isinstance(value, dict):
+            value = value.values()
         for i, field in zip(value, self._base_fields):
             if hasattr(field, "from_db_value"):
                 values.append(field.from_db_value(i, expression, connection))
diff --git a/clickhouse_backend/patch/__init__.py b/clickhouse_backend/patch/__init__.py
@@ -0,0 +1,7 @@
+from .fields import patch_jsonfield
+from .functions import patch_functions
+
+
+def patch_all():
+    patch_jsonfield()
+    patch_functions()
diff --git a/clickhouse_backend/patch/fields.py b/clickhouse_backend/patch/fields.py
@@ -0,0 +1,19 @@
+from django.db.models.fields import json
+
+
+def key_transform_as_clickhouse(self, compiler, connection):
+    lhs, params, key_transforms = self.preprocess_lhs(compiler, connection)
+    sql = lhs
+    params = list(params)
+    for key in key_transforms:
+        if key.isdigit():
+            sql = f"{sql}[%s]"
+            params.append(int(key) + 1)
+        else:
+            sql = f"tupleElement({sql}, %s)"
+            params.append(key)
+    return sql, tuple(params)
+
+
+def patch_jsonfield():
+    json.KeyTransform.as_clickhouse = key_transform_as_clickhouse
diff --git a/clickhouse_backend/patch/functions.py b/clickhouse_backend/patch/functions.py
@@ -0,0 +1,11 @@
+from django.db.models import functions
+
+
+def random_as_clickhouse(self, compiler, connection, **extra_context):
+    return functions.Random.as_sql(
+        self, compiler, connection, function="rand64", **extra_context
+    )
+
+
+def patch_functions():
+    functions.Random.as_clickhouse = random_as_clickhouse
diff --git a/docs/Fields.md b/docs/Fields.md
@@ -73,6 +73,7 @@ Supported date types are:
 - LowCardinality(T)
 - Tuple(T1, T2, ...)
 - Map(key, value)
+- JSON
 
 
 ### [U]Int(8|16|32|64|128|256)
@@ -551,7 +552,7 @@ Valid key fields are:
 - GenericIPAddressField
 
 
-When query from the database, MapFile get dict.
+When query from the database, MapField get dict.
 
 
 #### Lookups
@@ -567,10 +568,10 @@ class MapModel(models.ClickhouseModel):
 
 MapModel.objects.create(
     map={
-          "baidu": "39.156.66.10",
-          "bing.com": "13.107.21.200",
-          "google.com": "172.217.163.46"
-      }
+        "baidu": "39.156.66.10",
+        "bing.com": "13.107.21.200",
+        "google.com": "172.217.163.46"
+    }
 )
 ```
 
@@ -634,3 +635,74 @@ MapModel.objects.annotate(
 ).values('value')
 # <QuerySet [{'value': '::ffff:d6b:15c8'}]>
 ```
+
+
+### JSON
+
+Field importing path: `clickhouse_backend.models.JSONField`.
+
+Neither Nullable nor LowCardinality is supported.
+
+When query from the database, JSONField get dict or list.
+
+The JSON data type is an experimental feature. To use it, set `allow_experimental_object_type = 1` in the database settings.
+For example:
+
+```python
+DATABASES = {
+    'default': {
+        'ENGINE': 'clickhouse_backend.backend',
+        'OPTIONS': {
+            'settings': {
+                'allow_experimental_object_type': 1,
+            }
+        }
+    }
+}
+```
+
+#### Lookups
+
+Currently only key lookup is supported.
+
+```python
+from clickhouse_backend import models
+
+class JSONModel(models.ClickhouseModel):
+    json = models.JSONField()
+
+v = {'a': [1, 2, 3], 'b': [{'c': 1}, {'d': 2}], 'c': {'d': 'e'}}
+instance = JSONModel.objects.create(json=v)
+instance.refresh_from_db()
+instance.json
+# {'a': [1, 2, 3], 'b': [{'c': 1, 'd': 0}, {'c': 0, 'd': 2}], 'c': {'d': 'e'}}
+```
+
+**Note** JSONField value may change after saved to database. This is because clickhouse internally store [JSON](https://clickhouse.com/docs/en/sql-reference/data-types/json)
+as [Tuple](https://clickhouse.com/docs/en/sql-reference/data-types/tuple) and [Array](https://clickhouse.com/docs/en/sql-reference/data-types/array).
+[Clickhouse try best to store JSON in a uniform schema](https://clickhouse.com/docs/en/integrations/data-formats/json#handling-data-changes).
+Sometimes when you insert a JSON value that is not compatible with existing schema, clickhouse will fail.
+
+
+##### key
+
+Get the value of specific key.
+
+```python
+JSONModel.objects.values('json__a')
+# <QuerySet [{'json__a': [1, 2, 3]}]>
+JSONModel.objects.values('json__b__0__c')
+# <QuerySet [{'json__b__0__c': 1}]>
+JSONModel.objects.values('json__c__d')
+# <QuerySet [{'json__c__d': 'e'}]>
+JSONModel.objects.values('json__c')
+# <QuerySet [{'json__c': {'d': 'e'}}]>
+
+JSONModel.objects.filter(json__c={'any_key': 'e'}).exists()
+# True
+JSONModel.objects.filter(json__c=('e',)).exists()
+# True
+```
+
+Note the strange behaviors of the last two examples. That's because clickhouse store python dict as [named tuple](https://clickhouse.com/docs/en/sql-reference/data-types/tuple#addressing-tuple-elements),
+named tuples are just normal tuples when compared, the `name` is not taken into account.
diff --git a/example/config/settings.py b/example/config/settings.py
@@ -71,6 +71,11 @@
 DATABASES = {
     'default': {
         'ENGINE': 'clickhouse_backend.backend',
+        'OPTIONS': {
+            'settings': {
+                'allow_experimental_object_type': 1,
+            }
+        },
         'TEST': {
             'MIGRATE': False
         }
diff --git a/example/testapp/models.py b/example/testapp/models.py
diff --git a/tests/ch_fields/models.py b/tests/ch_fields/models.py
diff --git a/tests/ch_fields/test_jsonfield.py b/tests/ch_fields/test_jsonfield.py
diff --git a/tests/ch_fields/test_mapfield.py b/tests/ch_fields/test_mapfield.py
diff --git a/tests/settings.py b/tests/settings.py

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+from clickhouse_backend.patch import patch_all`
`1`	`2`	`from .base import ClickhouseModel`
`2`	`3`	`from .engines import *`
`3`	`4`	`from .engines import __all__ as engines_all # NOQA`
`@@ -15,3 +16,4 @@`
`15`	`16`	`*fucntions_all,`
`16`	`17`	`*indexes_all,`
`17`	`18`	`]`
	`19`	`+patch_all()`