From a5d0061b1c1d9c362d31a6da3b7f0e26a648ae2f Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Thu, 12 Sep 2024 00:22:51 +0800 Subject: [PATCH 1/6] fix(#99): fix update value containing "where" cause exception. --- CHANGELOG.md | 4 +++ clickhouse_backend/backend/introspection.py | 4 +-- clickhouse_backend/driver/connection.py | 27 ++++++++++++++------- tests/queries/tests.py | 7 ++++++ 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a4549e..077ddb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### unreleased + +- fix: #99 fix update value containing "where" cause exception. + ### 1.3.0 - fix #92 last_executed_query() when params is a mappinglast_executed_query() when params is a mapping. diff --git a/clickhouse_backend/backend/introspection.py b/clickhouse_backend/backend/introspection.py index 2f8d80c..d38d3fb 100644 --- a/clickhouse_backend/backend/introspection.py +++ b/clickhouse_backend/backend/introspection.py @@ -12,10 +12,10 @@ TableInfo = namedtuple("TableInfo", BaseTableInfo._fields + ("comment",)) constraint_pattern = re.compile( - r"CONSTRAINT (`)?((?(1)(?:[^\\`]|\\.)+|\S+))(?(1)`|) (CHECK .+?),?\n" + r"CONSTRAINT (`)?((?(1)(?:\\.|[^`])+|\S+))(?(1)`|) (CHECK .+?),?\n" ) index_pattern = re.compile( - r"INDEX (`)?((?(1)(?:[^\\`]|\\.)+|\S+))(?(1)`|) (.+? TYPE ([a-zA-Z_][0-9a-zA-Z_]*)\(.+?\) GRANULARITY \d+)" + r"INDEX (`)?((?(1)(?:\\.|[^`])+|\S+))(?(1)`|) (.+? TYPE ([a-zA-Z_][0-9a-zA-Z_]*)\(.+?\) GRANULARITY \d+)" ) diff --git a/clickhouse_backend/driver/connection.py b/clickhouse_backend/driver/connection.py index e254ee9..f5ec18d 100644 --- a/clickhouse_backend/driver/connection.py +++ b/clickhouse_backend/driver/connection.py @@ -8,10 +8,10 @@ from .escape import escape_params from .pool import ClickhousePool -update_pattern = re.compile( - r"^\s*alter\s+table\s+(\S+)\s+.*?update.+?where\s+(.+?)(?:settings\s+.+)?$", - flags=re.IGNORECASE, -) +name_regex = r'"(?:[^"]|\\.)+"' +value_regex = r"(')?(?(1)(?:[^']|\\.)+|\S+)(?(1)'|)" +name_value_regex = f"{name_regex} = {value_regex}" +update_pattern = re.compile(f"^ALTER TABLE ({name_regex}) UPDATE ") def send_query(self, query, query_id=None, params=None): @@ -95,11 +95,20 @@ def execute(self, operation, parameters=None): query = self._client.substitute_params( operation, parameters, self._client.connection.context ) - table, where = update_pattern.match(query).groups() - super().execute(f"select count(*) from {table} where {where}") - (rowcount,) = self.fetchone() - self._reset_state() - self._rowcount = rowcount + m = update_pattern.match(query) + table = m.group(1) + query_upper = query.upper() + i = query_upper.rfind(" WHERE ") + if i > 0: + j = query_upper.rfind(" SETTINGS ", i + 7) + if j > 0: + where = query[i + 7 : j] + else: + where = query[i + 7 :] + super().execute(f"select count(*) from {table} where {where}") + (rowcount,) = self.fetchone() + self._reset_state() + self._rowcount = rowcount super().execute(operation, parameters) diff --git a/tests/queries/tests.py b/tests/queries/tests.py index de69cf7..aaed34b 100644 --- a/tests/queries/tests.py +++ b/tests/queries/tests.py @@ -20,3 +20,10 @@ def test_update(self): self.a1.save() with self.assertRaises(models.Author.MultipleObjectsReturned): self.a1.refresh_from_db() + + # regression test for https://github.com/jayvynl/django-clickhouse-backend/issues/99 + def test_update_special_string_val(self): + self.a1.name = "where **" + self.a1.save() + self.a1.refresh_from_db() + self.assertEqual(self.a1.name, "where **") From b9c750de10c2f7f367f88cd1d2472a4a508bfb66 Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Fri, 13 Sep 2024 00:23:31 +0800 Subject: [PATCH 2/6] test: fix unit tests for the latest version of clickhouse --- tests/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/settings.py b/tests/settings.py index d8b60c1..cab6302 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -29,6 +29,7 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, + "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster"}, From 5ff9a5ad421eb564125d468aa61669ed212c6c51 Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Fri, 13 Sep 2024 23:58:40 +0800 Subject: [PATCH 3/6] fix test case for json type, and add allow_experimental_object_type setting to json field document. --- docs/Fields.md | 3 +++ tests/settings.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docs/Fields.md b/docs/Fields.md index d6a69cf..6745ff2 100644 --- a/docs/Fields.md +++ b/docs/Fields.md @@ -646,6 +646,9 @@ Neither Nullable nor LowCardinality is supported. When query from the database, JSONField get dict or list. The JSON data type is an experimental feature. To use it, set `allow_experimental_object_type = 1` in the database settings. + +**Note:** From [ClickHouse 24.8 LTS](https://clickhouse.com/blog/clickhouse-release-24-08), set `allow_experimental_object_type = 1` to use JSON type. + For example: ```python diff --git a/tests/settings.py b/tests/settings.py index cab6302..c630c36 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -47,6 +47,7 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, + "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster", "managed": False}, @@ -64,6 +65,7 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, + "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster", "managed": False}, From 3705f61c1a2a228342f9628f08c82f159a46cb1b Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Sat, 14 Sep 2024 00:01:42 +0800 Subject: [PATCH 4/6] add allow_experimental_object_type setting to json field document. --- docs/Fields.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Fields.md b/docs/Fields.md index 6745ff2..c50468e 100644 --- a/docs/Fields.md +++ b/docs/Fields.md @@ -647,7 +647,7 @@ When query from the database, JSONField get dict or list. The JSON data type is an experimental feature. To use it, set `allow_experimental_object_type = 1` in the database settings. -**Note:** From [ClickHouse 24.8 LTS](https://clickhouse.com/blog/clickhouse-release-24-08), set `allow_experimental_object_type = 1` to use JSON type. +**Note:** From [ClickHouse 24.8 LTS](https://clickhouse.com/blog/clickhouse-release-24-08), set `allow_experimental_json_type = 1` to use JSON type. For example: From 7e7326e9dec6633a66b1e938347ed2f79227fddd Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Tue, 24 Sep 2024 23:14:05 +0800 Subject: [PATCH 5/6] fix: #97 JSONField error in ClickHouse 24.8 --- CHANGELOG.md | 3 ++- clickhouse_backend/backend/base.py | 2 +- docs/Fields.md | 2 ++ tests/clickhouse_fields/test_jsonfield.py | 2 +- tests/settings.py | 3 --- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 077ddb2..af598ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ### unreleased -- fix: #99 fix update value containing "where" cause exception. +- fix: #99 update value containing "where" cause exception. +- fix: #97 JSONField error in ClickHouse 24.8. ### 1.3.0 diff --git a/clickhouse_backend/backend/base.py b/clickhouse_backend/backend/base.py index 80f0221..72610bc 100644 --- a/clickhouse_backend/backend/base.py +++ b/clickhouse_backend/backend/base.py @@ -34,7 +34,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): "BigAutoField": "Int64", "IPAddressField": "IPv4", "GenericIPAddressField": "IPv6", - "JSONField": "JSON", + "JSONField": "Object('json')", "BinaryField": "String", "CharField": "FixedString(%(max_length)s)", "DateField": "Date32", diff --git a/docs/Fields.md b/docs/Fields.md index c50468e..7e32fa0 100644 --- a/docs/Fields.md +++ b/docs/Fields.md @@ -639,6 +639,8 @@ MapModel.objects.annotate( ### JSON +**Note:** Object('json') type [is not production-ready and is now deprecated](https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type). + Field importing path: `clickhouse_backend.models.JSONField`. Neither Nullable nor LowCardinality is supported. diff --git a/tests/clickhouse_fields/test_jsonfield.py b/tests/clickhouse_fields/test_jsonfield.py index 4cd8ed9..692adf0 100644 --- a/tests/clickhouse_fields/test_jsonfield.py +++ b/tests/clickhouse_fields/test_jsonfield.py @@ -5,7 +5,7 @@ from .models import JSONModel -class MapFieldTests(TestCase): +class JsonFieldTests(TestCase): def test_disallow_nullable(self): field = models.JSONField(null=True, name="field") self.assertEqual( diff --git a/tests/settings.py b/tests/settings.py index c630c36..d8b60c1 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -29,7 +29,6 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, - "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster"}, @@ -47,7 +46,6 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, - "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster", "managed": False}, @@ -65,7 +63,6 @@ "alter_sync": 2, "allow_suspicious_low_cardinality_types": 1, "allow_experimental_object_type": 1, - "allow_experimental_json_type": 1, }, }, "TEST": {"cluster": "cluster", "managed": False}, From 8621c8bbdd2903e16b7a29f05d5abebc0807d135 Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Tue, 24 Sep 2024 23:14:44 +0800 Subject: [PATCH 6/6] fix: tuple function error in ClickHouse 24.8 --- CHANGELOG.md | 1 + clickhouse_backend/models/fields/tuple.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af598ba..a4bdffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - fix: #99 update value containing "where" cause exception. - fix: #97 JSONField error in ClickHouse 24.8. +- fix: tuple function error in ClickHouse 24.8. ### 1.3.0 diff --git a/clickhouse_backend/models/fields/tuple.py b/clickhouse_backend/models/fields/tuple.py index 3a7728f..4150d43 100644 --- a/clickhouse_backend/models/fields/tuple.py +++ b/clickhouse_backend/models/fields/tuple.py @@ -151,7 +151,12 @@ def _convert_type(self, value): if value is None or isinstance(value, self.container_class): return value if self.is_named_tuple: + if isinstance(value, dict): + return self.container_class(**value) return self.container_class(*value) + # From ClickHouse server 24.8 LTS, tuple("a", "b") returns NamedTuple. + if isinstance(value, dict): + return self.container_class(value.values()) return self.container_class(value) @property