Merge branch 'jayvynl:main' into main

Author: RuslanSibgatulin

CHANGELOG.md

@@ -1,3 +1,10 @@
+### 1.3.2
+
+- feat(aggragation-function): add anyLast function.
+- fix: pass DSN to clickhouse-client if configured.
+- feat: #108 Queryset.iterator use clickhouse_driver.Client.execute_iter.
+- chore: test for python3.13.
+
 ### 1.3.1
 
 - fix: #99 update value containing "where" cause exception.

CONTRIBUTING.md

@@ -98,3 +98,7 @@ run test for all supported Python version and Django version:
 ```shell
 tox
 ```
+
+### Other
+
+- Don't forget writing [Changelog](CHANGELOG.md)

clickhouse_backend/backend/base.py

@@ -163,6 +163,8 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS):
         self.migration_cluster = self.settings_dict["OPTIONS"].pop(
             "migration_cluster", None
         )
+        # https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results
+        self.max_block_size = self.settings_dict["OPTIONS"].pop("max_block_size", 65409)
         if not self.settings_dict["NAME"]:
             self.settings_dict["NAME"] = "default"
 
@@ -224,6 +226,12 @@ def init_connection_state(self):
     def create_cursor(self, name=None):
         return self.connection.cursor()
 
+    @async_unsafe
+    def chunked_cursor(self):
+        cursor = self._cursor()
+        cursor.cursor.set_stream_results(True, self.max_block_size)
+        return cursor
+
     def _savepoint(self, sid):
         pass
 

clickhouse_backend/backend/client.py

@@ -17,19 +17,23 @@ def settings_to_cmd_args_env(cls, settings_dict, parameters):
         user = settings_dict.get("USER")
         passwd = settings_dict.get("PASSWORD")
         secure = options.get("secure")
+        dsn = options.get("dsn")
 
-        if host:
-            args += ["-h", host]
-        if port:
-            args += ["--port", str(port)]
-        if user:
-            args += ["-u", user]
-        if passwd:
-            args += ["--password", passwd]
-        if dbname:
-            args += ["-d", dbname]
-        if secure:
-            args += ["--secure"]
+        if dsn:
+            args += [dsn]
+        else:
+            if host:
+                args += ["-h", host]
+            if port:
+                args += ["--port", str(port)]
+            if user:
+                args += ["-u", user]
+            if passwd:
+                args += ["--password", passwd]
+            if dbname:
+                args += ["-d", dbname]
+            if secure:
+                args += ["--secure"]
         args.extend(parameters)
         return args, None
 

clickhouse_backend/driver/connection.py

@@ -116,9 +116,9 @@ class Connection(dbapi_connection.Connection):
     """Connection class with support for connection pool."""
 
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
         kwargs.setdefault("connections_min", 10)
         kwargs.setdefault("connections_max", 100)
+        super().__init__(*args, **kwargs)
         self.pool = ClickhousePool(
             dsn=self.dsn,
             host=self.host,

clickhouse_backend/driver/pool.py

@@ -17,8 +17,8 @@ def __init__(self, **kwargs):
         self.dsn = kwargs.pop("dsn", None)
         self.connections_min = kwargs.pop("connections_min", 10)
         self.connections_max = kwargs.pop("connections_max", 20)
-
-        self.connection_args = {"host": kwargs.pop("host", "localhost"), **kwargs}
+        kwargs.setdefault("host", "localhost")
+        self.connection_args = kwargs
         self.closed = False
         self._pool = []
         self._used = {}

docs/Configurations.md

@@ -40,10 +40,13 @@ Migration table will be created on this cluster with [Distributed Engine](https:
 - `connections_min` is maximum number of connections can be kept in connection pool, default 10. Set this value to 0 will disable connection pool.
 - `connections_max` is maximum number of connections can be used, default 100. In fact, `connections_max` is maximum numbers of queries one can execute concurrently.
 Because [source code of DBAPI Connection](https://github.com/mymarilyn/clickhouse-driver/blob/0.2.5/clickhouse_driver/dbapi/connection.py#L46) shows that every cursor creates a new connection.
+- `max_block_size` is used for [streaming results](https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results).
 - `dsn` provide connection url, for example `clickhouse://localhost/test?param1=value1&...`. If dsn is provided, all other connection parameters are ignored.
 - All other [clickhouse_driver.connection.Connection](https://clickhouse-driver.readthedocs.io/en/latest/api.html#connection) parameters.
 - `settings` can contain [clickhouse_driver.Client](https://clickhouse-driver.readthedocs.io/en/latest/api.html?highlight=client#clickhouse_driver.Client) settings and [clickhouse settings](https://clickhouse.com/docs/en/operations/settings/settings).
 
+> *Changed in version 1.3.2:* Add `max_block_size`, refer [#108](https://github.com/jayvynl/django-clickhouse-backend/issues/108).
+
 Valid `TEST` keys:
 
 - `managed`: whether create(`True`) test database or not(`False`), default `True`.

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
     "Framework :: Django :: 4.0",
     "Framework :: Django :: 4.1",
     "Framework :: Django :: 4.2",
+    "Framework :: Django :: 5",
     "Framework :: Django :: 5.0",
     "Framework :: Django :: 5.1",
     "Intended Audience :: Developers",
@@ -32,6 +33,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
     "django>=3.2",

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 requires =
     tox>=4
-env_list = py3.7-django3.2, py{3.8,3.9,3.10,3.11,3.12}-django{3.2,4.0,4.1,4.2}, py{3.10,3.11,3.12}-django{5.0,5.1}
+env_list = py3.7-django3.2, py{3.8,3.9,3.10,3.11,3.12,3.13}-django{3.2,4.0,4.1,4.2}, py{3.10,3.11,3.12,3.13}-django{5.0,5.1}
 
 [variables]
 code = clickhouse_backend example tests
@@ -25,15 +25,17 @@ description = lint code
 skip_install = true
 deps =
     flake8
+    isort
 commands =
+    isort -c {[variables]code}
     flake8 --max-line-length=88 --extend-ignore=E203,E501 {[variables]code}
 
 [testenv:format]
 description = format code
 skip_install = true
 deps =
     black==23.7.0
-    isort==5.12.0
+    isort
 commands =
     isort {[variables]code}
     black -t py37 -t py38 -t py39 -t py310 -t py311 -t py312 {[variables]code}