feat: #108 Queryset.iterator use clickhouse_driver.Client.execute_iter

Author: jayvynl

clickhouse_backend/backend/base.py

@@ -163,6 +163,8 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS):
         self.migration_cluster = self.settings_dict["OPTIONS"].pop(
             "migration_cluster", None
         )
+        # https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results
+        self.max_block_size = self.settings_dict["OPTIONS"].pop("max_block_size", 65409)
         if not self.settings_dict["NAME"]:
             self.settings_dict["NAME"] = "default"
 
@@ -224,6 +226,12 @@ def init_connection_state(self):
     def create_cursor(self, name=None):
         return self.connection.cursor()
 
+    @async_unsafe
+    def chunked_cursor(self):
+        cursor = self._cursor()
+        cursor.cursor.set_stream_results(True, self.max_block_size)
+        return cursor
+
     def _savepoint(self, sid):
         pass
 

clickhouse_backend/driver/connection.py

@@ -116,9 +116,9 @@ class Connection(dbapi_connection.Connection):
     """Connection class with support for connection pool."""
 
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
         kwargs.setdefault("connections_min", 10)
         kwargs.setdefault("connections_max", 100)
+        super().__init__(*args, **kwargs)
         self.pool = ClickhousePool(
             dsn=self.dsn,
             host=self.host,

clickhouse_backend/driver/pool.py

@@ -17,8 +17,8 @@ def __init__(self, **kwargs):
         self.dsn = kwargs.pop("dsn", None)
         self.connections_min = kwargs.pop("connections_min", 10)
         self.connections_max = kwargs.pop("connections_max", 20)
-
-        self.connection_args = {"host": kwargs.pop("host", "localhost"), **kwargs}
+        kwargs.setdefault("host", "localhost")
+        self.connection_args = kwargs
         self.closed = False
         self._pool = []
         self._used = {}

docs/Configurations.md

@@ -40,10 +40,13 @@ Migration table will be created on this cluster with [Distributed Engine](https:
 - `connections_min` is maximum number of connections can be kept in connection pool, default 10. Set this value to 0 will disable connection pool.
 - `connections_max` is maximum number of connections can be used, default 100. In fact, `connections_max` is maximum numbers of queries one can execute concurrently.
 Because [source code of DBAPI Connection](https://github.com/mymarilyn/clickhouse-driver/blob/0.2.5/clickhouse_driver/dbapi/connection.py#L46) shows that every cursor creates a new connection.
+- `max_block_size` is used for [streaming results](https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results).
 - `dsn` provide connection url, for example `clickhouse://localhost/test?param1=value1&...`. If dsn is provided, all other connection parameters are ignored.
 - All other [clickhouse_driver.connection.Connection](https://clickhouse-driver.readthedocs.io/en/latest/api.html#connection) parameters.
 - `settings` can contain [clickhouse_driver.Client](https://clickhouse-driver.readthedocs.io/en/latest/api.html?highlight=client#clickhouse_driver.Client) settings and [clickhouse settings](https://clickhouse.com/docs/en/operations/settings/settings).
 
+> *Changed in version 1.3.2:* Add `max_block_size`, refer [#108](https://github.com/jayvynl/django-clickhouse-backend/issues/108).
+
 Valid `TEST` keys:
 
 - `managed`: whether create(`True`) test database or not(`False`), default `True`.