From cc00d0eb60799f74cc97eeccf81c3d2c676c0415 Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Mon, 30 Dec 2024 21:41:27 +0800 Subject: [PATCH 1/5] feat: #108 Queryset.iterator use clickhouse_driver.Client.execute_iter --- clickhouse_backend/backend/base.py | 8 ++++++++ clickhouse_backend/driver/connection.py | 2 +- clickhouse_backend/driver/pool.py | 4 ++-- docs/Configurations.md | 3 +++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/clickhouse_backend/backend/base.py b/clickhouse_backend/backend/base.py index 72610bc..3e382fa 100644 --- a/clickhouse_backend/backend/base.py +++ b/clickhouse_backend/backend/base.py @@ -163,6 +163,8 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS): self.migration_cluster = self.settings_dict["OPTIONS"].pop( "migration_cluster", None ) + # https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results + self.max_block_size = self.settings_dict["OPTIONS"].pop("max_block_size", 65409) if not self.settings_dict["NAME"]: self.settings_dict["NAME"] = "default" @@ -224,6 +226,12 @@ def init_connection_state(self): def create_cursor(self, name=None): return self.connection.cursor() + @async_unsafe + def chunked_cursor(self): + cursor = self._cursor() + cursor.cursor.set_stream_results(True, self.max_block_size) + return cursor + def _savepoint(self, sid): pass diff --git a/clickhouse_backend/driver/connection.py b/clickhouse_backend/driver/connection.py index f5ec18d..766b1cf 100644 --- a/clickhouse_backend/driver/connection.py +++ b/clickhouse_backend/driver/connection.py @@ -116,9 +116,9 @@ class Connection(dbapi_connection.Connection): """Connection class with support for connection pool.""" def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) kwargs.setdefault("connections_min", 10) kwargs.setdefault("connections_max", 100) + super().__init__(*args, **kwargs) self.pool = ClickhousePool( dsn=self.dsn, host=self.host, diff --git a/clickhouse_backend/driver/pool.py b/clickhouse_backend/driver/pool.py index 11a009b..4c51cd4 100644 --- a/clickhouse_backend/driver/pool.py +++ b/clickhouse_backend/driver/pool.py @@ -17,8 +17,8 @@ def __init__(self, **kwargs): self.dsn = kwargs.pop("dsn", None) self.connections_min = kwargs.pop("connections_min", 10) self.connections_max = kwargs.pop("connections_max", 20) - - self.connection_args = {"host": kwargs.pop("host", "localhost"), **kwargs} + kwargs.setdefault("host", "localhost") + self.connection_args = kwargs self.closed = False self._pool = [] self._used = {} diff --git a/docs/Configurations.md b/docs/Configurations.md index 185a6ce..7ee7845 100644 --- a/docs/Configurations.md +++ b/docs/Configurations.md @@ -40,10 +40,13 @@ Migration table will be created on this cluster with [Distributed Engine](https: - `connections_min` is maximum number of connections can be kept in connection pool, default 10. Set this value to 0 will disable connection pool. - `connections_max` is maximum number of connections can be used, default 100. In fact, `connections_max` is maximum numbers of queries one can execute concurrently. Because [source code of DBAPI Connection](https://github.com/mymarilyn/clickhouse-driver/blob/0.2.5/clickhouse_driver/dbapi/connection.py#L46) shows that every cursor creates a new connection. +- `max_block_size` is used for [streaming results](https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results). - `dsn` provide connection url, for example `clickhouse://localhost/test?param1=value1&...`. If dsn is provided, all other connection parameters are ignored. - All other [clickhouse_driver.connection.Connection](https://clickhouse-driver.readthedocs.io/en/latest/api.html#connection) parameters. - `settings` can contain [clickhouse_driver.Client](https://clickhouse-driver.readthedocs.io/en/latest/api.html?highlight=client#clickhouse_driver.Client) settings and [clickhouse settings](https://clickhouse.com/docs/en/operations/settings/settings). +> *Changed in version 1.3.2:* Add `max_block_size`, refer [#108](https://github.com/jayvynl/django-clickhouse-backend/issues/108). + Valid `TEST` keys: - `managed`: whether create(`True`) test database or not(`False`), default `True`. From f22333e740c0d67bf3399e254aeb1f63c8052fa9 Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Mon, 30 Dec 2024 21:44:36 +0800 Subject: [PATCH 2/5] chore: test for python3.13 --- pyproject.toml | 2 ++ tox.ini | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 60fc426..73f3cc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Framework :: Django :: 4.0", "Framework :: Django :: 4.1", "Framework :: Django :: 4.2", + "Framework :: Django :: 5", "Framework :: Django :: 5.0", "Framework :: Django :: 5.1", "Intended Audience :: Developers", @@ -32,6 +33,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ "django>=3.2", diff --git a/tox.ini b/tox.ini index f94251f..5ba8e41 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] requires = tox>=4 -env_list = py3.7-django3.2, py{3.8,3.9,3.10,3.11,3.12}-django{3.2,4.0,4.1,4.2}, py{3.10,3.11,3.12}-django{5.0,5.1} +env_list = py3.7-django3.2, py{3.8,3.9,3.10,3.11,3.12,3.13}-django{3.2,4.0,4.1,4.2}, py{3.10,3.11,3.12,3.13}-django{5.0,5.1} [variables] code = clickhouse_backend example tests From 4fa008c51404bc4cba1911d2fed8be19c58e21cd Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Mon, 30 Dec 2024 21:50:46 +0800 Subject: [PATCH 3/5] chore: run isort -c in code lint --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 5ba8e41..d86c85a 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,9 @@ description = lint code skip_install = true deps = flake8 + isort==5.12.0 commands = + isort -c {[variables]code} flake8 --max-line-length=88 --extend-ignore=E203,E501 {[variables]code} [testenv:format] From ba279cb9ae2f6fccf400834059856005c399c3ca Mon Sep 17 00:00:00 2001 From: Lin Zhiwen Date: Mon, 30 Dec 2024 22:02:19 +0800 Subject: [PATCH 4/5] docs: 1.3.2 changelog --- CHANGELOG.md | 7 +++++++ CONTRIBUTING.md | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d6bb67..6e5fea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +### 1.3.2 + +- feat(aggragation-function): add anyLast function. +- fix: pass DSN to clickhouse-client if configured. +- feat: #108 Queryset.iterator use clickhouse_driver.Client.execute_iter. +- chore: test for python3.13. + ### 1.3.1 - fix: #99 update value containing "where" cause exception. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0c6e65..2d5a194 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,3 +98,7 @@ run test for all supported Python version and Django version: ```shell tox ``` + +### Other + +- Don't forget writing [Changelog](CHANGELOG.md) From 232c4efb0da679f8956a8fc046a5778eeab49b7f Mon Sep 17 00:00:00 2001 From: jayvynl <34599950+jayvynl@users.noreply.github.com> Date: Tue, 31 Dec 2024 20:53:09 +0800 Subject: [PATCH 5/5] Update tox.ini --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d86c85a..e0e7433 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ description = lint code skip_install = true deps = flake8 - isort==5.12.0 + isort commands = isort -c {[variables]code} flake8 --max-line-length=88 --extend-ignore=E203,E501 {[variables]code} @@ -35,7 +35,7 @@ description = format code skip_install = true deps = black==23.7.0 - isort==5.12.0 + isort commands = isort {[variables]code} black -t py37 -t py38 -t py39 -t py310 -t py311 -t py312 {[variables]code}