Pulling in new changes from parent

.github/workflows/base-test.yml

@@ -8,16 +8,29 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.12"]
-        django-version: ["3.2", "4.0", "4.1", "4.2", "5.0", "5.1"]
+        python-version: ["3.10", "3.13"]
+        django-version: ["3.2", "4.0", "4.1", "4.2", "5.0", "5.1", "5.2"]
         clickhouse-version: ["23.8", "latest"]
         include:
           - python-version: "3.7"
             django-version: "3.2"
+          - python-version: "3.8"
+            django-version: "3.2"
+          - python-version: "3.8"
+            django-version: "4.0"
+          - python-version: "3.8"
+            django-version: "4.1"
+          - python-version: "3.8"
+            django-version: "4.2"
+        exclude:
+          - python-version: "3.13"
+            django-version: "3.2"
+          - python-version: "3.13"
+            django-version: "4.0"
 
     name: ClickHouse${{ matrix.clickhouse-version }} Python${{ matrix.python-version }} Django${{ matrix.django-version }}
     steps:

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: python3 -m build
     - name: Store the distribution packages
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -43,7 +43,7 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
@@ -64,12 +64,12 @@ jobs:
 
     steps:
     - name: Download all the dists
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: python-package-distributions
         path: dist/
     - name: Sign the dists with Sigstore
-      uses: sigstore/gh-action-sigstore-python@v2.1.1
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
       with:
         inputs: >-
           ./dist/*.tar.gz

CHANGELOG.md

@@ -1,3 +1,22 @@
+### 1.5.0
+- feat: #133: Fix simultaneous queries error when iteration is interrupted
+- feat: #130: Add `distributed_migrations` database setting to support distributed migration queries.
+- feat: #129: Add `toYearWeek` datetime functionality
+
+### 1.4.0
+
+- feat: #119 Allow query results returned in columns and deserialized to `numpy` objects
+- feat: #125 Add database functions `toStartOfMinute`, `toStartOfFiveMinutes`, `toStartOfTenMinutes`, `toStartOfFifteenMinutes` and `toStartofHour`
+- feat: #122 Django 5.2 Support
+
+### 1.3.2
+
+- feat(aggragation-function): add anyLast function.
+- fix: pass DSN to clickhouse-client if configured.
+- feat: #108 Queryset.iterator use clickhouse_driver.Client.execute_iter.
+- chore: test for python3.13.
+- refactor: Using collections.abc.Iterable instead of deprecated django.utils.itercompat.is_iterable
+
 ### 1.3.1
 
 - fix: #99 update value containing "where" cause exception.

CONTRIBUTING.md

@@ -98,3 +98,7 @@ run test for all supported Python version and Django version:
 ```shell
 tox
 ```
+
+### Other
+
+- Don't forget writing [Changelog](CHANGELOG.md)

README.md

@@ -29,6 +29,7 @@ Read [Documentation](https://github.com/jayvynl/django-clickhouse-backend/blob/m
 - Support most clickhouse data types.
 - Support [SETTINGS in SELECT Query](https://clickhouse.com/docs/en/sql-reference/statements/select/#settings-in-select-query).
 - Support [PREWHERE clause](https://clickhouse.com/docs/en/sql-reference/statements/select/prewhere).
+- Support query results returned in columns and [deserialized to `numpy` objects](https://clickhouse-driver.readthedocs.io/en/latest/features.html#numpy-pandas-support).
 
 **Notes:**
 
@@ -381,6 +382,60 @@ and [distributed table engine](https://clickhouse.com/docs/en/engines/table-engi
 The following example assumes that a cluster defined by [docker compose in this repository](https://github.com/jayvynl/django-clickhouse-backend/blob/main/compose.yaml) is used.
 This cluster name is `cluster`, it has 2 shards, every shard has 2 replica.
 
+Query results returned as columns and/or deserialized into `numpy` objects 
+---
+
+`clickhouse-driver` allows results to be returned as columns and/or deserialized into
+`numpy` objects. This backend supports both options by using the context manager, 
+`Cursor.set_query_execution_args()`.
+
+```python
+import numpy as np
+from django.db import connection
+
+sql = """
+    SELECT toDateTime32('2022-01-01 01:00:05', 'UTC'), number, number*2.5
+    FROM system.numbers
+    LIMIT 3
+"""
+with connection.cursor() as cursorWrapper:
+    with cursorWrapper.cursor.set_query_execution_args(
+        columnar=True, use_numpy=True
+    ) as cursor:
+        cursor.execute(sql)
+        np.testing.assert_equal(
+            cursor.fetchall(),
+            [
+                np.array(
+                    [
+                        np.datetime64("2022-01-01T01:00:05"),
+                        np.datetime64("2022-01-01T01:00:05"),
+                        np.datetime64("2022-01-01T01:00:05"),
+                    ],
+                    dtype="datetime64[s]",
+                ),
+                np.array([0, 1, 2], dtype=np.uint64),
+                np.array([0, 2.5, 5.0], dtype=np.float64),
+            ],
+        )
+
+        cursor.execute(sql)
+        np.testing.assert_equal(
+            cursor.fetchmany(2),
+            [
+                np.array(
+                    [
+                        np.datetime64("2022-01-01T01:00:05"),
+                        np.datetime64("2022-01-01T01:00:05"),
+                        np.datetime64("2022-01-01T01:00:05"),
+                    ],
+                    dtype="datetime64[s]",
+                ),
+                np.array([0, 1, 2], dtype=np.uint64),
+            ],
+        )
+```
+
 ### Configuration
 
 ```python
@@ -470,6 +525,34 @@ Extra settings explanation:
   Do not hardcode database name when you define replicated table or distributed table.
   Because test database name is different from deployed database name.
 
+#### Clickhouse cluster behind a load balancer
+
+If your clickhouse cluster is running behind a load balancer, you can optionally set `distributed_migrations` to `True` under database OPTIONS. 
+Then a distributed migration table will be created on all nodes of the cluster, and all migration operations will be performed on this 
+distributed migrations table instead of a local migrations table. Otherwise, sequentially running migrations will have no effect on other nodes.
+
+Configuration example:
+
+```python
+DATABASES = {
+    "default": {
+        "HOST": "clickhouse-load-balancer",
+        "PORT": 9000,
+        "ENGINE": "clickhouse_backend.backend",
+        "OPTIONS": {
+            "migration_cluster": "cluster",
+            "distributed_migrations": True,
+            "settings": {
+                "mutations_sync": 2,
+                "insert_distributed_sync": 1,
+                "insert_quorum": 2,
+                "alter_sync": 2,
+            },
+        },
+    }
+}
+```
+
 ### Model
 
 `cluster` in `Meta` class will make models being created on cluster.

clickhouse-config/node1/remote-servers.xml

@@ -7,23 +7,27 @@
         <replica>
           <host>node1</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node2</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
       <shard>
         <internal_replication>true</internal_replication>
         <replica>
           <host>node3</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node4</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
     </cluster>
   </remote_servers>
-</clickhouse>
+</clickhouse>

clickhouse-config/node2/remote-servers.xml

@@ -7,23 +7,27 @@
         <replica>
           <host>node1</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node2</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
       <shard>
         <internal_replication>true</internal_replication>
         <replica>
           <host>node3</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node4</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
     </cluster>
   </remote_servers>
-</clickhouse>
+</clickhouse>

clickhouse-config/node3/remote-servers.xml

@@ -7,23 +7,27 @@
         <replica>
           <host>node1</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node2</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
       <shard>
         <internal_replication>true</internal_replication>
         <replica>
           <host>node3</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node4</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
     </cluster>
   </remote_servers>
-</clickhouse>
+</clickhouse>

clickhouse-config/node4/remote-servers.xml

@@ -7,23 +7,27 @@
         <replica>
           <host>node1</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node2</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
       <shard>
         <internal_replication>true</internal_replication>
         <replica>
           <host>node3</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
         <replica>
           <host>node4</host>
           <port>9000</port>
+          <password>clickhouse_password</password>
         </replica>
       </shard>
     </cluster>
   </remote_servers>
-</clickhouse>
+</clickhouse>

clickhouse_backend/__init__.py

@@ -1,5 +1,5 @@
 from clickhouse_backend.utils.version import get_version
 
-VERSION = (1, 3, 1, "final", 0)
+VERSION = (1, 4, 0, "final", 0)
 
 __version__ = get_version(VERSION)

clickhouse_backend/backend/base.py

@@ -163,6 +163,11 @@ def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS):
         self.migration_cluster = self.settings_dict["OPTIONS"].pop(
             "migration_cluster", None
         )
+        self.distributed_migrations = self.settings_dict["OPTIONS"].pop(
+            "distributed_migrations", None
+        )
+        # https://clickhouse-driver.readthedocs.io/en/latest/quickstart.html#streaming-results
+        self.max_block_size = self.settings_dict["OPTIONS"].pop("max_block_size", 65409)
         if not self.settings_dict["NAME"]:
             self.settings_dict["NAME"] = "default"
 
@@ -224,6 +229,12 @@ def init_connection_state(self):
     def create_cursor(self, name=None):
         return self.connection.cursor()
 
+    @async_unsafe
+    def chunked_cursor(self):
+        cursor = self._cursor()
+        cursor.cursor.set_stream_results(True, self.max_block_size)
+        return cursor
+
     def _savepoint(self, sid):
         pass
 

clickhouse_backend/backend/client.py

@@ -17,19 +17,23 @@ def settings_to_cmd_args_env(cls, settings_dict, parameters):
         user = settings_dict.get("USER")
         passwd = settings_dict.get("PASSWORD")
         secure = options.get("secure")
+        dsn = options.get("dsn")
 
-        if host:
-            args += ["-h", host]
-        if port:
-            args += ["--port", str(port)]
-        if user:
-            args += ["-u", user]
-        if passwd:
-            args += ["--password", passwd]
-        if dbname:
-            args += ["-d", dbname]
-        if secure:
-            args += ["--secure"]
+        if dsn:
+            args += [dsn]
+        else:
+            if host:
+                args += ["-h", host]
+            if port:
+                args += ["--port", str(port)]
+            if user:
+                args += ["-u", user]
+            if passwd:
+                args += ["--password", passwd]
+            if dbname:
+                args += ["-d", dbname]
+            if secure:
+                args += ["--secure"]
         args.extend(parameters)
         return args, None