From 7ef18a017a8199937c495e6ac315a4853b73a083 Mon Sep 17 00:00:00 2001 From: Andrew Speed Date: Thu, 29 May 2025 14:05:31 +0100 Subject: [PATCH 1/4] Add additional clickhouse database datetime functions Specifically, toStartOfMinute, toStartOfFiveMinutes, toStartOfTenminutes, toStartOfFifteenMinutes and toStartOfHour --- .../models/functions/datetime.py | 38 +++++ tests/clickhouse_functions/test_datetime.py | 130 +++++++++++++++++- 2 files changed, 164 insertions(+), 4 deletions(-) diff --git a/clickhouse_backend/models/functions/datetime.py b/clickhouse_backend/models/functions/datetime.py index 4929b11..9dd204a 100644 --- a/clickhouse_backend/models/functions/datetime.py +++ b/clickhouse_backend/models/functions/datetime.py @@ -6,6 +6,11 @@ from .base import Func __all__ = [ + "toStartOfMinute", + "toStartOfFiveMinutes", + "toStartOfTenMinutes", + "toStartOfFifteenMinutes", + "toStartOfHour", "toYYYYMM", "toYYYYMMDD", "toYYYYMMDDhhmmss", @@ -39,3 +44,36 @@ class toYYYYMMDD(toYYYYMM): class toYYYYMMDDhhmmss(toYYYYMM): output_field = fields.UInt64Field() + + +class toStartOfMinute(Func): + output_field = models.fields.DateTimeField() + + def __init__(self, *expressions): + arity = len(expressions) + if arity < 1 or arity > 1: + raise TypeError( + "'%s' takes 1 argument (%s given)" + % ( + self.__class__.__name__, + len(expressions), + ) + ) + + super().__init__(*expressions) + + +class toStartOfFiveMinutes(toStartOfMinute): + pass + + +class toStartOfTenMinutes(toStartOfMinute): + pass + + +class toStartOfFifteenMinutes(toStartOfMinute): + pass + + +class toStartOfHour(toStartOfMinute): + pass diff --git a/tests/clickhouse_functions/test_datetime.py b/tests/clickhouse_functions/test_datetime.py index 6e598b0..36e6cda 100644 --- a/tests/clickhouse_functions/test_datetime.py +++ b/tests/clickhouse_functions/test_datetime.py @@ -17,13 +17,15 @@ def setUpTestData(cls): alias="smithj", # https://stackoverflow.com/a/18862958 birthday=pytz.timezone(get_timezone()).localize( - datetime(2023, 11, 30, 16), is_dst=False + datetime(2023, 11, 30, hour=16, minute=12, second=15), is_dst=False ), ) cls.elena = Author.objects.create( name="Élena Jordan", alias="elena", - birthday=pytz.utc.localize(datetime(2023, 11, 30, 16), is_dst=False), + birthday=pytz.utc.localize( + datetime(2023, 11, 30, hour=16, minute=59, second=59), is_dst=False + ), ) def test_yyyymm(self): @@ -50,8 +52,128 @@ def test_yyyymmddhhmmss(self): john = Author.objects.annotate(v=models.toYYYYMMDDhhmmss("birthday")).get( id=self.john.id ) - self.assertEqual(john.v, 20231130160000) + self.assertEqual(john.v, 20231130161215) elena = Author.objects.annotate( v=models.toYYYYMMDDhhmmss("birthday", "Asia/Shanghai") ).get(id=self.elena.id) - self.assertEqual(elena.v, 20231201000000) + self.assertEqual(elena.v, 20231201005959) + + def test_tostartofminute(self): + john = Author.objects.annotate(v=models.toStartOfMinute("birthday")).get( + id=self.john.id + ) + self.assertEqual( + john.v, + datetime( + 2023, + 11, + 30, + hour=16, + minute=12, + second=00, + ), + ) + + elena = Author.objects.annotate(v=models.toStartOfMinute("birthday")).get( + id=self.elena.id + ) + self.assertEqual( + elena.v, + datetime(2023, 11, 30, hour=10, minute=59, second=00), + ) + + def test_tostartoffiveminutes(self): + john = Author.objects.annotate(v=models.toStartOfFiveMinutes("birthday")).get( + id=self.john.id + ) + self.assertEqual( + john.v, + datetime( + 2023, + 11, + 30, + hour=16, + minute=10, + second=00, + ), + ) + + elena = Author.objects.annotate(v=models.toStartOfFiveMinutes("birthday")).get( + id=self.elena.id + ) + self.assertEqual( + elena.v, + datetime(2023, 11, 30, hour=10, minute=55, second=00), + ) + + def test_tostartoftenminutes(self): + john = Author.objects.annotate(v=models.toStartOfTenMinutes("birthday")).get( + id=self.john.id + ) + self.assertEqual( + john.v, + datetime( + 2023, + 11, + 30, + hour=16, + minute=10, + second=00, + ), + ) + + elena = Author.objects.annotate(v=models.toStartOfTenMinutes("birthday")).get( + id=self.elena.id + ) + self.assertEqual( + elena.v, + datetime(2023, 11, 30, hour=10, minute=50, second=00), + ) + + def test_tostartoffifteenminutes(self): + john = Author.objects.annotate( + v=models.toStartOfFifteenMinutes("birthday") + ).get(id=self.john.id) + self.assertEqual( + john.v, + datetime( + 2023, + 11, + 30, + hour=16, + minute=00, + second=00, + ), + ) + + elena = Author.objects.annotate( + v=models.toStartOfFifteenMinutes("birthday") + ).get(id=self.elena.id) + self.assertEqual( + elena.v, + datetime(2023, 11, 30, hour=10, minute=45, second=00), + ) + + def test_tostartofhour(self): + john = Author.objects.annotate(v=models.toStartOfHour("birthday")).get( + id=self.john.id + ) + self.assertEqual( + john.v, + datetime( + 2023, + 11, + 30, + hour=16, + minute=00, + second=00, + ), + ) + + elena = Author.objects.annotate(v=models.toStartOfHour("birthday")).get( + id=self.elena.id + ) + self.assertEqual( + elena.v, + datetime(2023, 11, 30, hour=10, minute=00, second=00), + ) From ae4c6b8ac51480c297d349ea44a19e4bac0100eb Mon Sep 17 00:00:00 2001 From: Andrew Speed Date: Thu, 29 May 2025 14:21:09 +0100 Subject: [PATCH 2/4] Add changelog entry --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index adda7f1..c3b88c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +### 1.5.0 + +- feat: #125 Add database functions `toStartOfMinute`, `toStartOfFiveMinutes`, `toStartOfTenMinutes`, `toStartOfFifteenMinutes` and `toStartofHour` + + ### 1.4.0 - feat: #119 Allow query results returned in columns and deserialized to `numpy` objects From 6e6acea0da6e25c4cc98f00e72daf92001d68b49 Mon Sep 17 00:00:00 2001 From: Andrew Speed Date: Mon, 2 Jun 2025 17:46:21 +0100 Subject: [PATCH 3/4] Update CHANGELOG --- CHANGELOG.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b88c6..5401934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,7 @@ -### 1.5.0 - -- feat: #125 Add database functions `toStartOfMinute`, `toStartOfFiveMinutes`, `toStartOfTenMinutes`, `toStartOfFifteenMinutes` and `toStartofHour` - - ### 1.4.0 - feat: #119 Allow query results returned in columns and deserialized to `numpy` objects +- feat: #125 Add database functions `toStartOfMinute`, `toStartOfFiveMinutes`, `toStartOfTenMinutes`, `toStartOfFifteenMinutes` and `toStartofHour` ### 1.3.2 From 3e8a49af9ffb4d33d1a13d830672f467ccb79d5d Mon Sep 17 00:00:00 2001 From: Andrew Speed Date: Mon, 2 Jun 2025 17:47:58 +0100 Subject: [PATCH 4/4] Set explicit password for default user in attempt to fix intermittent CI failures on latest Suggestion from https://stackoverflow.com/a/71802893 --- clickhouse-config/node1/remote-servers.xml | 6 +++++- clickhouse-config/node2/remote-servers.xml | 6 +++++- clickhouse-config/node3/remote-servers.xml | 6 +++++- clickhouse-config/node4/remote-servers.xml | 6 +++++- compose.yaml | 2 ++ tests/settings.py | 3 +++ 6 files changed, 25 insertions(+), 4 deletions(-) diff --git a/clickhouse-config/node1/remote-servers.xml b/clickhouse-config/node1/remote-servers.xml index 710591f..7bbf2d5 100644 --- a/clickhouse-config/node1/remote-servers.xml +++ b/clickhouse-config/node1/remote-servers.xml @@ -7,10 +7,12 @@ node1 9000 + clickhouse_password node2 9000 + clickhouse_password @@ -18,12 +20,14 @@ node3 9000 + clickhouse_password node4 9000 + clickhouse_password - \ No newline at end of file + diff --git a/clickhouse-config/node2/remote-servers.xml b/clickhouse-config/node2/remote-servers.xml index 710591f..7bbf2d5 100644 --- a/clickhouse-config/node2/remote-servers.xml +++ b/clickhouse-config/node2/remote-servers.xml @@ -7,10 +7,12 @@ node1 9000 + clickhouse_password node2 9000 + clickhouse_password @@ -18,12 +20,14 @@ node3 9000 + clickhouse_password node4 9000 + clickhouse_password - \ No newline at end of file + diff --git a/clickhouse-config/node3/remote-servers.xml b/clickhouse-config/node3/remote-servers.xml index 710591f..7bbf2d5 100644 --- a/clickhouse-config/node3/remote-servers.xml +++ b/clickhouse-config/node3/remote-servers.xml @@ -7,10 +7,12 @@ node1 9000 + clickhouse_password node2 9000 + clickhouse_password @@ -18,12 +20,14 @@ node3 9000 + clickhouse_password node4 9000 + clickhouse_password - \ No newline at end of file + diff --git a/clickhouse-config/node4/remote-servers.xml b/clickhouse-config/node4/remote-servers.xml index 710591f..7bbf2d5 100644 --- a/clickhouse-config/node4/remote-servers.xml +++ b/clickhouse-config/node4/remote-servers.xml @@ -7,10 +7,12 @@ node1 9000 + clickhouse_password node2 9000 + clickhouse_password @@ -18,12 +20,14 @@ node3 9000 + clickhouse_password node4 9000 + clickhouse_password - \ No newline at end of file + diff --git a/compose.yaml b/compose.yaml index 0c79a8d..9e3c6d4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,7 @@ x-base-service: &base-service image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-23.6.2.18} + environment: + CLICKHOUSE_PASSWORD: "clickhouse_password" restart: always ulimits: nofile: diff --git a/tests/settings.py b/tests/settings.py index d8b60c1..ae5fde7 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -19,6 +19,7 @@ DATABASES = { "default": { "ENGINE": "clickhouse_backend.backend", + "PASSWORD": "clickhouse_password", "OPTIONS": { "migration_cluster": "cluster", "connections_min": 1, @@ -35,6 +36,7 @@ }, "s1r2": { "ENGINE": "clickhouse_backend.backend", + "PASSWORD": "clickhouse_password", "PORT": 9001, "OPTIONS": { "migration_cluster": "cluster", @@ -52,6 +54,7 @@ }, "s2r1": { "ENGINE": "clickhouse_backend.backend", + "PASSWORD": "clickhouse_password", "PORT": 9002, "OPTIONS": { "migration_cluster": "cluster",