Skip to content

Add additional clickhouse database datetime functions #125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 3, 2025
Merged

Add additional clickhouse database datetime functions #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7ef18a0
Add additional clickhouse database datetime functions
AndrewSpeed May 29, 2025
ae4c6b8
Add changelog entry
AndrewSpeed May 29, 2025
6e6acea
Update CHANGELOG
AndrewSpeed Jun 2, 2025
3e8a49a
Set explicit password for default user in attempt to fix intermittent…
AndrewSpeed Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
### 1.4.0

- feat: #119 Allow query results returned in columns and deserialized to `numpy` objects
- feat: #125 Add database functions `toStartOfMinute`, `toStartOfFiveMinutes`, `toStartOfTenMinutes`, `toStartOfFifteenMinutes` and `toStartofHour`

### 1.3.2

Expand Down
6 changes: 5 additions & 1 deletion clickhouse-config/node1/remote-servers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,27 @@
<replica>
<host>node1</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node2</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>node3</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node4</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
</cluster>
</remote_servers>
</clickhouse>
</clickhouse>
6 changes: 5 additions & 1 deletion clickhouse-config/node2/remote-servers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,27 @@
<replica>
<host>node1</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node2</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>node3</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node4</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
</cluster>
</remote_servers>
</clickhouse>
</clickhouse>
6 changes: 5 additions & 1 deletion clickhouse-config/node3/remote-servers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,27 @@
<replica>
<host>node1</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node2</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>node3</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node4</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
</cluster>
</remote_servers>
</clickhouse>
</clickhouse>
6 changes: 5 additions & 1 deletion clickhouse-config/node4/remote-servers.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,27 @@
<replica>
<host>node1</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node2</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>node3</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
<replica>
<host>node4</host>
<port>9000</port>
<password>clickhouse_password</password>
</replica>
</shard>
</cluster>
</remote_servers>
</clickhouse>
</clickhouse>
38 changes: 38 additions & 0 deletions clickhouse_backend/models/functions/datetime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
from .base import Func

__all__ = [
"toStartOfMinute",
"toStartOfFiveMinutes",
"toStartOfTenMinutes",
"toStartOfFifteenMinutes",
"toStartOfHour",
"toYYYYMM",
"toYYYYMMDD",
"toYYYYMMDDhhmmss",
Expand Down Expand Up @@ -39,3 +44,36 @@ class toYYYYMMDD(toYYYYMM):

class toYYYYMMDDhhmmss(toYYYYMM):
output_field = fields.UInt64Field()


class toStartOfMinute(Func):
output_field = models.fields.DateTimeField()

def __init__(self, *expressions):
arity = len(expressions)
if arity < 1 or arity > 1:
raise TypeError(
"'%s' takes 1 argument (%s given)"
% (
self.__class__.__name__,
len(expressions),
)
)

super().__init__(*expressions)


class toStartOfFiveMinutes(toStartOfMinute):
pass


class toStartOfTenMinutes(toStartOfMinute):
pass


class toStartOfFifteenMinutes(toStartOfMinute):
pass


class toStartOfHour(toStartOfMinute):
pass
2 changes: 2 additions & 0 deletions compose.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
x-base-service: &base-service
image: clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-23.6.2.18}
environment:
CLICKHOUSE_PASSWORD: "clickhouse_password"
restart: always
ulimits:
nofile:
Expand Down
130 changes: 126 additions & 4 deletions tests/clickhouse_functions/test_datetime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ def setUpTestData(cls):
alias="smithj",
# https://stackoverflow.com/a/18862958
birthday=pytz.timezone(get_timezone()).localize(
datetime(2023, 11, 30, 16), is_dst=False
datetime(2023, 11, 30, hour=16, minute=12, second=15), is_dst=False
),
)
cls.elena = Author.objects.create(
name="Élena Jordan",
alias="elena",
birthday=pytz.utc.localize(datetime(2023, 11, 30, 16), is_dst=False),
birthday=pytz.utc.localize(
datetime(2023, 11, 30, hour=16, minute=59, second=59), is_dst=False
),
)

def test_yyyymm(self):
Expand All @@ -50,8 +52,128 @@ def test_yyyymmddhhmmss(self):
john = Author.objects.annotate(v=models.toYYYYMMDDhhmmss("birthday")).get(
id=self.john.id
)
self.assertEqual(john.v, 20231130160000)
self.assertEqual(john.v, 20231130161215)
elena = Author.objects.annotate(
v=models.toYYYYMMDDhhmmss("birthday", "Asia/Shanghai")
).get(id=self.elena.id)
self.assertEqual(elena.v, 20231201000000)
self.assertEqual(elena.v, 20231201005959)

def test_tostartofminute(self):
john = Author.objects.annotate(v=models.toStartOfMinute("birthday")).get(
id=self.john.id
)
self.assertEqual(
john.v,
datetime(
2023,
11,
30,
hour=16,
minute=12,
second=00,
),
)

elena = Author.objects.annotate(v=models.toStartOfMinute("birthday")).get(
id=self.elena.id
)
self.assertEqual(
elena.v,
datetime(2023, 11, 30, hour=10, minute=59, second=00),
)

def test_tostartoffiveminutes(self):
john = Author.objects.annotate(v=models.toStartOfFiveMinutes("birthday")).get(
id=self.john.id
)
self.assertEqual(
john.v,
datetime(
2023,
11,
30,
hour=16,
minute=10,
second=00,
),
)

elena = Author.objects.annotate(v=models.toStartOfFiveMinutes("birthday")).get(
id=self.elena.id
)
self.assertEqual(
elena.v,
datetime(2023, 11, 30, hour=10, minute=55, second=00),
)

def test_tostartoftenminutes(self):
john = Author.objects.annotate(v=models.toStartOfTenMinutes("birthday")).get(
id=self.john.id
)
self.assertEqual(
john.v,
datetime(
2023,
11,
30,
hour=16,
minute=10,
second=00,
),
)

elena = Author.objects.annotate(v=models.toStartOfTenMinutes("birthday")).get(
id=self.elena.id
)
self.assertEqual(
elena.v,
datetime(2023, 11, 30, hour=10, minute=50, second=00),
)

def test_tostartoffifteenminutes(self):
john = Author.objects.annotate(
v=models.toStartOfFifteenMinutes("birthday")
).get(id=self.john.id)
self.assertEqual(
john.v,
datetime(
2023,
11,
30,
hour=16,
minute=00,
second=00,
),
)

elena = Author.objects.annotate(
v=models.toStartOfFifteenMinutes("birthday")
).get(id=self.elena.id)
self.assertEqual(
elena.v,
datetime(2023, 11, 30, hour=10, minute=45, second=00),
)

def test_tostartofhour(self):
john = Author.objects.annotate(v=models.toStartOfHour("birthday")).get(
id=self.john.id
)
self.assertEqual(
john.v,
datetime(
2023,
11,
30,
hour=16,
minute=00,
second=00,
),
)

elena = Author.objects.annotate(v=models.toStartOfHour("birthday")).get(
id=self.elena.id
)
self.assertEqual(
elena.v,
datetime(2023, 11, 30, hour=10, minute=00, second=00),
)
3 changes: 3 additions & 0 deletions tests/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
DATABASES = {
"default": {
"ENGINE": "clickhouse_backend.backend",
"PASSWORD": "clickhouse_password",
"OPTIONS": {
"migration_cluster": "cluster",
"connections_min": 1,
Expand All @@ -35,6 +36,7 @@
},
"s1r2": {
"ENGINE": "clickhouse_backend.backend",
"PASSWORD": "clickhouse_password",
"PORT": 9001,
"OPTIONS": {
"migration_cluster": "cluster",
Expand All @@ -52,6 +54,7 @@
},
"s2r1": {
"ENGINE": "clickhouse_backend.backend",
"PASSWORD": "clickhouse_password",
"PORT": 9002,
"OPTIONS": {
"migration_cluster": "cluster",
Expand Down
Loading