Skip to content

Commit 3cd414f

Browse files
jayvynljayvynl
authored
fix distributed tests (#45)
* Fix distributed and replicated table engine tests, add test guide to README.md. * refactor migrations patch * fix flush command
1 parent 4e16435 commit 3cd414f

File tree

16 files changed

+264
-330
lines changed

16 files changed

+264
-330
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
- Use [flake8](https://flake8.pycqa.org/) to lint code.
33
- Add GitHub action which runs tests.
44
- Add test coverage to ci and send data to coveralls, add coverage badge.
5+
- Fix distributed and replicated table engine tests, add test guide to README.md.
56

67
### 1.1.1
78
- [Black](https://github.com/psf/black) code style.

‎README.md‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -385,8 +385,10 @@ DATABASES = {
385385
"OPTIONS": {
386386
"migration_cluster": "cluster",
387387
"settings": {
388-
"mutations_sync": 1,
388+
"mutations_sync": 2,
389389
"insert_distributed_sync": 1,
390+
"insert_quorum": 2,
391+
"alter_sync": 2,
390392
},
391393
},
392394
"TEST": {"cluster": "cluster"},
@@ -397,35 +399,41 @@ DATABASES = {
397399
"OPTIONS": {
398400
"migration_cluster": "cluster",
399401
"settings": {
400-
"mutations_sync": 1,
402+
"mutations_sync": 2,
401403
"insert_distributed_sync": 1,
404+
"insert_quorum": 2,
405+
"alter_sync": 2,
402406
},
403407
},
404-
"TEST": {"cluster": "cluster", "managed": False},
408+
"TEST": {"cluster": "cluster", "managed": False, "DEPENDENCIES": ["default"]},
405409
},
406410
"s2r1": {
407411
"ENGINE": "clickhouse_backend.backend",
408412
"PORT": 9002,
409413
"OPTIONS": {
410414
"migration_cluster": "cluster",
411415
"settings": {
412-
"mutations_sync": 1,
416+
"mutations_sync": 2,
413417
"insert_distributed_sync": 1,
418+
"insert_quorum": 2,
419+
"alter_sync": 2,
414420
},
415421
},
416-
"TEST": {"cluster": "cluster", "managed": False},
422+
"TEST": {"cluster": "cluster", "managed": False, "DEPENDENCIES": ["default"]},
417423
},
418424
"s2r2": {
419425
"ENGINE": "clickhouse_backend.backend",
420426
"PORT": 9003,
421427
"OPTIONS": {
422428
"migration_cluster": "cluster",
423429
"settings": {
424-
"mutations_sync": 1,
430+
"mutations_sync": 2,
425431
"insert_distributed_sync": 1,
432+
"insert_quorum": 2,
433+
"alter_sync": 2,
426434
},
427435
},
428-
"TEST": {"cluster": "cluster", "managed": False},
436+
"TEST": {"cluster": "cluster", "managed": False, "DEPENDENCIES": ["default"]},
429437
},
430438
}
431439
```
@@ -434,15 +442,25 @@ Extra settings explanation:
434442

435443
- `"migration_cluster": "cluster"`
436444
Migration table will be created on this cluster if this setting is specified, otherwise only local migration table is created.
437-
- `"mutations_sync": 1`
438-
This is suggested if you want to test [data mutations](https://clickhouse.com/docs/en/guides/developer/mutations).
445+
- `"mutations_sync": 2`
446+
This is suggested if you want to test [data mutations](https://clickhouse.com/docs/en/guides/developer/mutations) on replicated table.
447+
*Don't* set this in production environment.
439448
- `"insert_distributed_sync": 1`
440449
This is suggested if you want to test inserting data into distributed table.
441-
- `"TEST": {"cluster": "cluster", "managed": False}`
450+
*Don't* set this in production environment.
451+
- `"insert_quorum": 2`
452+
This is suggested if you want to test inserting data into replicated table.
453+
The value is set to replica number.
454+
- `"alter_sync": 2`
455+
This is suggested if you want to test altering or truncating replicated table.
456+
*Don't* set this in production environment.
457+
- `"TEST": {"cluster": "cluster", "managed": False, "DEPENDENCIES": ["default"]}`
442458
Test database will be created on this cluster.
443459
If you have multiple database connections to the same cluster and want to run tests over all these connections,
444460
then only one connection should set `"managed": True`(the default value), other connections should set `"managed": False`.
445461
So that test database will not be created multiple times.
462+
463+
If your managed database alias is not `s1r2`, `"DEPENDENCIES": ["s1r2"]` should be set to ensure the [creation order for test databases](https://docs.djangoproject.com/en/4.2/topics/testing/advanced/#controlling-creation-order-for-test-databases).
446464

447465
Do not hardcode database name when you define replicated table or distributed table.
448466
Because test database name is different from deployed database name.

‎clickhouse-config/node1/remote-servers.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<cluster>
44
<secret>mysecretphrase</secret>
55
<shard>
6-
<internal_replication>false</internal_replication>
6+
<internal_replication>true</internal_replication>
77
<replica>
88
<host>node1</host>
99
<port>9000</port>
@@ -14,7 +14,7 @@
1414
</replica>
1515
</shard>
1616
<shard>
17-
<internal_replication>false</internal_replication>
17+
<internal_replication>true</internal_replication>
1818
<replica>
1919
<host>node3</host>
2020
<port>9000</port>

‎clickhouse-config/node2/remote-servers.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<cluster>
44
<secret>mysecretphrase</secret>
55
<shard>
6-
<internal_replication>false</internal_replication>
6+
<internal_replication>true</internal_replication>
77
<replica>
88
<host>node1</host>
99
<port>9000</port>
@@ -14,7 +14,7 @@
1414
</replica>
1515
</shard>
1616
<shard>
17-
<internal_replication>false</internal_replication>
17+
<internal_replication>true</internal_replication>
1818
<replica>
1919
<host>node3</host>
2020
<port>9000</port>

‎clickhouse-config/node3/remote-servers.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<cluster>
44
<secret>mysecretphrase</secret>
55
<shard>
6-
<internal_replication>false</internal_replication>
6+
<internal_replication>true</internal_replication>
77
<replica>
88
<host>node1</host>
99
<port>9000</port>
@@ -14,7 +14,7 @@
1414
</replica>
1515
</shard>
1616
<shard>
17-
<internal_replication>false</internal_replication>
17+
<internal_replication>true</internal_replication>
1818
<replica>
1919
<host>node3</host>
2020
<port>9000</port>

‎clickhouse-config/node4/remote-servers.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<cluster>
44
<secret>mysecretphrase</secret>
55
<shard>
6-
<internal_replication>false</internal_replication>
6+
<internal_replication>true</internal_replication>
77
<replica>
88
<host>node1</host>
99
<port>9000</port>
@@ -14,7 +14,7 @@
1414
</replica>
1515
</shard>
1616
<shard>
17-
<internal_replication>false</internal_replication>
17+
<internal_replication>true</internal_replication>
1818
<replica>
1919
<host>node3</host>
2020
<port>9000</port>

‎clickhouse_backend/backend/operations.py‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -278,27 +278,30 @@ def regex_lookup(self, lookup_type):
278278
def table_model_dict(self):
279279
return {model._meta.db_table: model for model in apps.get_models()}
280280

281-
def _get_on_cluster(self, style, table, sync=False):
282-
if table in self.table_model_dict:
283-
cluster = getattr(self.table_model_dict[table], "cluster", None)
284-
if cluster:
285-
if sync:
286-
return (
287-
f"ON CLUSTER {style.SQL_FIELD(self.quote_name(cluster))} SYNC"
288-
)
289-
return f"ON CLUSTER {style.SQL_FIELD(self.quote_name(cluster))}"
290-
return ""
281+
def sql_flush_table(self, style, table):
282+
from clickhouse_backend import models
291283

292-
def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False):
293-
return [
294-
"%s %s %s"
295-
% (
284+
cluster = None
285+
if table in self.table_model_dict:
286+
opts = self.table_model_dict[table]._meta
287+
engine = getattr(opts, "engine", None)
288+
if isinstance(engine, models.Distributed):
289+
table = engine.table
290+
cluster = engine.cluster
291+
292+
if cluster:
293+
return "%s %s ON CLUSTER %s" % (
296294
style.SQL_KEYWORD("TRUNCATE"),
297295
style.SQL_FIELD(self.quote_name(table)),
298-
self._get_on_cluster(style, table),
296+
style.SQL_FIELD(self.quote_name(cluster)),
299297
)
300-
for table in tables
301-
]
298+
return "%s %s" % (
299+
style.SQL_KEYWORD("TRUNCATE"),
300+
style.SQL_FIELD(self.quote_name(table)),
301+
)
302+
303+
def sql_flush(self, style, tables, *, reset_sequences=False, allow_cascade=False):
304+
return [self.sql_flush_table(style, table) for table in tables]
302305

303306
def prep_for_iexact_query(self, x):
304307
return x

‎clickhouse_backend/models/base.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,11 @@ def get_queryset(self):
2929

3030
class ClickhouseModel(models.Model):
3131
objects = ClickhouseManager()
32+
_overwrite_base_manager = ClickhouseManager()
3233

3334
class Meta:
3435
abstract = True
36+
base_manager_name = "_overwrite_base_manager"
3537

3638
def _do_update(self, base_qs, using, pk_val, values, update_fields, forced_update):
3739
"""

0 commit comments

Comments
 (0)