add test

Author: jkartseva

tests/integration/test_s3_plain_rewritable_rotate_tables/configs/storage_conf.xml

@@ -10,6 +10,19 @@
                 <access_key_id>minio</access_key_id>
                 <secret_access_key>ClickHouse_Minio_P@ssw0rd</secret_access_key>
             </disk_s3_plain_rewritable>
+            <disk_cache_s3_plain_rewritable>
+                <type>cache</type>
+                <disk>disk_s3_plain_rewritable</disk>
+                <path>disks/cache/</path>
+                <max_size>1000000000</max_size>
+                <cache_on_write_operations>1</cache_on_write_operations>
+            </disk_cache_s3_plain_rewritable>
+            <disk_encrypted>
+                <type>encrypted</type>
+                <disk>disk_cache_s3_plain_rewritable</disk>
+                <key>1234567812345678</key>
+                <path>disks/encrypted/</path>
+            </disk_encrypted>
         </disks>
         <policies>
             <s3_plain_rewritable>
@@ -19,6 +32,20 @@
                     </main>
                 </volumes>
             </s3_plain_rewritable>
+            <cache>
+                <volumes>
+                    <main>
+                        <disk>disk_cache_s3_plain_rewritable</disk>
+                    </main>
+                </volumes>
+            </cache>
+            <encrypted>
+                <volumes>
+                    <main>
+                        <disk>disk_encrypted</disk>
+                    </main>
+                </volumes>
+            </encrypted>
         </policies>
     </storage_configuration>
 </clickhouse>

tests/integration/test_s3_plain_rewritable_rotate_tables/test.py

@@ -23,6 +23,37 @@ def randomize_name(table_name, random_suffix_length=8):
     return f"{table_name}_{''.join(random.choice(letters) for _ in range(random_suffix_length))}"
 
 
+s3 = """disk(
+    name={},
+    type='s3_plain_rewritable',
+    endpoint='http://minio1:9001/root/data/node1',
+    access_key_id='minio',
+    secret_access_key='ClickHouse_Minio_P@ssw0rd')"""
+
+cache = """disk(
+    type='cache',
+    disk={},
+    path='disks/cache/',
+    cache_on_write_operations=1,
+    max_size='1Gi')""".format(
+    s3
+)
+
+encrypted = """disk(
+    type='encrypted',
+    disk={},
+    path='disks/encrypted/',
+    key='1234567812345678')""".format(
+    cache
+)
+
+disk_defs = {
+    "s3_plain_rewritable": s3,
+    "cache": cache,
+    "encrypted": encrypted,
+}
+
+
 @pytest.fixture(scope="module", autouse=True)
 def start_cluster():
     cluster.add_instance(
@@ -51,7 +82,16 @@ def start_cluster():
         cluster.shutdown()
 
 
-def test():
+@pytest.mark.parametrize(
+    "storage_policy",
+    [
+        "s3_plain_rewritable",
+        "cache",
+        "encrypted",
+    ],
+)
+@pytest.mark.parametrize("partition_ops_on_same_disk", [1, 0])
+def test(start_cluster, storage_policy, partition_ops_on_same_disk):
     node1 = cluster.instances["node1"]
 
     def create_insert(node, table_name, insert_values):
@@ -63,7 +103,7 @@ def create_insert(node, table_name, insert_values):
             ) ENGINE=MergeTree()
             ORDER BY id
             PARTITION BY id%10
-            SETTINGS storage_policy='s3_plain_rewritable'
+            SETTINGS storage_policy='{storage_policy}'
             """
         )
 
@@ -80,44 +120,43 @@ def create_insert(node, table_name, insert_values):
 
     node1.query(f"DETACH TABLE {table1} PERMANENTLY SYNC")
 
-    disk_name = randomize_name("disk")
     node2 = cluster.instances["node2"]
     table2 = randomize_name("table2")
     node2.query(f"DROP TABLE IF EXISTS {table2} SYNC")
+
+    disk_name = randomize_name("disk")
+    disk_def = disk_defs[storage_policy].format(disk_name)
+    policy_def = (
+        f"disk={disk_def}"
+        if partition_ops_on_same_disk
+        else f"storage_policy='{storage_policy}'"
+    )
+
     node2.query(
         f"""CREATE TABLE {table2} (id Int64, data String)
         ENGINE=MergeTree()
         ORDER BY id
         PARTITION BY id%10
-        SETTINGS disk=disk(
-            name={disk_name},
-            type='s3_plain_rewritable',
-            endpoint='http://minio1:9001/root/data/node1',
-            access_key_id='minio',
-            secret_access_key='ClickHouse_Minio_P@ssw0rd')
+        SETTINGS {policy_def}
         """
     )
 
     rotated_table = f"{table1}_rotated"
     node2.query(f"""DROP TABLE IF EXISTS {rotated_table} SYNC""")
+
     node2.query(
         f"""ATTACH TABLE {rotated_table} UUID '{uuid1}' (id Int64, data String)
         ENGINE=MergeTree()
         ORDER BY id
         PARTITION BY id%10
-        SETTINGS disk=disk(
-            name={disk_name},
-            type='s3_plain_rewritable',
-            endpoint='http://minio1:9001/root/data/node1',
-            access_key_id='minio',
-            secret_access_key='ClickHouse_Minio_P@ssw0rd')
+        SETTINGS disk={disk_def}
         """
     )
 
     assert (
         int(
             node2.query(
-                f"SELECT count(*) FROM {rotated_table} WHERE _partition_id = '0'"
+                f"SELECT count(*) FROM {rotated_table} WHERE _partition_id= '0'"
             )
         )
         == 100
@@ -126,9 +165,10 @@ def create_insert(node, table_name, insert_values):
     assert int(node2.query(f"SELECT count(*) FROM {rotated_table}")) == 1000
 
     node2.query(f"""ALTER TABLE {rotated_table} MOVE PARTITION '0' TO TABLE {table2}""")
+    node2.query(f"""ALTER TABLE {table2} REPLACE PARTITION '1' FROM {rotated_table}""")
 
     assert int(node2.query(f"SELECT count(*) FROM {rotated_table}")) == 900
-    assert int(node2.query(f"SELECT count(*) FROM {table2}")) == 100
+    assert int(node2.query(f"SELECT count(*) FROM {table2}")) == 200
 
     node2.query(f"DROP TABLE {rotated_table} SYNC")
     node2.query(f"DROP TABLE {table2} SYNC")