add magma transaction tests

bochun · bochun · commit 8466112dffc8 · 2022-02-17T23:53:57.000Z
Change-Id: I18a48e81a8fc595668dd557a17c201441a7aded2 Reviewed-on: https://review.couchbase.org/c/perfrunner/+/170589 Tested-by: Build Bot <build@couchbase.com> Reviewed-by: Dragos Petru Taraban <dragos.petru@couchbase.com>
diff --git a/tests/hidd/n1ql_lat_Q1_1B_10KQops.test b/tests/hidd/n1ql_lat_Q1_1B_10KQops.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_lat_Q1_1B_10KQops_couchstore.test b/tests/hidd/n1ql_lat_Q1_1B_10KQops_couchstore.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_lat_Q2_1B_10Kops_3KQops.test b/tests/hidd/n1ql_lat_Q2_1B_10Kops_3KQops.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_lat_Q2_1B_10Kops_3KQops_couchstore.test b/tests/hidd/n1ql_lat_Q2_1B_10Kops_3KQops_couchstore.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q1_1B.test b/tests/hidd/n1ql_thr_Q1_1B.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q1_1B_couchstore.test b/tests/hidd/n1ql_thr_Q1_1B_couchstore.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q2_1B.test b/tests/hidd/n1ql_thr_Q2_1B.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q2_1B_couchstore.test b/tests/hidd/n1ql_thr_Q2_1B_couchstore.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q3_1B.test b/tests/hidd/n1ql_thr_Q3_1B.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/n1ql_thr_Q3_1B_couchstore.test b/tests/hidd/n1ql_thr_Q3_1B_couchstore.test
@@ -8,10 +8,10 @@ category = n1ql
 
 [cluster]
 mem_quota = 5120
-index_mem_quota = 21504
+index_mem_quota = 10752
 initial_nodes = 6
 num_buckets = 1
-kernel_mem_limit = 28G
+kernel_mem_limit = 16G
 kv_kernel_mem_limit = 10G
 kernel_mem_limit_services = kv index
 enable_cpu_cores = false
diff --git a/tests/hidd/ycsb_latency_workloadt_4nodes_1B_magma_TR1.test b/tests/hidd/ycsb_latency_workloadt_4nodes_1B_magma_TR1.test
@@ -2,7 +2,7 @@
 test = perfrunner.tests.magma.YCSBLatencyHiDDTest
 
 [showfast]
-title = Latency(ms), Workload T, 4 nodes, 1 bucket x 1B x 1KB, Persist_Majority, Magma
+title = Latency(ms), Workload T, 4 nodes, 1 bucket x 1B x 1KB, Persist_Majority, Uniform distribution, 100 ops/sec, 2%% Resident Ratio, Magma
 component = hidd
 category = transaction
 sub_category = Latency
@@ -31,7 +31,7 @@ items = 1000000000
 workers = 160
 kv_endpoints = 128
 size = 1000
-use_backup = false
+use_backup = true
 
 [extra_access]
 run_extra_access = true
@@ -50,22 +50,26 @@ requestdistribution = uniform
 workload_path = workloads/workloadta
 ycsb_client = couchbase3
 items = 1000000000
-ops = 1000000000
+ops = 0
+target = 100
 time = 1800
 boost = 12
 workers = 160
+kv_endpoints = 128
 workload_instances = 1
 transactionsenabled = 1
 num_atrs = 20480
 durability = 3
-kv_endpoints = 128
 timeseries = 1
 requestdistribution = uniform
 
 [ycsb]
-branch = couchbase3-dgm
+branch = couchbase3
 average_latency = 1
 latency_percentiles = 50,90,99
 
 [stats]
 client_processes = java
+
+[backup]
+backup_directory = /data2/4n_1b_transaction
diff --git a/tests/hidd/ycsb_latency_workloadt_4nodes_1B_magma_TR1_couchstore.test b/tests/hidd/ycsb_latency_workloadt_4nodes_1B_magma_TR1_couchstore.test
@@ -0,0 +1,57 @@
+[test_case]
+test = perfrunner.tests.magma.YCSBLatencyHiDDTest
+
+[showfast]
+title = Latency(ms), Workload T, 4 nodes, 1 bucket x 1B x 1KB, Persist_Majority, Uniform distribution, 100 ops/sec, 2%% Resident Ratio, Couchstore
+component = hidd
+category = transaction
+sub_category = Latency
+
+[cluster]
+mem_quota = 15360
+initial_nodes = 4
+num_buckets = 1
+kernel_mem_limit = 25G
+kernel_mem_limit_services = kv
+enable_cpu_cores = false
+
+[bucket]
+replica_number = 2
+eviction_policy = fullEviction
+
+[bucket_extras]
+num_reader_threads = 64
+num_writer_threads = 64
+
+[load]
+workload_path = workloads/workloada
+ycsb_client = couchbase3
+items = 1000000000
+workers = 160
+kv_endpoints = 128
+size = 1000
+
+[access]
+workload_path = workloads/workloadta
+ycsb_client = couchbase3
+items = 1000000000
+ops = 0
+target = 100
+time = 1800
+boost = 12
+workers = 160
+kv_endpoints = 128
+workload_instances = 1
+transactionsenabled = 1
+num_atrs = 20480
+durability = 3
+timeseries = 1
+requestdistribution = uniform
+
+[ycsb]
+branch = couchbase3
+average_latency = 1
+latency_percentiles = 50,90,99
+
+[stats]
+client_processes = java
diff --git a/tests/hidd/ycsb_latency_workloadt_4nodes_4B_magma_TR2.test b/tests/hidd/ycsb_latency_workloadt_4nodes_4B_magma_TR2.test
@@ -2,7 +2,7 @@
 test = perfrunner.tests.magma.YCSBLatencyHiDDTest
 
 [showfast]
-title = Latency(ms), Workload T, 4 nodes, 1 bucket x 4B x 1KB, Persist_Majority, Magma
+title = Latency(ms), Workload T, 4 nodes, 1 bucket x 4B x 1KB, Persist_Majority, Uniform distribution, 100 ops/sec, Magma
 component = hidd
 category = transaction
 sub_category = Latency
@@ -29,26 +29,27 @@ items = 4000000000
 workers = 160
 kv_endpoints = 128
 size = 1000
-use_backup = false
+use_backup = true
 
 [extra_access]
 run_extra_access = true
 workload_path = workloads/workloadupdate
 ycsb_client = couchbase3
 items = 4000000000
-ops = 4000000000
+ops = 1000000000
 time = 86400
 boost = 12
-workers = 160
+workers = 80
 kv_endpoints = 128
-workload_instances = 1
+workload_instances = 4
 requestdistribution = uniform
 
 [access]
 workload_path = workloads/workloadta
 ycsb_client = couchbase3
 items = 4000000000
-ops = 4000000000
+ops = 0
+target = 100
 time = 1800
 boost = 12
 workers = 160
@@ -61,9 +62,12 @@ timeseries = 1
 requestdistribution = uniform
 
 [ycsb]
-branch = couchbase3-dgm
+branch = couchbase3
 average_latency = 1
 latency_percentiles = 50,90,99
 
 [stats]
 client_processes = java
+
+[backup]
+backup_directory = /data2/4n_4b_transaction
diff --git a/tests/hidd/ycsb_load_backup_4n_1b_magma_transaction.test b/tests/hidd/ycsb_load_backup_4n_1b_magma_transaction.test
@@ -0,0 +1,53 @@
+[test_case]
+test = perfrunner.tests.magma.YCSBLoadBackupHIDDTest
+
+[showfast]
+title = 4 nodes, 1 bucket x 1B x 1KB, Persist_Majority, Magma
+component = hidd
+category = transaction
+sub_category = Throughput
+
+[cluster]
+mem_quota = 51200
+initial_nodes = 4
+num_buckets = 1
+enable_cpu_cores = false
+
+[bucket]
+replica_number = 2
+backend_storage = magma
+eviction_policy = fullEviction
+
+[bucket_extras]
+num_reader_threads = 64
+num_writer_threads = 64
+
+[load]
+workload_path = workloads/workloada
+ycsb_client = couchbase3
+items = 1000000000
+workers = 160
+kv_endpoints = 128
+size = 1000
+
+[extra_access]
+run_extra_access = true
+workload_path = workloads/workloadupdate
+ycsb_client = couchbase3
+items = 1000000000
+ops = 500000000
+time = 86400
+boost = 12
+workers = 160
+kv_endpoints = 128
+workload_instances = 2
+requestdistribution = uniform
+
+[ycsb]
+branch = couchbase3
+
+[stats]
+client_processes = java
+
+[backup]
+backup_directory = /data2/4n_1b_transaction
diff --git a/tests/hidd/ycsb_load_backup_4n_4b_magma_transaction.test b/tests/hidd/ycsb_load_backup_4n_4b_magma_transaction.test
diff --git a/tests/hidd/ycsb_workloadt_4nodes_1B_magma_TR1.test b/tests/hidd/ycsb_workloadt_4nodes_1B_magma_TR1.test
diff --git a/tests/hidd/ycsb_workloadt_4nodes_1B_magma_TR1_couchstore.test b/tests/hidd/ycsb_workloadt_4nodes_1B_magma_TR1_couchstore.test
diff --git a/tests/hidd/ycsb_workloadt_4nodes_4B_magma_TR2.test b/tests/hidd/ycsb_workloadt_4nodes_4B_magma_TR2.test
diff --git a/tests/pipelines/weekly-neo.json b/tests/pipelines/weekly-neo.json
diff --git a/tests/ycsb_workloadg_4nodes_magma.test b/tests/ycsb_workloadg_4nodes_magma.test
