Include extra tests for test-glob and introduce test-regex (#395)

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.9.33"
+version = "0.9.44"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "README.md"

redisbench_admin/run/args.py

@@ -24,6 +24,7 @@
 ENV = os.getenv("ENV", "oss-standalone,oss-cluster")
 SETUP = os.getenv("SETUP", "")
 BENCHMARK_GLOB = os.getenv("BENCHMARK_GLOB", "*.yml")
+BENCHMARK_REGEX = os.getenv("BENCHMARK_REGEX", ".*")
 S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "ci.benchmarks.redislabs")
 PUSH_S3 = bool(os.getenv("PUSH_S3", False))
 REDIS_7 = bool(os.getenv("REDIS_7", True))
@@ -82,6 +83,12 @@ def common_run_args(parser):
         default=BENCHMARK_GLOB,
         help="specify a test glob pattern to use on the tests directory. by default uses *.yml. If --test is defined this options has no effect.",
     )
+    parser.add_argument(
+        "--test-regex",
+        type=str,
+        default=BENCHMARK_REGEX,
+        help="specify a test regex pattern to use on the tests directory. by default uses '.*'. If --test is defined this options has no effect.",
+    )
 
     parser.add_argument(
         "--defaults_filename",

redisbench_admin/utils/benchmark_config.py

@@ -55,7 +55,10 @@ def parse_exporter_timemetric(metric_path: str, results_dict: dict):
 def prepare_benchmark_definitions(args):
     benchmark_definitions = {}
     result = True
-    defaults_filename, files = get_testfiles_to_process(args)
+    defaults_filename = args.defaults_filename
+    files = get_testfiles_to_process(
+        args.test_glob, args.test, defaults_filename, args.test_regex
+    )
 
     (
         default_kpis,
@@ -482,23 +485,29 @@ def min_ver_check(
     )
 
 
-def get_testfiles_to_process(args):
-    defaults_filename = args.defaults_filename
-    if args.test == "":
-        files = pathlib.Path().glob(args.test_glob)
+def get_testfiles_to_process(test_glob, test_name, defaults_filename, test_regex=".*"):
+    if test_name == "":
+        files = pathlib.Path().glob(test_glob)
         files = [str(x) for x in files]
-        if defaults_filename in files:
-            files.remove(defaults_filename)
-
+        for file in files:
+            if defaults_filename in file:
+                files.remove(file)
+        final_files = []
+        for file in files:
+            a = re.search(test_regex, file)
+            if a is not None:
+                final_files.append(a.string)
+        files = final_files
         logging.info(
             "Running all specified benchmarks: {}".format(
                 " ".join([str(x) for x in files])
             )
         )
+
     else:
-        files = args.test.split(",")
+        files = test_name.split(",")
         logging.info("Running specific benchmark in file: {}".format(files))
-    return defaults_filename, files
+    return files
 
 
 def check_required_modules(module_names, required_modules):

tests/test_benchmark_config.py

@@ -13,6 +13,7 @@
     get_termination_timeout_secs,
     prepare_benchmark_definitions,
     process_benchmark_definitions_remote_timeouts,
+    get_testfiles_to_process,
 )
 
 
@@ -196,7 +197,7 @@ def test_prepare_benchmark_definitions():
         clusterconfig,
     ) = prepare_benchmark_definitions(args)
     assert result == True
-    assert len(benchmark_definitions.keys()) == 2
+    assert len(benchmark_definitions.keys()) == 6
 
 
 def test_process_benchmark_definitions_remote_timeouts():
@@ -224,10 +225,41 @@ def test_process_benchmark_definitions_remote_timeouts():
         clusterconfig,
     ) = prepare_benchmark_definitions(args)
     assert result == True
-    assert len(benchmark_definitions.keys()) == 2
+    assert len(benchmark_definitions.keys()) == 6
     remote_envs_timeout = process_benchmark_definitions_remote_timeouts(
         benchmark_definitions
     )
-    assert len(remote_envs_timeout.keys()) == 1
+    assert len(remote_envs_timeout.keys()) == 2
     # we have the default timeout + the one specified
     assert list(remote_envs_timeout.values())[0] == 600 + 1200
+
+
+def test_get_testfiles_to_process():
+    test_glob_pattern_all = "./tests/test_data/benchmark_definitions/*.yml"
+    test_glob_pattern_graph500 = "./tests/test_data/benchmark_definitions/graph500*.yml"
+    test_files_to_process = get_testfiles_to_process(
+        test_glob_pattern_all, "", "defaults.yml"
+    )
+    assert 6 == len(test_files_to_process)
+    test_files_to_process_graph500_glob = get_testfiles_to_process(
+        test_glob_pattern_graph500, "", "defaults.yml"
+    )
+    assert 3 == len(test_files_to_process_graph500_glob)
+    test_files_to_process = get_testfiles_to_process(
+        "./tests/test_data/benchmark_definitions/*.yml",
+        "",
+        "defaults.yml",
+        "graph500.+.yml",
+    )
+    assert 3 == len(test_files_to_process)
+    assert test_files_to_process_graph500_glob == test_files_to_process
+
+    test_files_to_process = get_testfiles_to_process(
+        "./tests/test_data/benchmark_definitions/*.yml",
+        "",
+        "defaults.yml",
+        "^(?!.*graph500).*.yml",
+    )
+    assert 3 == len(test_files_to_process)
+    for test_graph in test_files_to_process_graph500_glob:
+        assert test_graph not in test_files_to_process

tests/test_data/benchmark_definitions/defaults.yml

@@ -0,0 +1,23 @@
+version: 0.1
+kpis:
+  - eq: { $.Totals.Total.Errors: 0 }
+exporter:
+  redistimeseries:
+    timemetric: "$.StartTime"
+    metrics:
+      - "$.OverallClientLatencies.Total.q50"
+      - "$.OverallClientLatencies.Total.q95"
+      - "$.OverallClientLatencies.Total.q99"
+      - "$.OverallClientLatencies.Total.avg"
+      - "$.OverallGraphInternalLatencies.Total.q50"
+      - "$.OverallGraphInternalLatencies.Total.q95"
+      - "$.OverallGraphInternalLatencies.Total.q99"
+      - "$.OverallGraphInternalLatencies.Total.avg"
+      - "$.OverallRelativeInternalExternalLatencyDiff.avg"
+      - "$.OverallRelativeInternalExternalLatencyDiff.q50"
+      - "$.OverallQueryRates.Total"
+  comparison:
+    metrics:
+      - "$.OverallQueryRates.Total"
+    mode: higher-better
+    baseline-branch: master

tests/test_data/benchmark_definitions/explicit_edge_deletion.yml

@@ -0,0 +1,20 @@
+name: "EXPLICIT-EDGE-DELETION"
+remote:
+  - setup: redisgraph-r5
+  - type: oss-standalone
+dbconfig:
+  - init_commands:
+    - '"GRAPH.QUERY" "g" "UNWIND range(0, 1000000) AS x CREATE (:N)-[:R]->(:N)"'
+clientconfig:
+  - tool: redisgraph-benchmark-go
+  - parameters:
+    - graph: "g"
+    - rps: 0
+    - clients: 1
+    - threads: 4
+    - connections: 1
+    - requests: 1000
+    - queries:
+        - { q: "MATCH (n:N)-[e]->() DELETE e RETURN 1 LIMIT 1", ratio: 1 }
+kpis:
+  - le: { $.OverallGraphInternalLatencies.Total.q50: 40.0 }

tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop.yml

@@ -0,0 +1,30 @@
+name: "GRAPH500-SCALE_18-EF_16-1_HOP"
+description: "Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
+                       - 262017 nodes with label 'Node'
+                       - 4194304 relations of type 'IS_CONNECTED'
+                       - Indexed properties: 
+                          - exact-match: Node; [external_id]
+             "
+remote:
+  - setup: redisgraph-r5
+  - type: oss-standalone
+timeout_seconds: 3600
+dbconfig:
+  - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
+  - dataset_load_timeout_secs: 180
+clientconfig:
+  - tool: redisgraph-benchmark-go
+  - parameters:
+    - graph: "graph500-scale18-ef16"
+    - rps: 0
+    - clients: 32
+    - threads: 4
+    - connections: 32
+    - requests: 1000000
+    - random-int-max: 262016
+    - random-seed: 12345
+    - queries:
+      - { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 1.0 }
+kpis:
+  - le: { $.OverallClientLatencies.Total.q50: 5.0 }
+  - ge: { $.OverallQueryRates.Total: 8000 }

tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop_MIXED_READ_65_WRITE_25p_DEL_10p.yml

@@ -0,0 +1,29 @@
+name: "GRAPH500-SCALE_18-EF_16-1_HOP_MIXED_READ_65perc_WRITE_25perc_DEL_10perc"
+description: "Test: tests mixed read+write+delete queries within the same benchmark
+              Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
+                       - 262017 nodes with label 'Node'
+                       - 4194304 relations of type 'IS_CONNECTED'
+                       - Indexed properties: 
+                          - exact-match: Node; [external_id]
+             "
+remote:
+  - setup: redisgraph-r5
+  - type: oss-standalone
+dbconfig:
+  - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
+  - dataset_load_timeout_secs: 180
+clientconfig:
+  - tool: redisgraph-benchmark-go
+  - parameters:
+    - graph: "graph500-scale18-ef16"
+    - rps: 0
+    - clients: 32
+    - threads: 4
+    - connections: 32
+    - requests: 10000
+    - random-int-max: 262016
+    - random-seed: 12345
+    - queries:
+      - { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 0.65 }
+      - { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1}) MATCH (n2:Node {external_id: $Id2}) MERGE(n1)-[rel:IS_CONNECTED]->(n2)", ratio: 0.25 }
+      - { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1})-[rel:IS_CONNECTED]->(n2:Node {external_id: $Id2}) DELETE rel", ratio: 0.10 }

tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop_MIXED_READ_75_WRITE_25p.yml

@@ -0,0 +1,27 @@
+name: "GRAPH500-SCALE_18-EF_16-1_HOP_MIXED_READ_75perc_WRITE_25perc"
+description: "Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
+                       - 262017 nodes with label 'Node'
+                       - 4194304 relations of type 'IS_CONNECTED'
+                       - Indexed properties: 
+                          - exact-match: Node; [external_id]
+             "
+remote:
+  - setup: redisgraph-r5
+  - type: oss-standalone
+dbconfig:
+  - dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
+  - dataset_load_timeout_secs: 180
+clientconfig:
+  - tool: redisgraph-benchmark-go
+  - parameters:
+    - graph: "graph500-scale18-ef16"
+    - rps: 0
+    - clients: 32
+    - threads: 4
+    - connections: 32
+    - requests: 10000
+    - random-int-max: 262016
+    - random-seed: 12345
+    - queries:
+      - { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 0.75 }
+      - { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1}) MATCH (n2:Node {external_id: $Id2}) MERGE (n1)-[rel:IS_CONNECTED]->(n2)", ratio: 0.25 }