wip

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.281"
+version = "0.1.291"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__common__/runner.py

@@ -196,6 +196,7 @@ def exporter_datasink_common(
             ]
         },
     )
+    print(overall_end_time_metrics)
     # 7 days from now
     expire_redis_metrics_ms = 7 * 24 * 60 * 60 * 1000
     export_redis_metrics(
@@ -234,3 +235,19 @@ def exporter_datasink_common(
         {"metric-type": "commandstats"},
         expire_redis_metrics_ms,
     )
+
+    # Update deployment tracking sets
+    deployment_type_and_name = f"{setup_type}_AND_{setup_name}"
+    deployment_type_and_name_and_version = f"{setup_type}_AND_{setup_name}_AND_{git_version}"
+
+    # Add to deployment-specific set
+    deployment_set_key = f"ci.benchmarks.redislabs/{tf_triggering_env}/{deployment_type_and_name_and_version}:set"
+    datasink_conn.sadd(deployment_set_key, test_name)
+
+    # Add to testcases set
+    testcases_set_key = f"ci.benchmarks.redislabs/{tf_triggering_env}/testcases:set"
+    datasink_conn.sadd(testcases_set_key, test_name)
+
+    # Add metadata fields to timeseries metadata
+    metadata["deployment_type_AND_deployment_name"] = deployment_type_and_name
+    metadata["deployment_type_AND_deployment_name_AND_version"] = deployment_type_and_name_and_version

redis_benchmarks_specification/__common__/timeseries.py

@@ -147,7 +147,11 @@ def extract_results_table(
 
                     # For precision_summary metrics, construct the full resolved path for display
                     display_path = metric_jsonpath
-                    if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path:
+                    if (
+                        "precision_summary" in metric_jsonpath
+                        and "*" in metric_jsonpath
+                        and use_metric_context_path
+                    ):
                         # Replace the wildcard with the actual precision level
                         display_path = metric_jsonpath.replace("*", metric_context_path)
 
@@ -156,7 +160,10 @@ def extract_results_table(
                         metric_name = metric_jsonpath
                     else:
                         # For display purposes, use the resolved path for precision_summary
-                        if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath:
+                        if (
+                            "precision_summary" in metric_jsonpath
+                            and "*" in metric_jsonpath
+                        ):
                             metric_name = display_path
                         else:
                             # Clean up the metric name for other cases
@@ -167,7 +174,11 @@ def extract_results_table(
                             metric_name = metric_name.replace(" ", "_")
 
                     # Apply standard cleaning to all metric names
-                    if not ("precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path):
+                    if not (
+                        "precision_summary" in metric_jsonpath
+                        and "*" in metric_jsonpath
+                        and use_metric_context_path
+                    ):
                         metric_name = metric_name.replace("'", "")
                         metric_name = metric_name.replace('"', "")
                         metric_name = metric_name.replace("(", "")

redis_benchmarks_specification/__compare__/args.py

@@ -61,13 +61,13 @@ def create_compare_arguments(parser):
         "--triggering_env_baseline",
         type=str,
         default=None,
-        help="Triggering environment for baseline data. If not specified, falls back to --triggering_env"
+        help="Triggering environment for baseline data. If not specified, falls back to --triggering_env",
     )
     parser.add_argument(
         "--triggering_env_comparison",
         type=str,
         default=None,
-        help="Triggering environment for comparison data. If not specified, falls back to --triggering_env"
+        help="Triggering environment for comparison data. If not specified, falls back to --triggering_env",
     )
     parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
     parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")
@@ -83,13 +83,13 @@ def create_compare_arguments(parser):
         "--running_platform_baseline",
         type=str,
         default=None,
-        help="Platform for baseline data. If not specified, falls back to --running_platform"
+        help="Platform for baseline data. If not specified, falls back to --running_platform",
     )
     parser.add_argument(
         "--running_platform_comparison",
         type=str,
         default=None,
-        help="Platform for comparison data. If not specified, falls back to --running_platform"
+        help="Platform for comparison data. If not specified, falls back to --running_platform",
     )
     parser.add_argument("--extra-filter", type=str, default=None)
     parser.add_argument(

redis_benchmarks_specification/__compare__/compare.py

@@ -255,7 +255,9 @@ def compare_command_logic(args, project_name, project_version):
     # Handle separate baseline and comparison platform/environment arguments
     # Fall back to general arguments if specific ones are not provided
     running_platform_baseline = args.running_platform_baseline or args.running_platform
-    running_platform_comparison = args.running_platform_comparison or args.running_platform
+    running_platform_comparison = (
+        args.running_platform_comparison or args.running_platform
+    )
     triggering_env_baseline = args.triggering_env_baseline or args.triggering_env
     triggering_env_comparison = args.triggering_env_comparison or args.triggering_env
 
@@ -731,7 +733,9 @@ def compute_regression_table(
         _,
         _,
         _,
-    ) = get_overall_dashboard_keynames(tf_github_org, tf_github_repo, tf_triggering_env_baseline)
+    ) = get_overall_dashboard_keynames(
+        tf_github_org, tf_github_repo, tf_triggering_env_baseline
+    )
     test_names = []
     used_key = testcases_setname
     test_filter = "test_name"
@@ -1167,33 +1171,41 @@ def from_rts_to_regression_table(
         multi_value_comparison = check_multi_value_filter(comparison_str)
 
         filters_baseline = [
-            "{}={}".format(by_str_baseline, baseline_str),
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
-            "deployment_name={}".format(baseline_deployment_name),
             "github_repo={}".format(baseline_github_repo),
             "triggering_env={}".format(tf_triggering_env_baseline),
         ]
+        if baseline_str != "":
+            filters_baseline.append("{}={}".format(by_str_baseline, baseline_str))
+        if baseline_deployment_name != "":
+            filters_baseline.append("deployment_name={}".format(baseline_deployment_name))
         if baseline_github_org != "":
             filters_baseline.append(f"github_org={baseline_github_org}")
         if running_platform_baseline is not None:
-            filters_baseline.append("running_platform={}".format(running_platform_baseline))
+            filters_baseline.append(
+                "running_platform={}".format(running_platform_baseline)
+            )
         filters_comparison = [
-            "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
-            "deployment_name={}".format(comparison_deployment_name),
             "github_repo={}".format(comparison_github_repo),
             "triggering_env={}".format(tf_triggering_env_comparison),
         ]
+        if comparison_str != "":
+            filters_comparison.append("{}={}".format(by_str_comparison, comparison_str))
+        if comparison_deployment_name != "":
+            filters_comparison.append("deployment_name={}".format(comparison_deployment_name))
         if comparison_github_org != "":
             filters_comparison.append(f"github_org={comparison_github_org}")
         if "hash" not in by_str_baseline:
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:
             filters_comparison.append("hash==")
         if running_platform_comparison is not None:
-            filters_comparison.append("running_platform={}".format(running_platform_comparison))
+            filters_comparison.append(
+                "running_platform={}".format(running_platform_comparison)
+            )
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
         comparison_timeseries = rts.ts().queryindex(filters_comparison)
 

redis_benchmarks_specification/__runner__/args.py

@@ -20,6 +20,12 @@ def create_client_runner_args(project_name):
         description=project_name,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=project_name,
+        help="Show version information and exit",
+    )
     parser.add_argument(
         "--platform-name",
         type=str,
@@ -34,6 +40,24 @@ def create_client_runner_args(project_name):
     )
     parser.add_argument("--triggering_env", type=str, default="ci")
     parser.add_argument("--setup_type", type=str, default="oss-standalone")
+    parser.add_argument(
+        "--deployment_type",
+        type=str,
+        default="oss-standalone",
+        help="Deployment type for the Redis instance (e.g., oss-standalone, oss-cluster, enterprise)"
+    )
+    parser.add_argument(
+        "--deployment_name",
+        type=str,
+        default="redis",
+        help="Deployment name identifier for the Redis instance"
+    )
+    parser.add_argument(
+        "--core_count",
+        type=int,
+        default=None,
+        help="Number of CPU cores available to the Redis instance"
+    )
     parser.add_argument("--github_repo", type=str, default="redis")
     parser.add_argument("--github_org", type=str, default="redis")
     parser.add_argument("--github_version", type=str, default="NA")
@@ -152,6 +176,18 @@ def create_client_runner_args(project_name):
         action="store_true",
         help="Run tests that contain a dbconfig with dataset",
     )
+    parser.add_argument(
+        "--skip-tests-without-dataset",
+        default=False,
+        action="store_true",
+        help="Skip tests that do not contain a dbconfig with dataset",
+    )
+    parser.add_argument(
+        "--memory-comparison-only",
+        default=False,
+        action="store_true",
+        help="Run memory comparison only - execute preload and measure memory usage without client benchmarks",
+    )
     parser.add_argument(
         "--client_aggregated_results_folder",
         type=str,