Added dockerhub E2E triggering/tests

Author: fcostaoliveira

Readme.md

@@ -389,6 +389,12 @@ To run a specific test:
 $ tox -- utils/tests/test_runner.py
 ```
 
+To run a specific test with verbose logging:
+
+```sh
+$ tox -- -vv --log-cli-level=INFO utils/tests/test_runner.py
+```
+
 ## License
 
 redis-benchmarks-specification is distributed under the BSD3 license - see [LICENSE](LICENSE)

redis_benchmarks_specification/__builder__/builder.py

@@ -429,48 +429,32 @@ def builder_process_stream(
                 build_duration = build_end_datetime - build_start_datetime
                 build_duration_secs = build_duration.total_seconds()
 
-                build_stream_fields = {
-                    "id": id,
-                    "git_hash": git_hash,
-                    "use_git_timestamp": str(use_git_timestamp),
-                    "build_image": build_image,
-                    "run_image": run_image,
-                    "compiler": compiler,
-                    "cpp_compiler": cpp_compiler,
-                    "os": build_os,
-                    "arch": build_arch,
-                    "build_vars": build_vars_str,
-                    "build_command": build_command,
-                    "metadata": json.dumps(build_config_metadata),
-                    "build_artifacts": ",".join(build_artifacts),
-                    "tests_regexp": tests_regexp,
-                    "tests_priority_upper_limit": tests_priority_upper_limit,
-                    "tests_priority_lower_limit": tests_priority_lower_limit,
-                    "tests_groups_regexp": tests_groups_regexp,
-                }
-                if pull_request is not None:
-                    build_stream_fields["pull_request"] = pull_request
-                if git_branch is not None:
-                    build_stream_fields["git_branch"] = git_branch
-                if git_version is not None:
-                    build_stream_fields["git_version"] = git_version
-                if git_timestamp_ms is not None:
-                    build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
-                for artifact in build_artifacts:
-                    bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact)
-                    bin_artifact = open(
-                        "{}src/{}".format(redis_temporary_dir, artifact), "rb"
-                    ).read()
-                    bin_artifact_len = len(bytes(bin_artifact))
-                    assert bin_artifact_len > 0
-                    conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
-                    build_stream_fields[artifact] = bin_key
-                    build_stream_fields["{}_len_bytes".format(artifact)] = (
-                        bin_artifact_len
-                    )
-                result = True
-                if b"platform" in testDetails:
-                    build_stream_fields["platform"] = testDetails[b"platform"]
+                build_stream_fields, result = generate_benchmark_stream_request(
+                    id,
+                    conn,
+                    run_image,
+                    build_arch,
+                    testDetails,
+                    build_os,
+                    build_artifacts,
+                    build_command,
+                    build_config_metadata,
+                    build_image,
+                    build_vars_str,
+                    compiler,
+                    cpp_compiler,
+                    git_branch,
+                    git_hash,
+                    git_timestamp_ms,
+                    git_version,
+                    pull_request,
+                    redis_temporary_dir,
+                    tests_groups_regexp,
+                    tests_priority_lower_limit,
+                    tests_priority_upper_limit,
+                    tests_regexp,
+                    use_git_timestamp,
+                )
                 if result is True:
                     benchmark_stream_id = conn.xadd(
                         STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
@@ -551,6 +535,84 @@ def builder_process_stream(
     return previous_id, new_builds_count, build_stream_fields_arr
 
 
+def generate_benchmark_stream_request(
+    id,
+    conn,
+    run_image,
+    build_arch,
+    testDetails,
+    build_os,
+    build_artifacts=[],
+    build_command=None,
+    build_config_metadata=None,
+    build_image=None,
+    build_vars_str=None,
+    compiler=None,
+    cpp_compiler=None,
+    git_branch=None,
+    git_hash=None,
+    git_timestamp_ms=None,
+    git_version=None,
+    pull_request=None,
+    redis_temporary_dir=None,
+    tests_groups_regexp=".*",
+    tests_priority_lower_limit=0,
+    tests_priority_upper_limit=10000,
+    tests_regexp=".*",
+    use_git_timestamp=False,
+):
+    build_stream_fields = {
+        "id": id,
+        "use_git_timestamp": str(use_git_timestamp),
+        "run_image": run_image,
+        "os": build_os,
+        "arch": build_arch,
+        "build_artifacts": ",".join(build_artifacts),
+        "tests_regexp": tests_regexp,
+        "tests_priority_upper_limit": tests_priority_upper_limit,
+        "tests_priority_lower_limit": tests_priority_lower_limit,
+        "tests_groups_regexp": tests_groups_regexp,
+    }
+    if build_config_metadata is not None:
+        build_stream_fields["metadata"] = json.dumps(build_config_metadata)
+    if compiler is not None:
+        build_stream_fields["compiler"] = compiler
+    if cpp_compiler is not None:
+        build_stream_fields["cpp_compiler"] = cpp_compiler
+    if build_vars_str is not None:
+        build_stream_fields["build_vars"] = build_vars_str
+    if build_command is not None:
+        build_stream_fields["build_command"] = build_command
+    if build_image is not None:
+        build_stream_fields["build_image"] = build_image
+    else:
+        build_stream_fields["build_image"] = run_image
+    if git_hash is not None:
+        build_stream_fields["git_hash"] = git_hash
+    if pull_request is not None:
+        build_stream_fields["pull_request"] = pull_request
+    if git_branch is not None:
+        build_stream_fields["git_branch"] = git_branch
+    if git_version is not None:
+        build_stream_fields["git_version"] = git_version
+    if git_timestamp_ms is not None:
+        build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
+    for artifact in build_artifacts:
+        bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact)
+        bin_artifact = open(
+            "{}src/{}".format(redis_temporary_dir, artifact), "rb"
+        ).read()
+        bin_artifact_len = len(bytes(bin_artifact))
+        assert bin_artifact_len > 0
+        conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
+        build_stream_fields[artifact] = bin_key
+        build_stream_fields["{}_len_bytes".format(artifact)] = bin_artifact_len
+    result = True
+    if b"platform" in testDetails:
+        build_stream_fields["platform"] = testDetails[b"platform"]
+    return build_stream_fields, result
+
+
 def build_spec_image_prefetch(builders_folder, different_build_specs):
     logging.info("checking build spec requirements")
     already_checked_images = []

redis_benchmarks_specification/__common__/runner.py

@@ -149,7 +149,9 @@ def exporter_datasink_common(
     default_metrics=None,
     git_hash=None,
 ):
-    logging.info("Using datapoint_time_ms: {}".format(datapoint_time_ms))
+    logging.info(
+        f"Using datapoint_time_ms: {datapoint_time_ms}. git_has={git_hash}, git_branch={git_branch}, git_version={git_version}"
+    )
     timeseries_test_sucess_flow(
         datasink_push_results_redistimeseries,
         git_version,
@@ -171,6 +173,7 @@ def exporter_datasink_common(
         metadata,
         build_variant_name,
         running_platform,
+        None,
         git_hash,
     )
     logging.info("Collecting memory metrics")

redis_benchmarks_specification/__common__/timeseries.py

@@ -283,10 +283,12 @@ def from_metric_kv_to_timeserie(
         tf_triggering_env,
         use_metric_context_path,
     )
+    logging.info(f"Adding timeserie named {ts_name} to time_series_dict.")
     time_series_dict[ts_name] = {
         "labels": timeserie_tags.copy(),
         "data": {datapoints_timestamp: metric_value},
     }
+
     original_ts_name = ts_name
     target_table_keyname = "target_tables:{triggering_env}:ci.benchmarks.redislabs/{break_by_key}/{break_by_str}/{tf_github_org}/{tf_github_repo}/{deployment_type}/{deployment_name}/{test_name}/{metric_name}".format(
         triggering_env=tf_triggering_env,
@@ -361,6 +363,8 @@ def common_timeseries_extraction(
     time_series_dict = {}
     target_tables = {}
     cleaned_metrics_arr = extract_results_table(metrics, results_dict)
+    total_metrics = len(cleaned_metrics_arr)
+    logging.info(f"Total of {total_metrics} cleaned metrics: {cleaned_metrics_arr}")
     for cleaned_metric in cleaned_metrics_arr:
 
         metric_jsonpath = cleaned_metric[0]
@@ -398,6 +402,48 @@ def common_timeseries_extraction(
     return time_series_dict, target_tables
 
 
+def extract_perhash_timeseries_from_results(
+    datapoints_timestamp: int,
+    metrics: list,
+    results_dict: dict,
+    git_hash: str,
+    tf_github_org: str,
+    tf_github_repo: str,
+    deployment_name: str,
+    deployment_type: str,
+    test_name: str,
+    tf_triggering_env: str,
+    metadata_tags={},
+    build_variant_name=None,
+    running_platform=None,
+    testcase_metric_context_paths=[],
+):
+    break_by_key = "hash"
+    break_by_str = "by.{}".format(break_by_key)
+    (
+        time_series_dict,
+        target_tables,
+    ) = common_timeseries_extraction(
+        break_by_key,
+        break_by_str,
+        datapoints_timestamp,
+        deployment_name,
+        deployment_type,
+        metrics,
+        git_hash,
+        results_dict,
+        test_name,
+        tf_github_org,
+        tf_github_repo,
+        tf_triggering_env,
+        metadata_tags,
+        build_variant_name,
+        running_platform,
+        testcase_metric_context_paths,
+    )
+    return True, time_series_dict, target_tables
+
+
 def extract_perversion_timeseries_from_results(
     datapoints_timestamp: int,
     metrics: list,
@@ -718,38 +764,46 @@ def common_exporter_logic(
             )
         )
     assert used_ts is not None
-
+    total_break_by_added = 0
     if (git_hash is not None) and (git_hash != ""):
-        break_by_key = "hash"
-        break_by_str = "by.{}".format(break_by_key)
+        # extract per-hash datapoints
         (
+            _,
             per_hash_time_series_dict,
-            hash_target_tables,
-        ) = common_timeseries_extraction(
-            break_by_key,
-            break_by_str,
-            datapoints_timestamp,
-            deployment_name,
-            deployment_type,
+            version_target_tables,
+        ) = extract_perhash_timeseries_from_results(
+            used_ts,
             metrics,
-            git_hash,
             results_dict,
-            test_name,
+            git_hash,
             tf_github_org,
             tf_github_repo,
+            deployment_name,
+            deployment_type,
+            test_name,
             tf_triggering_env,
             metadata_tags,
             build_variant_name,
             running_platform,
             testcase_metric_context_paths,
         )
-
+        total_break_by_added += 1
+    else:
+        logging.warning(
+            "there was no git hash information to push data brokedown by hash"
+        )
     if (
         artifact_version is not None
         and artifact_version != ""
         and artifact_version != "N/A"
     ):
         # extract per-version datapoints
+        total_hs_ts = len(per_hash_time_series_dict.keys())
+        logging.info(
+            f"Extending the by.hash {git_hash} timeseries ({total_hs_ts}) with version info {artifact_version}"
+        )
+        for hash_timeserie in per_hash_time_series_dict.values():
+            hash_timeserie["labels"]["version"] = artifact_version
         (
             _,
             per_version_time_series_dict,
@@ -770,7 +824,18 @@ def common_exporter_logic(
             running_platform,
             testcase_metric_context_paths,
         )
+        total_break_by_added += 1
+    else:
+        logging.warning(
+            "there was no git VERSION information to push data brokedown by VERSION"
+        )
     if tf_github_branch is not None and tf_github_branch != "":
+        total_hs_ts = len(per_hash_time_series_dict.keys())
+        logging.info(
+            f"Extending the by.hash {git_hash} timeseries ({total_hs_ts}) with branch info {tf_github_branch}"
+        )
+        for hash_timeserie in per_hash_time_series_dict.values():
+            hash_timeserie["labels"]["branch"] = tf_github_branch
         # extract per branch datapoints
         (
             _,
@@ -792,10 +857,14 @@ def common_exporter_logic(
             running_platform,
             testcase_metric_context_paths,
         )
+        total_break_by_added += 1
     else:
+        logging.warning(
+            "there was no git BRANCH information to push data brokedown by BRANCH"
+        )
+    if total_break_by_added == 0:
         logging.error(
-            "Requested to push data to RedisTimeSeries but "
-            'no exporter definition was found. Missing "exporter" config.'
+            "There was no BRANCH, HASH, or VERSION info to break this info by in timeseries"
         )
     return (
         per_version_time_series_dict,

redis_benchmarks_specification/__runner__/runner.py

@@ -211,7 +211,7 @@ def prepare_memtier_benchmark_parameters(
     server,
     password,
     local_benchmark_output_filename,
-    oss_cluster_api_enabled,
+    oss_cluster_api_enabled=False,
     tls_enabled=False,
     tls_skip_verify=False,
     tls_cert=None,

redis_benchmarks_specification/__self_contained_coordinator__/build_info.py

@@ -10,11 +10,13 @@ def extract_build_info_from_streamdata(testDetails):
     arch = "amd64"
     use_git_timestamp = False
     git_timestamp_ms = None
-    metadata = None
+    metadata = {}
     build_variant_name = None
     fields = [fieldname.decode() for fieldname in testDetails.keys()]
-    logging.info("Fields on stream {}".format(fields))
-    git_hash = testDetails[b"git_hash"]
+    logging.info("Fields on stream {}".format(testDetails))
+    git_hash = None
+    if b"git_hash" in testDetails:
+        git_hash = testDetails[b"git_hash"].decode()
     if b"use_git_timestamp" in testDetails:
         use_git_timestamp = bool(testDetails[b"use_git_timestamp"].decode())
     if b"git_timestamp_ms" in testDetails:

redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -18,9 +18,9 @@ def generate_standalone_redis_server_args(
         "no",
         "--port",
         "{}".format(port),
-        "--dir",
-        dbdir,
     ]
+    if dbdir != "":
+        command.extend(["--dbdir", dbdir])
     if configuration_parameters is not None:
         for parameter, parameter_value in configuration_parameters.items():
             if parameter not in added_params: