ci: shard tracer-release system tests four ways

.gitlab/generate-package.php

@@ -168,6 +168,26 @@ function appsec_image_from_tag_mapping(string $tag): string
     REQUIREMENTS_BLOCK_JSON_PATH: "loader/packaging/block_tests.json"
     REQUIREMENTS_ALLOW_JSON_PATH: "loader/packaging/allow_tests.json"
 
+"system tests shard selector test":
+  stage: prepare
+  image: registry.ddbuild.io/images/mirror/python:3.12-slim-bullseye
+  tags: [ "arch:amd64" ]
+  needs: []
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  script:
+    - python3 .gitlab/tests/test_package_system_tests_sharding.py -v
+
+"system tests pinning contract test":
+  stage: prepare
+  image: registry.ddbuild.io/images/mirror/php:8.2-cli
+  tags: [ "arch:amd64" ]
+  needs: []
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  script:
+    - php .gitlab/tests/test_package_system_tests_pinning.php
+
 
 # dd-trace-php release packaging
 "prepare code":
@@ -176,6 +196,13 @@ function appsec_image_from_tag_mapping(string $tag): string
   tags: [ "arch:amd64" ]
   script:
     - ./.gitlab/append-build-id.sh
+    - |
+      SYSTEM_TESTS_SHA=$(git ls-remote https://github.com/DataDog/system-tests.git refs/heads/main | awk 'NR == 1 { print $1 }')
+      if ! printf '%s\n' "$SYSTEM_TESTS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then
+        echo "Failed to resolve a valid system-tests commit: $SYSTEM_TESTS_SHA"
+        exit 1
+      fi
+      printf 'SYSTEM_TESTS_SHA=%s\n' "$SYSTEM_TESTS_SHA" > system-tests.env
     # Upgrading composer
     - composer self-update --no-interaction
     # Installing dependencies with composer
@@ -190,6 +217,8 @@ function appsec_image_from_tag_mapping(string $tag): string
     paths:
       - VERSION
       - ./src/bridge/_generated*.php
+    reports:
+      dotenv: system-tests.env
 
 <?php
 foreach ($build_platforms as $platform) {
@@ -1305,7 +1334,20 @@ function appsec_image_from_tag_mapping(string $tag): string
       pip install -U pip virtualenv
 <?php dockerhub_login() ?>
     - /tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true
-    - git clone https://github.com/DataDog/system-tests.git
+    - |
+      if ! printf '%s\n' "${SYSTEM_TESTS_SHA:-}" | grep -Eq '^[0-9a-f]{40}$'; then
+        echo "Missing or invalid SYSTEM_TESTS_SHA: ${SYSTEM_TESTS_SHA:-}"
+        exit 1
+      fi
+      git init -q system-tests
+      git -C system-tests remote add origin https://github.com/DataDog/system-tests.git
+      git -C system-tests fetch --depth=1 origin "$SYSTEM_TESTS_SHA"
+      git -C system-tests checkout --detach "$SYSTEM_TESTS_SHA"
+      CHECKED_OUT_SYSTEM_TESTS_SHA=$(git -C system-tests rev-parse HEAD)
+      if [ "$CHECKED_OUT_SYSTEM_TESTS_SHA" != "$SYSTEM_TESTS_SHA" ]; then
+        echo "Checked out system-tests $CHECKED_OUT_SYSTEM_TESTS_SHA, expected $SYSTEM_TESTS_SHA"
+        exit 1
+      fi
     - mv packages/{datadog-setup.php,dd-library-php-*x86_64-linux-gnu.tar.gz} system-tests/binaries
     - cd system-tests
     - ./build.sh $BUILD_SH_ARGS
@@ -1385,6 +1427,7 @@ function appsec_image_from_tag_mapping(string $tag): string
 "System Tests: [<?= $weblog ?>, tracer-release]":
   extends: .system_tests
   timeout: 4h
+  parallel: 4
   variables:
     BUILD_SH_ARGS: -w <?= $weblog ?> php
     # Expand the DinD loopback volume to avoid running out of disk space.
@@ -1400,7 +1443,12 @@ function appsec_image_from_tag_mapping(string $tag): string
   script:
     - DD_API_KEY=$(cat /tmp/.dd-api-key 2>/dev/null) || { echo "Failed to fetch DD_API_KEY"; exit 1; }
     - export DD_API_KEY
-    - SCENARIOS=$(PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json | python3 -c "import sys,json;d=json.load(sys.stdin);s=set();[s.update(v['scenarios']) for v in d.values() if isinstance(v,dict) and 'scenarios' in v];print(' '.join(sorted(s)))")
+    - |
+      set -o pipefail
+      SCENARIOS=$(
+        PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json |
+          python3 "$CI_PROJECT_DIR/.gitlab/select-system-tests-shard.py"
+      ) || exit $?
     - FAILED=""; for S in $SCENARIOS; do echo "=== Running $S ==="; ./run.sh $S || FAILED="$FAILED $S"; done; if [ -n "$FAILED" ]; then echo "Failed scenarios:$FAILED"; exit 1; fi
 
 <?php endforeach; ?>

.gitlab/select-system-tests-shard.py

@@ -0,0 +1,70 @@
+import json
+import os
+import sys
+
+
+def fail(message):
+    raise SystemExit(f"Failed to select tracer-release scenarios: {message}")
+
+
+def validate_scenario_group(group, location):
+    if not isinstance(group, list) or any(
+        not isinstance(scenario, str) or not scenario or any(character.isspace() for character in scenario)
+        for scenario in group
+    ):
+        fail(f"expected {location} to be a list of non-empty names without whitespace")
+    return group
+
+
+def main():
+    try:
+        data = json.load(sys.stdin)
+    except (json.JSONDecodeError, UnicodeDecodeError) as error:
+        fail(f"invalid scenario JSON: {error}")
+
+    if not isinstance(data, dict):
+        fail("expected a JSON object")
+
+    try:
+        shard_count = int(os.environ["CI_NODE_TOTAL"])
+        shard_number = int(os.environ["CI_NODE_INDEX"])
+    except (KeyError, ValueError) as error:
+        fail(f"invalid shard configuration: {error}")
+
+    if shard_count != 4 or not 1 <= shard_number <= shard_count:
+        fail(f"invalid shard {shard_number} of {shard_count}")
+
+    endtoend_defs = data.get("endtoend_defs")
+    if not isinstance(endtoend_defs, dict):
+        fail("expected endtoend_defs to be an object")
+
+    parallel_jobs = endtoend_defs.get("parallel_jobs")
+    if not isinstance(parallel_jobs, list) or not parallel_jobs:
+        fail("expected endtoend_defs.parallel_jobs to be a non-empty list")
+
+    scenarios = set()
+    for index, job in enumerate(parallel_jobs):
+        if not isinstance(job, dict) or "scenarios" not in job:
+            fail(f"expected endtoend_defs.parallel_jobs[{index}] to contain scenarios")
+        scenarios.update(
+            validate_scenario_group(
+                job["scenarios"],
+                f"endtoend_defs.parallel_jobs[{index}].scenarios",
+            )
+        )
+
+    for name, value in data.items():
+        if name in ("endtoend", "endtoend_defs"):
+            continue
+        if isinstance(value, dict) and "scenarios" in value:
+            scenarios.update(validate_scenario_group(value["scenarios"], f"{name}.scenarios"))
+
+    selected = sorted(scenarios)[shard_number - 1::shard_count]
+    if not selected:
+        fail(f"shard {shard_number} of {shard_count} is empty")
+
+    print(" ".join(selected))
+
+
+if __name__ == "__main__":
+    main()

.gitlab/tests/test_package_system_tests_pinning.php

@@ -0,0 +1,137 @@
+<?php
+
+function fail(string $message): void
+{
+    fwrite(STDERR, "$message\n");
+    exit(1);
+}
+
+function require_contains(string $configuration, string $expected, string $contract): void
+{
+    if (!str_contains($configuration, $expected)) {
+        fail("Generated pipeline is missing $contract");
+    }
+}
+
+function generated_definition(string $configuration, string $name): string
+{
+    $marker = "$name:\n";
+    $start = strpos($configuration, $marker);
+    if ($start === false) {
+        fail("Generated pipeline is missing $name");
+    }
+
+    $following = substr($configuration, $start + strlen($marker));
+    if (!preg_match('/^(?=(?:"[^\n]+"|[A-Za-z_.][^:\n]*):\n)/m', $following, $match, PREG_OFFSET_CAPTURE)) {
+        return substr($configuration, $start);
+    }
+
+    return substr($configuration, $start, strlen($marker) + $match[0][1]);
+}
+
+$root = dirname(__DIR__, 2);
+$original_directory = getcwd();
+chdir("$root/.gitlab");
+ob_start();
+require "$root/.gitlab/generate-package.php";
+$configuration = ob_get_clean();
+chdir($original_directory);
+
+$prepare_code = generated_definition($configuration, '"prepare code"');
+require_contains(
+    $prepare_code,
+    "SYSTEM_TESTS_SHA=\$(git ls-remote https://github.com/DataDog/system-tests.git refs/heads/main | " .
+        "awk 'NR == 1 { print \$1 }')",
+    'system-tests revision resolution'
+);
+require_contains(
+    $prepare_code,
+    "if ! printf '%s\\n' \"\$SYSTEM_TESTS_SHA\" | grep -Eq '^[0-9a-f]{40}\$'; then\n" .
+        "        echo \"Failed to resolve a valid system-tests commit: \$SYSTEM_TESTS_SHA\"\n" .
+        "        exit 1\n" .
+        "      fi",
+    'resolved system-tests revision validation'
+);
+require_contains(
+    $prepare_code,
+    "printf 'SYSTEM_TESTS_SHA=%s\\n' \"\$SYSTEM_TESTS_SHA\" > system-tests.env",
+    'system-tests dotenv creation'
+);
+require_contains(
+    $prepare_code,
+    "  artifacts:\n" .
+        "    paths:\n" .
+        "      - VERSION\n" .
+        "      - ./src/bridge/_generated*.php\n" .
+        "    reports:\n" .
+        "      dotenv: system-tests.env",
+    'system-tests dotenv artifact report'
+);
+
+$system_tests = generated_definition($configuration, '.system_tests');
+require_contains(
+    $system_tests,
+    "- job: \"prepare code\"\n      artifacts: true",
+    'prepare code artifact dependency'
+);
+require_contains(
+    $system_tests,
+    "if ! printf '%s\\n' \"\${SYSTEM_TESTS_SHA:-}\" | grep -Eq '^[0-9a-f]{40}\$'; then\n" .
+        "        echo \"Missing or invalid SYSTEM_TESTS_SHA: \${SYSTEM_TESTS_SHA:-}\"\n" .
+        "        exit 1\n" .
+        "      fi",
+    'checkout revision validation'
+);
+require_contains(
+    $system_tests,
+    "git init -q system-tests\n" .
+        "      git -C system-tests remote add origin https://github.com/DataDog/system-tests.git",
+    'system-tests checkout initialization'
+);
+require_contains(
+    $system_tests,
+    'git -C system-tests fetch --depth=1 origin "$SYSTEM_TESTS_SHA"',
+    'exact shallow system-tests fetch'
+);
+require_contains(
+    $system_tests,
+    'git -C system-tests checkout --detach "$SYSTEM_TESTS_SHA"',
+    'detached system-tests checkout'
+);
+require_contains(
+    $system_tests,
+    'CHECKED_OUT_SYSTEM_TESTS_SHA=$(git -C system-tests rev-parse HEAD)',
+    'checked-out system-tests revision lookup'
+);
+require_contains(
+    $system_tests,
+    "if [ \"\$CHECKED_OUT_SYSTEM_TESTS_SHA\" != \"\$SYSTEM_TESTS_SHA\" ]; then\n" .
+        "        echo \"Checked out system-tests \$CHECKED_OUT_SYSTEM_TESTS_SHA, expected \$SYSTEM_TESTS_SHA\"\n" .
+        "        exit 1\n" .
+        "      fi",
+    'checked-out system-tests revision mismatch failure'
+);
+
+if (!preg_match_all(
+    '/^"System Tests: \[[^,\]\n]+, tracer-release\]":$/m',
+    $configuration,
+    $tracer_release_jobs
+) || count($tracer_release_jobs[0]) !== 25) {
+    fail('Generated pipeline must contain 25 tracer-release definitions');
+}
+
+foreach ($tracer_release_jobs[0] as $heading) {
+    $name = substr($heading, 0, -1);
+    $definition = generated_definition($configuration, $name);
+    require_contains($definition, "  parallel: 4\n", "$name four-way parallel expansion");
+    require_contains($definition, "      set -o pipefail\n", "$name scenario pipeline failure detection");
+    require_contains(
+        $definition,
+        "PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json |\n" .
+            "          python3 \"\$CI_PROJECT_DIR/.gitlab/select-system-tests-shard.py\"",
+        "$name scenario producer-to-selector pipeline"
+    );
+    require_contains($definition, ') || exit $?', "$name selector failure propagation");
+}
+
+echo "Generated pipeline system-tests pinning contract: OK\n";