forward-port observabilitySRE image creation into main

This is a cherry-pick of the merge-commit from the observabilitySRE feature
branch into 8.x in PR #17541 (0b1d299),
resolving conflicts in `docker/*` and `rakelib/artifacts.rake` by manually
migrating the 8.x-style docker structure to the refactored structure present
on `main`.

Author: donoghuc

.buildkite/pull_request_pipeline.yml

@@ -19,6 +19,27 @@ steps:
       export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
       ci/license_check.sh -m 4G
 
+steps:
+  - label: ":passport_control: License check - Fedramp High Mode"
+    key: "license-check-fedramp-high"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      source .buildkite/scripts/common/container-agent.sh
+      export JRUBY_OPTS="-J-Xmx1g"
+      export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e ORG_GRADLE_PROJECT_fedrampHighMode=true test-runner-image ci/license_check.sh -m 4G
+
   - label: ":rspec: Ruby unit tests"
     key: "ruby-unit-tests"
     agents:
@@ -39,6 +60,25 @@ steps:
     artifact_paths:
       - "coverage/coverage.json"
 
+  - label: ":rspec: Ruby unit tests - FIPS mode"
+    key: "ruby-unit-tests-fips"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true rubyTests
+    artifact_paths:
+      - "coverage/coverage.json"
+
   - label: ":java: Java unit tests"
     key: "java-unit-tests"
     agents:
@@ -60,6 +100,29 @@ steps:
       - "**/jacocoTestReport.xml"
       - "**/build/classes/**/*.*"
 
+  - label: ":java: Java unit tests - FIPS mode"
+    key: "java-unit-tests-fips"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    env:
+      ENABLE_SONARQUBE: true
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run test-runner-image ./gradlew --info --stacktrace -PfedrampHighMode=true javaTests
+    artifact_paths:
+      - "**/build/test-results/javaTests/TEST-*.xml"
+      - "**/jacocoTestReport.xml"
+      - "**/build/classes/**/*.*"
+
   - label: ":sonarqube: Continuous Code Inspection"
     if: | 
       build.pull_request.id != null || 
@@ -81,6 +144,82 @@ steps:
       manual:
         allowed: true
 
+  - label: "Observability SRE container smoke test"
+    key: "observability-sre-container-smoke-test"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+      source .buildkite/scripts/common/vm-agent.sh
+      QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+      # Build the image locally with the gradle task
+      ./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
+      # Ensure it can at least start logstash
+      docker run docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
+        logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }'
+      # Run the smoke tests on the PR code
+      docker tag docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
+        pr-built-observability-sre-image
+      ./gradlew observabilitySREsmokeTests --stacktrace
+
+  - label: ":lab_coat: Integration Tests - FIPS mode / part 1-of-3"
+    key: "integration-tests-fips-part-1-of-3"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split 0 3
+
+  - label: ":lab_coat: Integration Tests - FIPS mode / part 2-of-3"
+    key: "integration-tests-fips-part-2-of-3"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split 1 3
+
+  - label: ":lab_coat: Integration Tests - FIPS mode / part 3-of-3"
+    key: "integration-tests-fips-part-3-of-3"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e FEDRAMP_HIGH_MODE=true test-runner-image ci/integration_tests.sh split  2 3
+
   - label: ":lab_coat: Integration Tests / part 1-of-3"
     key: "integration-tests-part-1-of-3"
     agents:
@@ -228,6 +367,40 @@ steps:
       source .buildkite/scripts/common/container-agent.sh
       x-pack/ci/integration_tests.sh
 
+  - label: ":lab_coat: x-pack unit tests - FIPS mode"
+    key: "x-pack-unit-tests-fips"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e FEDRAMP_HIGH_MODE=true test-runner-image x-pack/ci/unit_tests.sh
+
+  - label: ":lab_coat: x-pack integration - FIPS mode"
+    key: "integration-tests-x-pack-fips"
+    agents:
+      provider: gcp
+      imageProject: elastic-images-prod
+      image: family/platform-ingest-logstash-ubuntu-2204
+      machineType: "n2-standard-4"
+      diskSizeGb: 64
+    retry:
+      automatic:
+        - limit: 3
+    command: |
+      set -euo pipefail
+
+      docker build -t test-runner-image -f x-pack/distributions/internal/observabilitySRE/docker/Dockerfile .
+      docker run -e FEDRAMP_HIGH_MODE=true test-runner-image x-pack/ci/integration_tests.sh
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/scripts/common/qualified-version.sh

@@ -17,6 +17,11 @@ export QUALIFIED_VERSION="$(
   # e.g: 8.0.0-alpha1
   printf '%s' "${VERSION_QUALIFIER:+-${VERSION_QUALIFIER}}"
 
+  # Include git SHA if requested
+  if [[ -n "${INCLUDE_COMMIT_ID:+x}" ]]; then
+    printf '%s' "-$(git rev-parse --short HEAD)"
+  fi
+
   # add the SNAPSHOT tag unless WORKFLOW_TYPE=="staging" or RELEASE=="1"
   if [[ ! ( "${WORKFLOW_TYPE:-}" == "staging" || "${RELEASE:+$RELEASE}" == "1" ) ]]; then
     printf '%s' "-SNAPSHOT"

.buildkite/scripts/dra/build-and-push-observability-sre.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Script to build and publish ObservabilitySRE container
+# Currently this is built on a host with the target architecture.
+# This allows us to utilize the make file for building the container and
+# to ensure the best compatability with the host architecture.
+# A later step in CI will take care of pushing a tag that references the right
+# image using `docker manifest` commands.
+
+echo "Setting up environment"
+source .buildkite/scripts/common/vm-agent.sh
+source .buildkite/scripts/dra/docker-env-setup.sh
+
+echo "Building ObservabilitySRE container"
+./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
+
+echo "Pushing ObservabilitySRE container to Docker repository"
+docker_login
+
+# Get qualified version without SHA (this is what the gradle task will produce)
+QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+
+# Set environment variable to include SHA and get version with SHA
+QUALIFIED_VERSION_WITH_SHA="$(INCLUDE_COMMIT_ID=1 .buildkite/scripts/common/qualified-version.sh)"
+
+REGISTRY_PATH=docker.elastic.co/logstash/logstash-observability-sre
+
+# Current architecture
+ARCH="${ARCH:-x86_64}"  # Default to x86_64 if ARCH is not set
+echo "Architecture: ${ARCH}"
+
+# Create the full tag with SHA and architecture
+FULL_TAG="${QUALIFIED_VERSION_WITH_SHA}-${ARCH}"
+echo "Tagging and pushing: ${REGISTRY_PATH}:${QUALIFIED_VERSION} as ${REGISTRY_PATH}:${FULL_TAG}"
+docker tag ${REGISTRY_PATH}:${QUALIFIED_VERSION} ${REGISTRY_PATH}:${FULL_TAG}
+docker push ${REGISTRY_PATH}:${FULL_TAG}
+
+# Teardown Docker environment
+source .buildkite/scripts/dra/docker-env-teardown.sh

.buildkite/scripts/dra/generatesteps.py

@@ -79,6 +79,61 @@ def package_aarch64_docker_step(branch, workflow_type):
 
     return step
 
+def ship_observability_sre_image_steps(branch, workflow_type):
+    step = f'''
+- label: ":package: Build & Ship aarch64 ObservabilitySRE container / {branch}-{workflow_type.upper()}"
+  key: "logstash_build_and_ship_observability_sre_aarch64"
+  soft_fail: true
+  depends_on: logstash_publish_dra
+  agents:
+    provider: aws
+    imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
+    instanceType: "m6g.4xlarge"
+    diskSizeGb: 200
+  artifact_paths:
+    - "**/*.hprof"
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    export ARCH="aarch64"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build-and-push-observability-sre.sh
+- label: ":package: Build & Ship x86_64 ObservabilitySRE container / {branch}-{workflow_type.upper()}"
+  key: "logstash_build_and_ship_observability_sre_x86_64"
+  soft_fail: true
+  depends_on: logstash_publish_dra
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-16"
+    diskSizeGb: 200
+  artifact_paths:
+    - "**/*.hprof"
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    export ARCH="x86_64"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build-and-push-observability-sre.sh
+- label: ":docker: Create & Push ObservabilitySRE Multi-Arch Manifest / {branch}-{workflow_type.upper()}"
+  key: "logstash_create_observability_sre_manifest"
+  depends_on:
+    - "logstash_build_and_ship_observability_sre_aarch64"
+    - "logstash_build_and_ship_observability_sre_x86_64"
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-8"
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/multi-architecture-observability-sre.sh
+'''
+    return step
+
 def publish_dra_step(branch, workflow_type, depends_on):
     step = f'''
 - label: ":elastic-stack: Publish  / {branch}-{workflow_type.upper()} DRA artifacts"
@@ -139,9 +194,14 @@ def build_steps_to_yaml(branch, workflow_type):
             "steps": build_steps_to_yaml(branch, workflow_type),
         })
 
-        # Final step: pull artifacts built above and publish them via the release-manager
+        # Pull artifacts built above and publish them via the release-manager
         structure["steps"].extend(
             yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
         )
 
+        # Once published, do the same for observabilitySRE image
+        structure["steps"].extend(
+            yaml.safe_load(ship_observability_sre_image_steps(branch, workflow_type)),
+        )
+
     print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))

.buildkite/scripts/dra/multi-architecture-observability-sre.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Script to create and push Docker manifest for multi-architecture support
+# This MUST be run after build-and-push-observabilty-sre.sh!
+
+source .buildkite/scripts/common/vm-agent.sh
+source .buildkite/scripts/dra/docker-env-setup.sh
+
+docker_login
+
+# Set INCLUDE_COMMIT_ID to include git SHA in version
+QUALIFIED_VERSION="$(INCLUDE_COMMIT_ID=1 .buildkite/scripts/common/qualified-version.sh)"
+REGISTRY_PATH=docker.elastic.co/logstash/logstash-observability-sre
+
+# Architecture-specific tags (created by the build steps)
+X86_64_TAG="${QUALIFIED_VERSION}-x86_64"
+AARCH64_TAG="${QUALIFIED_VERSION}-aarch64"
+
+# Target manifest tags - already has SHA from QUALIFIED_VERSION
+VERSION_MANIFEST_TAG="${QUALIFIED_VERSION}"
+
+# Create and push manifest with version (which already includes SHA)
+echo "Creating manifest list for: ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}"
+docker manifest create ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
+  ${REGISTRY_PATH}:${X86_64_TAG} \
+  ${REGISTRY_PATH}:${AARCH64_TAG}
+
+docker manifest annotate ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
+  ${REGISTRY_PATH}:${X86_64_TAG} --os linux --arch amd64
+
+docker manifest annotate ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG} \
+  ${REGISTRY_PATH}:${AARCH64_TAG} --os linux --arch arm64
+
+echo "Pushing manifest: ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}"
+docker manifest push ${REGISTRY_PATH}:${VERSION_MANIFEST_TAG}
+
+# Also create version without SHA for effective "latest" tag
+BASE_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+echo "Creating manifest list for: ${REGISTRY_PATH}:${BASE_VERSION}"
+docker manifest create ${REGISTRY_PATH}:${BASE_VERSION} \
+  ${REGISTRY_PATH}:${X86_64_TAG} \
+  ${REGISTRY_PATH}:${AARCH64_TAG}
+
+docker manifest annotate ${REGISTRY_PATH}:${BASE_VERSION} \
+  ${REGISTRY_PATH}:${X86_64_TAG} --os linux --arch amd64
+
+docker manifest annotate ${REGISTRY_PATH}:${BASE_VERSION} \
+  ${REGISTRY_PATH}:${AARCH64_TAG} --os linux --arch arm64
+
+echo "Pushing manifest: ${REGISTRY_PATH}:${BASE_VERSION}"
+docker manifest push ${REGISTRY_PATH}:${BASE_VERSION}
+
+# Teardown Docker environment
+source .buildkite/scripts/dra/docker-env-teardown.sh

.ruby-version

@@ -1 +1 @@
-jruby-9.4.9.0
+jruby-9.4.13.0