Harmonize observability sre acceptance (#17781)

* Move ES config from ENV to file

* Use fips filebeat impl for smoke tests

* Standardize container orchestration

Make rspec responsible for container orchestration. DRY shared functionality.

* Ensure we test against the right versions

Dynamically look up the version of logstash and test against the corresponding ES image.
Also, extract this to make local dev easier.

(cherry picked from commit 1195743)

Author: donoghuc

.buildkite/pull_request_pipeline.yml

@@ -157,18 +157,7 @@ steps:
     command: |
       set -euo pipefail
       source .buildkite/scripts/common/vm-agent.sh
-      QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
-      # Build the image locally with the gradle task
-      ./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
-      # Ensure it can at least start logstash
-      docker run docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
-        logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }'
-      # Run the smoke tests on the PR code
-      docker tag docker.elastic.co/logstash/logstash-observability-sre:$${QUALIFIED_VERSION} \
-        pr-built-observability-sre-image
-      # observabilitySREsmokeTests orchestrates FIPS-mode docker images
-      # and validates assertions separately, so it does not need FIPS flag.
-      ./gradlew observabilitySREsmokeTests --stacktrace
+      ./ci/observabilitySREsmoke_tests.sh
 
   - label: ":lab_coat: Integration Tests - FIPS mode / part 1-of-3"
     key: "integration-tests-fips-part-1-of-3"

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -183,7 +183,7 @@ def fips_test_runner_step() -> dict[str, typing.Any]:
         "command": LiteralScalarString("""#!/usr/bin/env bash
 set -euo pipefail
 source .buildkite/scripts/common/vm-agent.sh
-./gradlew observabilitySREacceptanceTests --stacktrace
+./ci/observabilitySREacceptance_tests.sh
 """),
     }
     return step

ci/observabilitySREacceptance_tests.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+
+./gradlew observabilitySREacceptanceTests --stacktrace

ci/observabilitySREsmoke_tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}"
+
+./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
+
+docker run docker.elastic.co/logstash/logstash-observability-sre:${QUALIFIED_VERSION} \
+  logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }'
+
+docker tag docker.elastic.co/logstash/logstash-observability-sre:${QUALIFIED_VERSION} pr-built-observability-sre-image
+
+./gradlew observabilitySREsmokeTests --stacktrace

x-pack/build.gradle

@@ -77,8 +77,8 @@ tasks.register("buildFipsValidationGem") {
 }
 
 tasks.register("observabilitySREsmokeTests", Test) {
-    description = "Run ObservabilitySRE smoke tests using docker-compose and RSpec"
-    // Need to have set up the ruby environment for rspec even through we are running in container
+    description = "Run ObservabilitySRE smoke tests using RSpec with docker-compose orchestration"
+    // Need to have set up the ruby environment for rspec
     dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
     inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
     doFirst {
@@ -88,40 +88,20 @@ tasks.register("observabilitySREsmokeTests", Test) {
             commandLine 'bash', './generate.sh'
             ignoreExitValue = false
         }
-        def result = exec {
-            workingDir file("distributions/internal/observabilitySRE/qa/smoke/docker")
-            commandLine 'docker-compose', 'up', '--detach'
-            ignoreExitValue = true
-        }
-        if (result.exitValue != 0) {
-            throw new GradleException("Docker compose failed to start")
-        }
-        // Give containers time to start and show logs
-        sleep(30000)
-        exec {
-            workingDir file("distributions/internal/observabilitySRE/qa/smoke/docker")
-            commandLine 'docker-compose', 'logs'
-        }
     }
     systemProperty 'logstash.root.dir', projectDir.parent
     include '**/org/logstash/xpack/test/RSpecObservabilitySRETests.class'
     doLast {
-        exec {
-            workingDir file("distributions/internal/observabilitySRE/qa/smoke/docker")
-            commandLine 'docker-compose', 'down', '--volumes'
-            ignoreExitValue = true
-        }
         // Clean up the generated certificates
         delete fileTree("distributions/internal/observabilitySRE/qa/smoke/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
     }
 }
 
 tasks.register("observabilitySREacceptanceTests", Test) {
-    description = "Run ObservabilitySRE acceptance tests"
-    // Need to have set up the ruby environment for rspec even through we are running in container
+    description = "Run ObservabilitySRE acceptance tests using RSpec with docker-compose orchestration"
+    // Need to have set up the ruby environment for rspec
     dependsOn(":bootstrap", ":logstash-core:assemble", ":installDevelopmentGems")
-
-    inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/smoke")
+    inputs.files fileTree("${projectDir}/distributions/internal/observabilitySRE/qa/acceptance")
     doFirst {
         // Generate the certificates first
         exec {
@@ -136,4 +116,4 @@ tasks.register("observabilitySREacceptanceTests", Test) {
         // Clean up the generated certificates
         delete fileTree("distributions/internal/observabilitySRE/qa/acceptance/docker/certs").include("*.key", "*.crt", "*.csr", "*.srl")
     }
-}
+}

x-pack/distributions/internal/observabilitySRE/README.md

@@ -0,0 +1,27 @@
+# ObservabilitySRE container DEV guide
+
+This is a top level guide for working on the ObservabiltySRE container locally. 
+
+## Building the image
+
+You can use the gradle task in the top level of the logstash dir. This will build an image based on your local checkout. 
+
+```
+./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
+```
+
+## Smoke tests
+
+The smoke tests are designed run against an image you have built locally. You can run the smoke tests with the helper in the `ci` dir. This sets the version of the ES fips image based on the logstash version. If you need to override that you can set the environment variables (see the script for details). 
+
+```
+./ci/observabilitySREsmoke_tests.sh
+```
+
+## Acceptance tests
+
+The acceptance tests are meant to run against an image that has been published to the elastic container repository. You can run the acceptance tests with the helper in the `ci` dir. This sets the version of the ES fips image based on the logstash version. If you need to override that you can set the environment variables (see the script for details). 
+
+```
+./ci/observabilitySREacceptance_tests.sh
+```

x-pack/distributions/internal/observabilitySRE/qa/acceptance/spec/acceptance_tests_spec.rb

@@ -1,83 +1,18 @@
-require 'net/http'
-require 'uri'
-require 'json'
-require 'timeout'
+require_relative '../../spec/shared_helpers.rb'
 
 describe "ObservabilitySRE FIPS container" do
-  def es_request(path, body = nil)
-    es_url = "https://localhost:9200"
-    es_user = 'elastic'
-    es_password = 'changeme'
-    uri = URI.parse(es_url + path)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.use_ssl = true
-    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-
-    request = body ? Net::HTTP::Post.new(uri.request_uri) : Net::HTTP::Get.new(uri.request_uri)
-    request.basic_auth(es_user, es_password)
-    request["Content-Type"] = "application/json"
-    request.body = body if body
-
-    http.request(request)
-  end
-
-  def wait_until(timeout: 30, interval: 1, message: nil)
-    Timeout.timeout(timeout) do
-      loop do
-        break if yield
-        sleep interval
-      end
-    end
-  rescue Timeout::Error
-    raise message || "Condition not met within #{timeout} seconds"
-  end
-  
-  def wait_for_elasticsearch(max_retries = 120)
-    retries = 0
-    ready = false
-
-    while !ready && retries < max_retries
-      begin
-        response = es_request("/_cluster/health")
-        if response.code == "200"
-          health = JSON.parse(response.body)
-          if ["green", "yellow"].include?(health["status"])
-            ready = true
-          end
-        end
-      rescue => e
-        puts "Waiting for Elasticsearch: #{e.message}"
-      ensure
-        unless ready
-          retries += 1
-          sleep 1
-          puts "Retry #{retries}/#{max_retries}"
-        end
-      end
-    end
-
-    raise "System not ready after #{max_retries} seconds" unless ready
-  end
-
-  def docker_compose_invoke(subcommand, env={})
-    env_str = env.map{ |k,v| "#{k.to_s.upcase}=#{Shellwords.escape(v)} "}.join
-    work_dir = Pathname.new("#{__dir__}/../docker").cleanpath
-    command = "#{env_str}docker-compose --project-directory=#{Shellwords.escape(work_dir)} #{subcommand}"
-    system(command) or fail "Failed to invoke Docker Compose with command `#{command}`"
-  end
-
-  def docker_compose_up(env={}) = docker_compose_invoke("up --detach", env)
-
-  def docker_compose_down(env={}) = docker_compose_invoke("down --volumes", env)
+  include SharedHelpers
 
   context "when running LS to ES with FIPS-compliant configuration" do
     before(:all) do
-      docker_compose_up
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_up({}, work_dir)
       wait_for_elasticsearch
     end
 
     after(:all) do
-      docker_compose_down
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_down({}, work_dir)
     end
 
     it "data flows from Logstash to Elasticsearch using FIPS-approved SSL" do
@@ -100,12 +35,14 @@ def docker_compose_down(env={}) = docker_compose_invoke("down --volumes", env)
 
   context "when running LS to ES with non-FIPS compliant configuration" do
     before(:all) do
-      docker_compose_up({"LOGSTASH_PIPELINE" => "logstash-to-elasticsearch-weak.conf"})
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_up({"LOGSTASH_PIPELINE" => "logstash-to-elasticsearch-weak.conf"}, work_dir)
       wait_for_elasticsearch
     end
 
     after(:all) do
-      docker_compose_down
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_down({}, work_dir)
     end
 
     it "prevents data flow when using TLSv1.1 which is not FIPS-compliant" do
@@ -129,14 +66,16 @@ def docker_compose_down(env={}) = docker_compose_invoke("down --volumes", env)
 
   context "When running Filebeat through LS to ES in a FIPS compliant configuration" do
     before(:all) do
-      docker_compose_up({"LOGSTASH_PIPELINE" => "filebeat-to-ls-to-es.conf"})
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_up({"LOGSTASH_PIPELINE" => "filebeat-to-ls-to-es.conf"}, work_dir)
       wait_for_elasticsearch
     end
-  
+
     after(:all) do
-      docker_compose_down
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_down({}, work_dir)
     end
-  
+
     it "data flows from Filebeat through Logstash to Elasticsearch" do
       # Wait for index to appear, indicating data is flowing
       wait_until(timeout: 30, message: "Index filebeat-test not found") do
@@ -157,12 +96,14 @@ def docker_compose_down(env={}) = docker_compose_invoke("down --volumes", env)
 
   context "when running Filebeat through LS to ES with non-FIPS compliant configuration" do
     before(:all) do
-      docker_compose_up({"LOGSTASH_PIPELINE" => "filebeat-to-ls-weak.conf"})
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_up({"LOGSTASH_PIPELINE" => "filebeat-to-ls-weak.conf"}, work_dir)
       wait_for_elasticsearch
     end
 
     after(:all) do
-      docker_compose_down
+      work_dir = File.expand_path("../docker", __dir__)
+      docker_compose_down({}, work_dir)
     end
 
     it "prevents data flow when using TLSv1.1 which is not FIPS-compliant" do

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/docker-compose.yml

@@ -1,24 +1,14 @@
-version: '3'
-
 services:
   elasticsearch:
-    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:8.19.0-SNAPSHOT
+    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${ELASTICSEARCH_IMAGE_VERSION:-8.19.0-SNAPSHOT}
     environment:
       - discovery.type=single-node
-      - xpack.security.enabled=true
+      - ES_JAVA_OPTS=-Xms1g -Xmx1g
       - ELASTIC_PASSWORD=changeme
-      - ES_JAVA_OPTS=-Xms512m -Xmx512m
-      - xpack.security.http.ssl.enabled=true
-      - xpack.security.http.ssl.key=/usr/share/elasticsearch/config/certs/elasticsearch.key
-      - xpack.security.http.ssl.certificate=/usr/share/elasticsearch/config/certs/elasticsearch.crt
-      - xpack.security.http.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/ca.crt
-      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.key=/usr/share/elasticsearch/config/certs/elasticsearch.key
-      - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/config/certs/elasticsearch.crt
-      - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/config/certs/ca.crt
     ports:
       - "9200:9200"
     volumes:
+      - ./elasticsearch/config/${ELASTICSEARCH_CONFIG:-elasticsearch-fips.yml}:/usr/share/elasticsearch/config/elasticsearch.yml
       - ./certs:/usr/share/elasticsearch/config/certs
     networks:
       - smoketest
@@ -40,12 +30,22 @@ services:
       - smoketest
 
   filebeat:
-    image: docker.elastic.co/beats/filebeat:8.19.0-SNAPSHOT
-    # Test runner mounts volume with non root user, do not require this file be root
-    entrypoint: "filebeat -e --strict.perms=false"
+    # The filebeat shipped with the elasticsearch-fips container is built for FIPS support
+    # There is no stand alone distribution. This uses the shipped version for testing. 
+    image: docker.elastic.co/cloud-release/elasticsearch-cloud-ess-fips:${FILEBEAT_IMAGE_VERSION:-8.19.0-SNAPSHOT}
+    container_name: fips_test_filebeat
+    working_dir: /usr/share/filebeat
+    entrypoint: ["/bin/bash", "-c"]
+    # Start Filebeat with /tmp for data (always writable)
+    command:
+      - |
+        exec /opt/filebeat/filebeat -e \
+          --strict.perms=false \
+          -c /usr/share/filebeat/filebeat.yml \
+          --path.data /tmp/filebeat_data
     volumes:
       - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-      - ./certs:/usr/share/filebeat/certs
+      - ./certs:/usr/share/filebeat/certs:ro
       - ./test-logs:/test-logs:ro
     depends_on:
       - logstash

x-pack/distributions/internal/observabilitySRE/qa/smoke/docker/elasticsearch/config/elasticsearch-fips.yml

@@ -0,0 +1,15 @@
+# Elasticsearch settings
+discovery.type: single-node
+http.port: 9200
+network.host: 0.0.0.0
+# Security settings
+xpack.security.enabled: true
+xpack.security.transport.ssl.enabled: true
+xpack.security.transport.ssl.verification_mode: certificate
+xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.transport.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certs/elasticsearch.key
+xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certs/elasticsearch.crt
+xpack.security.http.ssl.certificate_authorities: ["/usr/share/elasticsearch/config/certs/ca.crt"]