forward-port observabilitySRE image creation into main (re-implament)

donoghuc · yaauie · commit 3a68a0914b76 · 2025-07-15T20:54:27.000Z
This is a forward-port of _functionality_ from the observabilitySRE feature branch into 8.x in PR #17541 (0b1d299), wholly re-implementing the changes in `docker/*` and `rakelib/artifacts.rake` from the 8.x-style docker structure to the refactored structure present on `main`.
diff --git a/build.gradle b/build.gradle
@@ -324,6 +324,34 @@ tasks.register("compileGrammar") {
     }
 }
 
+tasks.register("artifactDockerObservabilitySRE") {
+    dependsOn bootstrap
+    inputs.files fileTree("${projectDir}/rakelib")
+    inputs.files fileTree("${projectDir}/bin")
+    inputs.files fileTree("${projectDir}/config")
+    inputs.files fileTree("${projectDir}/lib")
+    inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
+    inputs.files fileTree("${projectDir}/logstash-core/lib")
+    inputs.files fileTree("${projectDir}/logstash-core/src")
+    inputs.files fileTree("${projectDir}/x-pack")
+    outputs.files fileTree("${buildDir}") {
+        include "Dockerfile-observability-sre"
+        include "logstash-observability-sre-${project.version}-SNAPSHOT-linux-*.tar.gz"
+        include "logstash-observability-sre-${project.version}-SNAPSHOT-docker-build-context.tar.gz"
+        include "plugin_aliases_hashed.yml"
+        include "jdk-*-linux-*.tar.gz"
+    }
+    doFirst {
+        if (!fedrampHighMode) {
+            logger.error("NOT in Fedramp High mode. Aborting.")
+            throw new GradleException("cannot build docker artifact for observabilitySRE without `-PfedrampHighMode=true`")
+        }
+    }
+    doLast {
+        rake(projectDir, buildDir, 'artifact:docker_observabilitySRE')
+    }
+}
+
 tasks.register("assembleTarDistribution") {
   dependsOn bootstrap
   inputs.files fileTree("${projectDir}/rakelib")
diff --git a/docker/Makefile b/docker/Makefile
@@ -20,7 +20,7 @@ else
   endif
 endif
 
-IMAGE_FLAVORS ?= oss full wolfi
+IMAGE_FLAVORS ?= oss full wolfi observability-sre
 DEFAULT_IMAGE_FLAVOR ?= full
 
 IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash
@@ -58,6 +58,15 @@ build-from-local-wolfi-artifacts: dockerfile
 	  (docker kill $(HTTPD); false);
 	-docker kill $(HTTPD)
 
+build-from-local-observability-sre-artifacts: dockerfile
+	docker run --rm -d --name=$(HTTPD) \
+	           -p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
+	           python:3 bash -c 'cd /mnt && python3 -m http.server'
+	timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-observability-sre:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-observability-sre data/logstash || \
+	  (docker kill $(HTTPD); false);
+	-docker kill $(HTTPD)
+
 COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
 COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
 COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml.go $(ARTIFACTS_DIR)/docker/env2yaml/go.mod $(ARTIFACTS_DIR)/docker/env2yaml/go.sum
@@ -113,15 +122,15 @@ ironbank_docker_paths:
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
 	mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline
 
-public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_ironbank
+public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_observability-sre public-dockerfiles_ironbank
 
 public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
 		elastic_version="${ELASTIC_VERSION}" \
 		arch="${ARCHITECTURE}" \
 		version_tag="${VERSION_TAG}" \
-	  release="${RELEASE}" \
+		release="${RELEASE}" \
 		image_flavor="full" \
 		local_artifacts="false" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
@@ -142,7 +151,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		elastic_version="${ELASTIC_VERSION}" \
 		arch="${ARCHITECTURE}" \
 		version_tag="${VERSION_TAG}" \
-	  release="${RELEASE}" \
+		release="${RELEASE}" \
 		image_flavor="oss" \
 		local_artifacts="false" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
@@ -163,7 +172,7 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		elastic_version="${ELASTIC_VERSION}" \
 		arch="${ARCHITECTURE}" \
 		version_tag="${VERSION_TAG}" \
-	  release="${RELEASE}" \
+		release="${RELEASE}" \
 		image_flavor="wolfi" \
 		local_artifacts="false" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
@@ -178,6 +187,27 @@ build-from-dockerfiles_wolfi: public-dockerfiles_wolfi
 	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
 	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) .
 
+public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES)
+	../vendor/jruby/bin/jruby -S erb -T "-"\
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+		release="${RELEASE}" \
+		image_flavor="observability-sre" \
+		local_artifacts="false" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \
+	cd $(ARTIFACTS_DIR)/docker && \
+	cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \
+	tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
+
+build-from-dockerfiles_observability-sre: public-dockerfiles_observability-sre
+	cd $(ARTIFACTS_DIR)/docker && \
+	mkdir -p dockerfile_build_observability-sre && cd dockerfile_build_observability-sre && \
+	tar -zxf ../../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz && \
+	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
+	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-observability-sre:$(VERSION_TAG) .
+
 public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 	  elastic_version="${ELASTIC_VERSION}" \
@@ -187,7 +217,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro
 		elastic_version="${ELASTIC_VERSION}" \
 		arch="${ARCHITECTURE}" \
 		version_tag="${VERSION_TAG}" \
-	  release="${RELEASE}" \
+		release="${RELEASE}" \
 		image_flavor="ironbank" \
 		local_artifacts="false" \
 		templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
diff --git a/docker/templates/Dockerfile.erb b/docker/templates/Dockerfile.erb
@@ -1,5 +1,5 @@
 # This Dockerfile was generated from templates/Dockerfile.erb
-<%# image_flavor 'full', oss', 'wolfi' -%>
+<%# image_flavor 'full', oss', 'wolfi', 'observability-sre' -%>
 <% if local_artifacts == 'false' -%>
   <%   url_root = 'https://artifacts.elastic.co/downloads/logstash' -%>
 <% else -%>
@@ -11,6 +11,9 @@
 <% elsif image_flavor == 'full' %>
   <%   tarball = "logstash-#{elastic_version}-linux-${arch}.tar.gz" -%>
   <%   license = 'Elastic License' -%>
+<% elsif image_flavor == 'observability-sre' -%><%# 'observability-sre' needs arch to be injected from the outside -%>
+  <%   tarball = "logstash-observability-sre-#{elastic_version}-linux-#{arch}.tar.gz" -%>
+  <%   license = 'Elastic License' -%>
 <% else -%><%# 'wolfi' needs arch to be injected from the outside -%>
   <%   tarball = "logstash-#{elastic_version}-linux-#{arch}.tar.gz" -%>
   <%   license = 'Elastic License' -%>
@@ -19,6 +22,10 @@
   <%   base_image = 'redhat/ubi9-minimal:latest' -%>
   <%   go_image = 'golang:1.23' -%>
   <%   package_manager = 'microdnf' -%>
+<% elsif image_flavor == 'observability-sre' -%>
+  <%   base_image = 'docker.elastic.co/wolfi/chainguard-base-fips' -%>
+  <%   go_image = 'docker.elastic.co/wolfi/go:1.23' -%>
+  <%   package_manager = 'apk' -%>
 <% else -%>
   <%   base_image = 'docker.elastic.co/wolfi/chainguard-base' -%>
   <%   go_image = 'docker.elastic.co/wolfi/go:1.23' -%>
@@ -52,7 +59,7 @@ RUN \
   <%= package_manager %> install -y openssl && \
   <%= package_manager %> install -y which shadow-utils && \
   <%= package_manager %> clean all
-<% else -%><%# 'wolfi' -%>
+<% else -%><%# 'wolfi', 'observability-sre' -%>
   <%= package_manager %> add --no-cache curl bash openssl
 <% end -%>
 
@@ -64,7 +71,7 @@ RUN groupadd --gid 1000 logstash && \
   --home "/usr/share/logstash" \
   --no-create-home \
   logstash && \
-<% else -%><%# 'wolfi' -%>
+<% else -%><%# 'wolfi', 'observability-sre' -%>
 RUN addgroup -g 1000 logstash && \
   adduser -u 1000 -G logstash \
   --disabled-password \
@@ -77,7 +84,7 @@ RUN addgroup -g 1000 logstash && \
 <% if image_flavor == 'full' || image_flavor == 'oss' -%>
   arch="$(rpm --query --queryformat='%{ARCH}' rpm)" && \
 <% end -%>
-  curl -f -Lo logstash.tar.gz <%= url_root %>/<%= tarball %> && \
+  curl --fail --location --output logstash.tar.gz <%= url_root %>/<%= tarball %> && \
   tar -zxf logstash.tar.gz -C /usr/share && \
   rm logstash.tar.gz && \
   mv /usr/share/logstash-<%= elastic_version %> /usr/share/logstash && \
@@ -93,12 +100,64 @@ COPY --from=builder-env2yaml /tmp/go/src/env2yaml/env2yaml /usr/local/bin/env2ya
 COPY --chown=logstash:root config/pipelines.yml config/log4j2.properties config/log4j2.file.properties /usr/share/logstash/config/
 <% if image_flavor == 'oss' -%>
 COPY --chown=logstash:root config/logstash-oss.yml /usr/share/logstash/config/logstash.yml
-<% else -%><%# 'full', 'wolfi' -%>
+<% else -%><%# 'full', 'wolfi', 'observability-sre' -%>
 COPY --chown=logstash:root config/logstash-full.yml /usr/share/logstash/config/logstash.yml
 <% end -%>
 COPY --chown=logstash:root pipeline/default.conf /usr/share/logstash/pipeline/logstash.conf
 COPY --chmod=0755 bin/docker-entrypoint /usr/local/bin/
 
+<% if image_flavor == 'observability-sre' -%>
+# Add FIPS configuration for observability-sre image flavor
+RUN mkdir -p /usr/share/logstash/config/security
+
+# Copy JVM security configuration files from the unpacked tarball
+RUN cp /usr/share/logstash/x-pack/distributions/internal/observabilitySRE/config/security/java.security /usr/share/logstash/config/security/ && \
+    cp /usr/share/logstash/x-pack/distributions/internal/observabilitySRE/config/security/java.policy /usr/share/logstash/config/security/ && \
+    chown --recursive logstash:root /usr/share/logstash/config/security/
+
+# list the classes provided by the fips BC
+RUN find /usr/share/logstash -name *.jar | grep lib
+
+# Convert JKS to BCFKS for truststore and keystore
+RUN /usr/share/logstash/jdk/bin/keytool -importkeystore \
+    -srckeystore /usr/share/logstash/jdk/lib/security/cacerts \
+    -destkeystore /usr/share/logstash/config/security/cacerts.bcfks \
+    -srcstoretype jks \
+    -deststoretype bcfks \
+    -providerpath /usr/share/logstash/logstash-core/lib/jars/bc-fips-2.0.0.jar \
+    -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \
+    -deststorepass changeit \
+    -srcstorepass changeit \
+    -noprompt
+
+RUN /usr/share/logstash/jdk/bin/keytool -importkeystore \
+    -srckeystore /usr/share/logstash/jdk/lib/security/cacerts \
+    -destkeystore /usr/share/logstash/config/security/keystore.bcfks \
+    -srcstoretype jks \
+    -deststoretype bcfks \
+    -providerpath /usr/share/logstash/logstash-core/lib/jars/bc-fips-2.0.0.jar \
+    -provider org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \
+    -deststorepass changeit \
+    -srcstorepass changeit \
+    -noprompt
+
+# Set Java security properties through LS_JAVA_OPTS
+ENV LS_JAVA_OPTS="\
+    -Djava.security.properties=/usr/share/logstash/config/security/java.security \
+    -Djava.security.policy=/usr/share/logstash/config/security/java.policy \
+    -Djavax.net.ssl.keyStore=/usr/share/logstash/config/security/keystore.bcfks \
+    -Djavax.net.ssl.keyStoreType=BCFKS \
+    -Djavax.net.ssl.keyStoreProvider=BCFIPS \
+    -Djavax.net.ssl.keyStorePassword=changeit \
+    -Djavax.net.ssl.trustStore=/usr/share/logstash/config/security/cacerts.bcfks \
+    -Djavax.net.ssl.trustStoreType=BCFKS \
+    -Djavax.net.ssl.trustStoreProvider=BCFIPS \
+    -Djavax.net.ssl.trustStorePassword=changeit \
+    -Dssl.KeyManagerFactory.algorithm=PKIX \
+    -Dssl.TrustManagerFactory.algorithm=PKIX \
+    -Dorg.bouncycastle.fips.approved_only=true"
+<% end -%>
+
 WORKDIR /usr/share/logstash
 
 USER 1000
diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake
@@ -163,7 +163,7 @@ namespace "artifact" do
 
   desc "Generate rpm, deb, tar and zip artifacts"
   task "all" => ["prepare", "build"]
-  task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"]
+  task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi", "build_docker_observabilitySRE"]
 
   desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies"
   task "archives" => ["prepare", "generate_build_metadata"] do
@@ -255,6 +255,27 @@ namespace "artifact" do
     safe_system("./gradlew bootstrap") # force the build of Logstash jars
   end
 
+  desc "Build jdk bundled tar.gz of observabilitySRE logstash plugins with all dependencies for docker"
+  task "archives_docker_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do
+    #with bundled JDKs
+    @bundles_jdk = true
+    exclude_paths = default_exclude_paths + %w(
+      bin/logstash-plugin
+      bin/logstash-plugin.bat
+      bin/logstash-keystore
+      bin/logstash-keystore.bat
+    )
+    license_details = ['ELASTIC-LICENSE','-observability-sre', exclude_paths]
+    %w(x86_64 arm64).each do |arch|
+      create_archive_pack(license_details, arch, "linux") do |dedicated_directory_tar|
+        # injection point: Use `DedicatedDirectoryTarball#write(source_file, destination_path)` to
+        # copy additional files into the tarball
+        puts "HELLO(#{dedicated_directory_tar})"
+      end
+    end
+    safe_system("./gradlew bootstrap") # force the build of Logstash jars
+  end
+
   desc "Build an RPM of logstash with all dependencies"
   task "rpm" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
@@ -353,6 +374,12 @@ namespace "artifact" do
     build_docker('oss')
   end
 
+  desc "Build observabilitySRE docker image"
+  task "docker_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata", "archives_docker_observabilitySRE"] do
+    puts("[docker_observabilitySRE] Building observabilitySRE docker image")
+    build_docker('observability-sre')
+  end
+
   desc "Build wolfi docker image"
   task "docker_wolfi" => %w(prepare generate_build_metadata archives_docker) do
     puts("[docker_wolfi] Building Wolfi docker image")
@@ -365,6 +392,7 @@ namespace "artifact" do
     build_dockerfile('oss')
     build_dockerfile('full')
     build_dockerfile('wolfi')
+    build_dockerfile('observability-sre')
     build_dockerfile('ironbank')
   end
 
@@ -381,6 +409,19 @@ namespace "artifact" do
     end
   end
 
+  desc "Generate Dockerfile for observability-sre images"
+  task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do
+    puts("[dockerfiles] Building observability-sre Dockerfile")
+    build_dockerfile('observability-sre')
+  end
+
+  namespace "dockerfile_observabilitySRE" do
+    desc "Build ObservabilitySrE Docker image from Dockerfile context files"
+    task "docker" => ["archives_docker_observabilitySRE", "dockerfile_observabilitySRE"] do
+      build_docker_from_dockerfiles('observability-sre')
+    end
+  end
+
   desc "Generate Dockerfile for full images"
   task "dockerfile_full" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building full Dockerfiles")
@@ -425,6 +466,7 @@ namespace "artifact" do
       Rake::Task["artifact:docker_wolfi"].invoke
       Rake::Task["artifact:dockerfiles"].invoke
       Rake::Task["artifact:docker_oss"].invoke
+      Rake::Task["artifact:docker_observabilitySRE"].invoke
     end
 
     Rake::Task["artifact:deb_oss"].invoke
@@ -444,6 +486,12 @@ namespace "artifact" do
     Rake::Task["artifact:dockerfile_oss:docker"].invoke
   end
 
+  task "build_docker_observabilitySRE" => [:generate_build_metadata] do
+    Rake::Task["artifact:docker_observabilitySRE"].invoke
+    Rake::Task["artifact:dockerfile_observabilitySRE"].invoke
+    Rake::Task["artifact:dockerfile_observabilitySRE:docker"].invoke
+  end
+
   task "build_docker_wolfi" => [:generate_build_metadata] do
     Rake::Task["artifact:docker_wolfi"].invoke
     Rake::Task["artifact:dockerfile_wolfi"].invoke
@@ -527,6 +575,17 @@ namespace "artifact" do
     end
   end
 
+  task "prepare-observabilitySRE" do
+    if ENV['SKIP_PREPARE'] != "1"
+      Rake::Task['bootstrap'].invoke
+      Rake::Task['plugin:install-default'].invoke
+      Rake::Task['plugin:install'].invoke('logstash-filter-age')
+      Rake::Task['plugin:trim-for-observabilitySRE'].invoke
+      Rake::Task['plugin:install-fips-validation-plugin'].invoke
+      Rake::Task['artifact:clean-bundle-config'].invoke
+    end
+  end
+
   def ensure_logstash_version_constant_defined
     # we do not want this file required when rake (ruby) parses this file
     # only when there is a task executing, not at the very top of this file
