[9.1] (backport #18181) Remove redundant testing and circular dependency from docker acceptance testing (#18254)

* Remove redundant testing and circular dependency from docker acceptance testing (#18181)

* Use locally build artifact to build container from public dockerfile

Previously we would build an image (which would not actually be used), build
dockerfiles, modify dockerfiles to curl from
`https://snapshots.elastic.co/downloads/logstash'` then build the image used for
testing based on the modified dockerfile. This resulted in testing the last
published image to `snapshots`. This presents two problems 1. The test is
running against the last published image (not the tip of the branch being
tested) 2. this carries a dependency on both a DRA and unified stack release
having been run. Therefor acceptance tests will fail in between the time we bump
logstash version and a successful run of unified release.

This commit modifies the dockerfile to use the artifact prepared in the first
step instead of curling the last published one. This solves both issues as the
tests run against the code from the tip fo the branch being tested and there is
no dependency on an artifact existing as a result of a unified release pipeline.

* Remove redundant docker steps from workflows

Previously for docker acceptance tests three steps were performed: 1. Build
container images (based on local artifacts) 2. Build "public" dockerfiles 3.
Build container based on (a modified) file from step 2. The ONLY difference
between the dockerfile that ultimately is used to define an image between 1 and
2 is WHERE the logstash source is downloaded from. In acceptance testing we WANT
to use the source at the current checkout of logstash (not a remote). Using
remote causes a dependency issue when changes are not published. Publishing is
tied to unified release and gated on tests so naturally this is a bad fit for
that dependency.

This commit removes the redundancy by ONLY generating images for testing (step 1
from above). This also firms up our use of LOCAL_ARTIFACTS. Namely, the ONLY
time we want that set to `false` is when we build a "public" dockerfile. We
explicitly set that in the corresponding DRA script now. Similarly we explicitly
set it to `true` when testing.

* Remove unused function and argument

This commit removes the unused function for building from dockerfiles.
It also removes an unused argument for the make task for build_docker.

(cherry picked from commit a994c7c)

# Conflicts:
#	docker/Makefile
#	rakelib/artifacts.rake

* fix merge conflicts

---------

Co-authored-by: Cas Donoghue <[email protected]>

Author: mergify[bot]

.buildkite/scripts/dra/build_docker.sh

@@ -24,6 +24,9 @@ esac
 rake artifact:docker || error "artifact:docker build failed."
 rake artifact:docker_oss || error "artifact:docker_oss build failed."
 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
+
+# Generating public dockerfiles is the primary use case for NOT using local artifacts
+export LOCAL_ARTIFACTS=false
 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
 
 STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)"

ci/docker_acceptance_tests.sh

@@ -8,6 +8,9 @@ set -x
 export JRUBY_OPTS="-J-Xmx1g"
 export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
+# Use local artifacts for acceptance test Docker builds
+export LOCAL_ARTIFACTS=true
+
 if [ -n "$BUILD_JAVA_HOME" ]; then
   GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
 fi
@@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then
 elif [[ $SELECTED_TEST_SUITE == "full" ]]; then
   echo "--- Building $SELECTED_TEST_SUITE docker images"
   cd $LS_HOME
-  rake artifact:build_docker_full
+  rake artifact:docker
   echo "--- Acceptance: Installing dependencies"
   cd $QA_DIR
   bundle install

docker/Makefile

@@ -123,19 +123,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 	  release="${RELEASE}" \
 		image_flavor="full" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
 	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_full: public-dockerfiles_full
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_full && cd dockerfile_build_full && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) .
-
 public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
@@ -144,19 +137,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 	  release="${RELEASE}" \
 		image_flavor="oss" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
 	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_oss: public-dockerfiles_oss
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) .
-
 public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
@@ -165,18 +151,25 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 	  release="${RELEASE}" \
 		image_flavor="wolfi" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
 	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_wolfi: public-dockerfiles_wolfi
+public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES)
+	../vendor/jruby/bin/jruby -S erb -T "-"\
+		created_date="${BUILD_DATE}" \
+		elastic_version="${ELASTIC_VERSION}" \
+		arch="${ARCHITECTURE}" \
+		version_tag="${VERSION_TAG}" \
+		release="${RELEASE}" \
+		image_flavor="observability-sre" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
+		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \
 	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) .
+	cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \
+	tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
 public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
@@ -189,7 +182,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro
 		version_tag="${VERSION_TAG}" \
 	  release="${RELEASE}" \
 		image_flavor="ironbank" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
 	cd $(ARTIFACTS_DIR)/ironbank && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \

rakelib/artifacts.rake

@@ -169,7 +169,7 @@ namespace "artifact" do
 
   desc "Generate rpm, deb, tar and zip artifacts"
   task "all" => ["prepare", "build"]
-  task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"]
+  task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"]
 
   desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies"
   task "archives" => ["prepare", "generate_build_metadata"] do
@@ -371,39 +371,18 @@ namespace "artifact" do
     build_dockerfile('oss')
   end
 
-  namespace "dockerfile_oss" do
-    desc "Build Oss Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_oss"]  do
-      build_docker_from_dockerfiles('oss')
-    end
-  end
-
   desc "Generate Dockerfile for full images"
   task "dockerfile_full" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building full Dockerfiles")
     build_dockerfile('full')
   end
 
-  namespace "dockerfile_full" do
-    desc "Build Full Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_full"]  do
-      build_docker_from_dockerfiles('full')
-    end
-  end
-
   desc "Generate Dockerfile for wolfi images"
   task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building wolfi Dockerfiles")
     build_dockerfile('wolfi')
   end
 
-  namespace "dockerfile_wolfi" do
-    desc "Build Wolfi Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_wolfi"]  do
-      build_docker_from_dockerfiles('wolfi')
-    end
-  end
-
   desc "Generate build context for ironbank"
   task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building ironbank Dockerfiles")
@@ -429,24 +408,6 @@ namespace "artifact" do
     Rake::Task["artifact:archives_oss"].invoke
   end
 
-  task "build_docker_full" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker"].invoke
-    Rake::Task["artifact:dockerfile_full"].invoke
-    Rake::Task["artifact:dockerfile_full:docker"].invoke
-  end
-
-  task "build_docker_oss" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker_oss"].invoke
-    Rake::Task["artifact:dockerfile_oss"].invoke
-    Rake::Task["artifact:dockerfile_oss:docker"].invoke
-  end
-
-  task "build_docker_wolfi" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker_wolfi"].invoke
-    Rake::Task["artifact:dockerfile_wolfi"].invoke
-    Rake::Task["artifact:dockerfile_wolfi:docker"].invoke
-  end
-
   task "generate_build_metadata" do
     require 'time'
     require 'tempfile'
@@ -826,27 +787,13 @@ namespace "artifact" do
       "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
       "RELEASE" => ENV["RELEASE"],
       "VERSION_QUALIFIER" => VERSION_QUALIFIER,
-      "BUILD_DATE" => BUILD_DATE,
-      "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS
+      "BUILD_DATE" => BUILD_DATE
     }
     Dir.chdir("docker") do |dir|
       safe_system(env, "make build-from-local-#{flavor}-artifacts")
     end
   end
 
-  def build_docker_from_dockerfiles(flavor)
-    env = {
-      "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
-      "RELEASE" => ENV["RELEASE"],
-      "VERSION_QUALIFIER" => VERSION_QUALIFIER,
-      "BUILD_DATE" => BUILD_DATE,
-      "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS
-    }
-    Dir.chdir("docker") do |dir|
-      safe_system(env, "make build-from-dockerfiles_#{flavor}")
-    end
-  end
-
   def build_dockerfile(flavor)
     env = {
       "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),