Remove redundant docker steps from workflows

Previously for docker acceptance tests three steps were performed: 1. Build
container images (based on local artifacts) 2. Build "public" dockerfiles 3.
Build container based on (a modified) file from step 2. The ONLY difference
between the dockerfile that ultimately is used to define an image between 1 and
2 is WHERE the logstash source is downloaded from. In acceptance testing we WANT
to use the source at the current checkout of logstash (not a remote). Using
remote causes a dependency issue when changes are not published. Publishing is
tied to unified release and gated on tests so naturally this is a bad fit for
that dependency.

This commit removes the redundancy by ONLY generating images for testing (step 1
from above). This also firms up our use of LOCAL_ARTIFACTS. Namely, the ONLY
time we want that set to `false` is when we build a "public" dockerfile. We
explicitly set that in the corresponding DRA script now. Similarly we explicitly
set it to `true` when testing.

Author: donoghuc

.buildkite/scripts/dra/build_docker.sh

@@ -24,6 +24,9 @@ esac
 rake artifact:docker || error "artifact:docker build failed."
 rake artifact:docker_oss || error "artifact:docker_oss build failed."
 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
+
+# Generating public dockerfiles is the primary use case for NOT using local artifacts
+export LOCAL_ARTIFACTS=false
 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
 
 STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)"

ci/docker_acceptance_tests.sh

@@ -8,6 +8,9 @@ set -x
 export JRUBY_OPTS="-J-Xmx1g"
 export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
+# Use local artifacts for acceptance test Docker builds
+export LOCAL_ARTIFACTS=true
+
 if [ -n "$BUILD_JAVA_HOME" ]; then
   GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
 fi
@@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then
 elif [[ $SELECTED_TEST_SUITE == "full" ]]; then
   echo "--- Building $SELECTED_TEST_SUITE docker images"
   cd $LS_HOME
-  rake artifact:build_docker_full
+  rake artifact:docker
   echo "--- Acceptance: Installing dependencies"
   cd $QA_DIR
   bundle install

docker/Makefile

@@ -132,24 +132,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 		release="${RELEASE}" \
 		image_flavor="full" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
 	tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_full: public-dockerfiles_full
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_full && cd dockerfile_build_full && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	cp ../../logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz . && \
-	awk '/# Add Logstash itself and set permissions/{print; print "COPY logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz /tmp/logstash.tar.gz"; next}1' Dockerfile > Dockerfile.tmp && \
-	sed '/curl --fail --location --output logstash.tar.gz.*tar.gz &&/d' Dockerfile.tmp > Dockerfile.tmp2 && \
-	sed 's|tar -zxf logstash.tar.gz|tar -zxf /tmp/logstash.tar.gz|' Dockerfile.tmp2 > Dockerfile.tmp3 && \
-	sed 's|rm logstash.tar.gz|rm /tmp/logstash.tar.gz|' Dockerfile.tmp3 > Dockerfile && \
-	rm -f Dockerfile.tmp* && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) .
-
 public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
@@ -158,19 +146,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 		release="${RELEASE}" \
 		image_flavor="oss" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
 	tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_oss: public-dockerfiles_oss
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) .
-
 public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
@@ -179,19 +160,12 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 		version_tag="${VERSION_TAG}" \
 		release="${RELEASE}" \
 		image_flavor="wolfi" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
 	tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_wolfi: public-dockerfiles_wolfi
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \
-	tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) .
-
 public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 		created_date="${BUILD_DATE}" \
@@ -200,19 +174,12 @@ public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(CO
 		version_tag="${VERSION_TAG}" \
 		release="${RELEASE}" \
 		image_flavor="observability-sre" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \
 	cd $(ARTIFACTS_DIR)/docker && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \
 	tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
 
-build-from-dockerfiles_observability-sre: public-dockerfiles_observability-sre
-	cd $(ARTIFACTS_DIR)/docker && \
-	mkdir -p dockerfile_build_observability-sre && cd dockerfile_build_observability-sre && \
-	tar -zxf ../../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz && \
-	sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
-	docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-observability-sre:$(VERSION_TAG) .
-
 public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
 	../vendor/jruby/bin/jruby -S erb -T "-"\
 	  elastic_version="${ELASTIC_VERSION}" \
@@ -224,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro
 		version_tag="${VERSION_TAG}" \
 		release="${RELEASE}" \
 		image_flavor="ironbank" \
-		local_artifacts="false" \
+		local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
 		templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
 	cd $(ARTIFACTS_DIR)/ironbank && \
 	cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \

rakelib/artifacts.rake

@@ -169,7 +169,7 @@ namespace "artifact" do
 
   desc "Generate rpm, deb, tar and zip artifacts"
   task "all" => ["prepare", "build"]
-  task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi", "build_docker_observabilitySRE"]
+  task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"]
 
   desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies"
   task "archives" => ["prepare", "generate_build_metadata"] do
@@ -397,52 +397,24 @@ namespace "artifact" do
     build_dockerfile('oss')
   end
 
-  namespace "dockerfile_oss" do
-    desc "Build Oss Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_oss"]  do
-      build_docker_from_dockerfiles('oss')
-    end
-  end
-
   desc "Generate Dockerfile for observability-sre images"
   task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do
     puts("[dockerfiles] Building observability-sre Dockerfile")
     build_dockerfile('observability-sre')
   end
 
-  namespace "dockerfile_observabilitySRE" do
-    desc "Build ObservabilitySrE Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker_observabilitySRE", "dockerfile_observabilitySRE"] do
-      build_docker_from_dockerfiles('observability-sre')
-    end
-  end
-
   desc "Generate Dockerfile for full images"
   task "dockerfile_full" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building full Dockerfiles")
     build_dockerfile('full')
   end
 
-  namespace "dockerfile_full" do
-    desc "Build Full Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_full"]  do
-      build_docker_from_dockerfiles('full')
-    end
-  end
-
   desc "Generate Dockerfile for wolfi images"
   task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building wolfi Dockerfiles")
     build_dockerfile('wolfi')
   end
 
-  namespace "dockerfile_wolfi" do
-    desc "Build Wolfi Docker image from Dockerfile context files"
-    task "docker" => ["archives_docker", "dockerfile_wolfi"]  do
-      build_docker_from_dockerfiles('wolfi')
-    end
-  end
-
   desc "Generate build context for ironbank"
   task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do
     puts("[dockerfiles] Building ironbank Dockerfiles")
@@ -469,30 +441,6 @@ namespace "artifact" do
     Rake::Task["artifact:archives_oss"].invoke
   end
 
-  task "build_docker_full" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker"].invoke
-    Rake::Task["artifact:dockerfile_full"].invoke
-    Rake::Task["artifact:dockerfile_full:docker"].invoke
-  end
-
-  task "build_docker_oss" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker_oss"].invoke
-    Rake::Task["artifact:dockerfile_oss"].invoke
-    Rake::Task["artifact:dockerfile_oss:docker"].invoke
-  end
-
-  task "build_docker_observabilitySRE" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker_observabilitySRE"].invoke
-    Rake::Task["artifact:dockerfile_observabilitySRE"].invoke
-    Rake::Task["artifact:dockerfile_observabilitySRE:docker"].invoke
-  end
-
-  task "build_docker_wolfi" => [:generate_build_metadata] do
-    Rake::Task["artifact:docker_wolfi"].invoke
-    Rake::Task["artifact:dockerfile_wolfi"].invoke
-    Rake::Task["artifact:dockerfile_wolfi:docker"].invoke
-  end
-
   task "generate_build_metadata" do
     require 'time'
     require 'tempfile'