From 256ede5e6c69953357fc7a3c6c6edd2157e9477f Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 17 Sep 2025 15:35:54 -0700 Subject: [PATCH 1/3] Use locally build artifact to build container from public dockerfile Previously we would build an image (which would not actually be used), build dockerfiles, modify dockerfiles to curl from `https://snapshots.elastic.co/downloads/logstash'` then build the image used for testing based on the modified dockerfile. This resulted in testing the last published image to `snapshots`. This presents two problems 1. The test is running against the last published image (not the tip of the branch being tested) 2. this carries a dependency on both a DRA and unified stack release having been run. Therefor acceptance tests will fail in between the time we bump logstash version and a successful run of unified release. This commit modifies the dockerfile to use the artifact prepared in the first step instead of curling the last published one. This solves both issues as the tests run against the code from the tip fo the branch being tested and there is no dependency on an artifact existing as a result of a unified release pipeline. --- docker/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/Makefile b/docker/Makefile index cdc9915b6bf..4aab90a12d8 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -142,7 +142,12 @@ build-from-dockerfiles_full: public-dockerfiles_full cd $(ARTIFACTS_DIR)/docker && \ mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ + cp ../../logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz . && \ + awk '/# Add Logstash itself and set permissions/{print; print "COPY logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz /tmp/logstash.tar.gz"; next}1' Dockerfile > Dockerfile.tmp && \ + sed '/curl --fail --location --output logstash.tar.gz.*tar.gz &&/d' Dockerfile.tmp > Dockerfile.tmp2 && \ + sed 's|tar -zxf logstash.tar.gz|tar -zxf /tmp/logstash.tar.gz|' Dockerfile.tmp2 > Dockerfile.tmp3 && \ + sed 's|rm logstash.tar.gz|rm /tmp/logstash.tar.gz|' Dockerfile.tmp3 > Dockerfile && \ + rm -f Dockerfile.tmp* && \ docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) From 2f2e3727bd467ae25fe0679ab96bce57fbe67936 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Fri, 26 Sep 2025 13:03:28 -0700 Subject: [PATCH 2/3] Remove redundant docker steps from workflows Previously for docker acceptance tests three steps were performed: 1. Build container images (based on local artifacts) 2. Build "public" dockerfiles 3. Build container based on (a modified) file from step 2. The ONLY difference between the dockerfile that ultimately is used to define an image between 1 and 2 is WHERE the logstash source is downloaded from. In acceptance testing we WANT to use the source at the current checkout of logstash (not a remote). Using remote causes a dependency issue when changes are not published. Publishing is tied to unified release and gated on tests so naturally this is a bad fit for that dependency. This commit removes the redundancy by ONLY generating images for testing (step 1 from above). This also firms up our use of LOCAL_ARTIFACTS. Namely, the ONLY time we want that set to `false` is when we build a "public" dockerfile. We explicitly set that in the corresponding DRA script now. Similarly we explicitly set it to `true` when testing. --- .buildkite/scripts/dra/build_docker.sh | 3 ++ ci/docker_acceptance_tests.sh | 5 ++- docker/Makefile | 43 +++----------------- rakelib/artifacts.rake | 54 +------------------------- 4 files changed, 13 insertions(+), 92 deletions(-) diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 0a5ee1998ec..3799b0a8251 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -24,6 +24,9 @@ esac rake artifact:docker || error "artifact:docker build failed." rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." + +# Generating public dockerfiles is the primary use case for NOT using local artifacts +export LOCAL_ARTIFACTS=false rake artifact:dockerfiles || error "artifact:dockerfiles build failed." STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)" diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index b1e62de2c26..74df2cc1023 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -8,6 +8,9 @@ set -x export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +# Use local artifacts for acceptance test Docker builds +export LOCAL_ARTIFACTS=true + if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:build_docker_full + rake artifact:docker echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/docker/Makefile b/docker/Makefile index 4aab90a12d8..c220f57f898 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -132,24 +132,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="full" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_full: public-dockerfiles_full - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - cp ../../logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz . && \ - awk '/# Add Logstash itself and set permissions/{print; print "COPY logstash-$(VERSION_TAG)-linux-$(ARCHITECTURE).tar.gz /tmp/logstash.tar.gz"; next}1' Dockerfile > Dockerfile.tmp && \ - sed '/curl --fail --location --output logstash.tar.gz.*tar.gz &&/d' Dockerfile.tmp > Dockerfile.tmp2 && \ - sed 's|tar -zxf logstash.tar.gz|tar -zxf /tmp/logstash.tar.gz|' Dockerfile.tmp2 > Dockerfile.tmp3 && \ - sed 's|rm logstash.tar.gz|rm /tmp/logstash.tar.gz|' Dockerfile.tmp3 > Dockerfile && \ - rm -f Dockerfile.tmp* && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . - public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -158,19 +146,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="oss" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_oss: public-dockerfiles_oss - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) . - public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -179,19 +160,12 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="wolfi" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_wolfi: public-dockerfiles_wolfi - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . - public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -200,19 +174,12 @@ public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(CO version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="observability-sre" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \ tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_observability-sre: public-dockerfiles_observability-sre - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_observability-sre && cd dockerfile_build_observability-sre && \ - tar -zxf ../../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-observability-sre:$(VERSION_TAG) . - public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ elastic_version="${ELASTIC_VERSION}" \ @@ -224,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="ironbank" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 87a57d262b2..9638d5babe6 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,7 +169,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi", "build_docker_observabilitySRE"] + task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -397,52 +397,24 @@ namespace "artifact" do build_dockerfile('oss') end - namespace "dockerfile_oss" do - desc "Build Oss Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_oss"] do - build_docker_from_dockerfiles('oss') - end - end - desc "Generate Dockerfile for observability-sre images" task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do puts("[dockerfiles] Building observability-sre Dockerfile") build_dockerfile('observability-sre') end - namespace "dockerfile_observabilitySRE" do - desc "Build ObservabilitySrE Docker image from Dockerfile context files" - task "docker" => ["archives_docker_observabilitySRE", "dockerfile_observabilitySRE"] do - build_docker_from_dockerfiles('observability-sre') - end - end - desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building full Dockerfiles") build_dockerfile('full') end - namespace "dockerfile_full" do - desc "Build Full Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_full"] do - build_docker_from_dockerfiles('full') - end - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end - namespace "dockerfile_wolfi" do - desc "Build Wolfi Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_wolfi"] do - build_docker_from_dockerfiles('wolfi') - end - end - desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -469,30 +441,6 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end - task "build_docker_full" => [:generate_build_metadata] do - Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:dockerfile_full"].invoke - Rake::Task["artifact:dockerfile_full:docker"].invoke - end - - task "build_docker_oss" => [:generate_build_metadata] do - Rake::Task["artifact:docker_oss"].invoke - Rake::Task["artifact:dockerfile_oss"].invoke - Rake::Task["artifact:dockerfile_oss:docker"].invoke - end - - task "build_docker_observabilitySRE" => [:generate_build_metadata] do - Rake::Task["artifact:docker_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE:docker"].invoke - end - - task "build_docker_wolfi" => [:generate_build_metadata] do - Rake::Task["artifact:docker_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi:docker"].invoke - end - task "generate_build_metadata" do require 'time' require 'tempfile' From a219f32c549285aea305c99f4837cba94954938f Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 1 Oct 2025 13:29:49 -0700 Subject: [PATCH 3/3] Remove unused function and argument This commit removes the unused function for building from dockerfiles. It also removes an unused argument for the make task for build_docker. --- rakelib/artifacts.rake | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 9638d5babe6..2739e1fc6a7 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -875,27 +875,13 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + "BUILD_DATE" => BUILD_DATE } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") end end - def build_docker_from_dockerfiles(flavor) - env = { - "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), - "RELEASE" => ENV["RELEASE"], - "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS - } - Dir.chdir("docker") do |dir| - safe_system(env, "make build-from-dockerfiles_#{flavor}") - end - end - def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),