From 290018240bbfd8f70cda4554ed1b5b6d20e799e9 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Wed, 1 Oct 2025 16:40:41 -0700 Subject: [PATCH 1/2] Remove redundant testing and circular dependency from docker acceptance testing (#18181) * Use locally build artifact to build container from public dockerfile Previously we would build an image (which would not actually be used), build dockerfiles, modify dockerfiles to curl from `https://snapshots.elastic.co/downloads/logstash'` then build the image used for testing based on the modified dockerfile. This resulted in testing the last published image to `snapshots`. This presents two problems 1. The test is running against the last published image (not the tip of the branch being tested) 2. this carries a dependency on both a DRA and unified stack release having been run. Therefor acceptance tests will fail in between the time we bump logstash version and a successful run of unified release. This commit modifies the dockerfile to use the artifact prepared in the first step instead of curling the last published one. This solves both issues as the tests run against the code from the tip fo the branch being tested and there is no dependency on an artifact existing as a result of a unified release pipeline. * Remove redundant docker steps from workflows Previously for docker acceptance tests three steps were performed: 1. Build container images (based on local artifacts) 2. Build "public" dockerfiles 3. Build container based on (a modified) file from step 2. The ONLY difference between the dockerfile that ultimately is used to define an image between 1 and 2 is WHERE the logstash source is downloaded from. In acceptance testing we WANT to use the source at the current checkout of logstash (not a remote). Using remote causes a dependency issue when changes are not published. Publishing is tied to unified release and gated on tests so naturally this is a bad fit for that dependency. This commit removes the redundancy by ONLY generating images for testing (step 1 from above). This also firms up our use of LOCAL_ARTIFACTS. Namely, the ONLY time we want that set to `false` is when we build a "public" dockerfile. We explicitly set that in the corresponding DRA script now. Similarly we explicitly set it to `true` when testing. * Remove unused function and argument This commit removes the unused function for building from dockerfiles. It also removes an unused argument for the make task for build_docker. (cherry picked from commit a994c7cb6e4cb4293a0830de77a5805573187cc7) # Conflicts: # docker/Makefile # rakelib/artifacts.rake --- .buildkite/scripts/dra/build_docker.sh | 3 ++ ci/docker_acceptance_tests.sh | 5 ++- docker/Makefile | 38 +++++++++++----------- rakelib/artifacts.rake | 44 +++++++++----------------- 4 files changed, 42 insertions(+), 48 deletions(-) diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 0a5ee1998ec..3799b0a8251 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -24,6 +24,9 @@ esac rake artifact:docker || error "artifact:docker build failed." rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." + +# Generating public dockerfiles is the primary use case for NOT using local artifacts +export LOCAL_ARTIFACTS=false rake artifact:dockerfiles || error "artifact:dockerfiles build failed." STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)" diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index b1e62de2c26..74df2cc1023 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -8,6 +8,9 @@ set -x export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +# Use local artifacts for acceptance test Docker builds +export LOCAL_ARTIFACTS=true + if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:build_docker_full + rake artifact:docker echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/docker/Makefile b/docker/Makefile index 5fd218b67e9..f376e0927d5 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -123,19 +123,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="full" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_full: public-dockerfiles_full - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . - public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -144,19 +137,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="oss" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_oss: public-dockerfiles_oss - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) . - public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -165,18 +151,34 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="wolfi" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline +<<<<<<< HEAD build-from-dockerfiles_wolfi: public-dockerfiles_wolfi cd $(ARTIFACTS_DIR)/docker && \ mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . +======= +public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES) + ../vendor/jruby/bin/jruby -S erb -T "-"\ + created_date="${BUILD_DATE}" \ + elastic_version="${ELASTIC_VERSION}" \ + arch="${ARCHITECTURE}" \ + version_tag="${VERSION_TAG}" \ + release="${RELEASE}" \ + image_flavor="observability-sre" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ + templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \ + cd $(ARTIFACTS_DIR)/docker && \ + cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \ + tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline +>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ @@ -189,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="ironbank" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 752c83ad318..26179547892 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,7 +169,11 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] +<<<<<<< HEAD task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"] +======= + task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] +>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -371,11 +375,18 @@ namespace "artifact" do build_dockerfile('oss') end +<<<<<<< HEAD namespace "dockerfile_oss" do desc "Build Oss Docker image from Dockerfile context files" task "docker" => ["archives_docker", "dockerfile_oss"] do build_docker_from_dockerfiles('oss') end +======= + desc "Generate Dockerfile for observability-sre images" + task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do + puts("[dockerfiles] Building observability-sre Dockerfile") + build_dockerfile('observability-sre') +>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) end desc "Generate Dockerfile for full images" @@ -384,26 +395,12 @@ namespace "artifact" do build_dockerfile('full') end - namespace "dockerfile_full" do - desc "Build Full Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_full"] do - build_docker_from_dockerfiles('full') - end - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end - namespace "dockerfile_wolfi" do - desc "Build Wolfi Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_wolfi"] do - build_docker_from_dockerfiles('wolfi') - end - end - desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -429,6 +426,7 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end +<<<<<<< HEAD task "build_docker_full" => [:generate_build_metadata] do Rake::Task["artifact:docker"].invoke Rake::Task["artifact:dockerfile_full"].invoke @@ -447,6 +445,8 @@ namespace "artifact" do Rake::Task["artifact:dockerfile_wolfi:docker"].invoke end +======= +>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) task "generate_build_metadata" do require 'time' require 'tempfile' @@ -826,27 +826,13 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + "BUILD_DATE" => BUILD_DATE } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") end end - def build_docker_from_dockerfiles(flavor) - env = { - "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), - "RELEASE" => ENV["RELEASE"], - "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS - } - Dir.chdir("docker") do |dir| - safe_system(env, "make build-from-dockerfiles_#{flavor}") - end - end - def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), From 330146787818055a0e32b93b1a62246ff2caf90f Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 6 Oct 2025 14:41:40 -0700 Subject: [PATCH 2/2] fix merge conflicts --- docker/Makefile | 9 --------- rakelib/artifacts.rake | 39 --------------------------------------- 2 files changed, 48 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index f376e0927d5..e29d68176d8 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -157,14 +157,6 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -<<<<<<< HEAD -build-from-dockerfiles_wolfi: public-dockerfiles_wolfi - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . -======= public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -178,7 +170,6 @@ public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(CO cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \ tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline ->>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 26179547892..0cd9e1caf48 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,11 +169,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] -<<<<<<< HEAD - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"] -======= task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] ->>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -375,20 +371,6 @@ namespace "artifact" do build_dockerfile('oss') end -<<<<<<< HEAD - namespace "dockerfile_oss" do - desc "Build Oss Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_oss"] do - build_docker_from_dockerfiles('oss') - end -======= - desc "Generate Dockerfile for observability-sre images" - task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do - puts("[dockerfiles] Building observability-sre Dockerfile") - build_dockerfile('observability-sre') ->>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) - end - desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building full Dockerfiles") @@ -426,27 +408,6 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end -<<<<<<< HEAD - task "build_docker_full" => [:generate_build_metadata] do - Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:dockerfile_full"].invoke - Rake::Task["artifact:dockerfile_full:docker"].invoke - end - - task "build_docker_oss" => [:generate_build_metadata] do - Rake::Task["artifact:docker_oss"].invoke - Rake::Task["artifact:dockerfile_oss"].invoke - Rake::Task["artifact:dockerfile_oss:docker"].invoke - end - - task "build_docker_wolfi" => [:generate_build_metadata] do - Rake::Task["artifact:docker_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi:docker"].invoke - end - -======= ->>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181)) task "generate_build_metadata" do require 'time' require 'tempfile'