diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 0a5ee1998ec..3799b0a8251 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -24,6 +24,9 @@ esac rake artifact:docker || error "artifact:docker build failed." rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." + +# Generating public dockerfiles is the primary use case for NOT using local artifacts +export LOCAL_ARTIFACTS=false rake artifact:dockerfiles || error "artifact:dockerfiles build failed." STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)" diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index b1e62de2c26..74df2cc1023 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -8,6 +8,9 @@ set -x export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +# Use local artifacts for acceptance test Docker builds +export LOCAL_ARTIFACTS=true + if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:build_docker_full + rake artifact:docker echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/docker/Makefile b/docker/Makefile index 5fd218b67e9..219062fec5b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -123,19 +123,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="full" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_full: public-dockerfiles_full - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . - public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -144,19 +137,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="oss" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_oss: public-dockerfiles_oss - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) . - public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -165,19 +151,12 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="wolfi" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_wolfi: public-dockerfiles_wolfi - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . - public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ elastic_version="${ELASTIC_VERSION}" \ @@ -189,7 +168,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="ironbank" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 752c83ad318..6b32438112a 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,7 +169,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"] + task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -384,26 +384,12 @@ namespace "artifact" do build_dockerfile('full') end - namespace "dockerfile_full" do - desc "Build Full Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_full"] do - build_docker_from_dockerfiles('full') - end - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end - namespace "dockerfile_wolfi" do - desc "Build Wolfi Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_wolfi"] do - build_docker_from_dockerfiles('wolfi') - end - end - desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -429,24 +415,6 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end - task "build_docker_full" => [:generate_build_metadata] do - Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:dockerfile_full"].invoke - Rake::Task["artifact:dockerfile_full:docker"].invoke - end - - task "build_docker_oss" => [:generate_build_metadata] do - Rake::Task["artifact:docker_oss"].invoke - Rake::Task["artifact:dockerfile_oss"].invoke - Rake::Task["artifact:dockerfile_oss:docker"].invoke - end - - task "build_docker_wolfi" => [:generate_build_metadata] do - Rake::Task["artifact:docker_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi:docker"].invoke - end - task "generate_build_metadata" do require 'time' require 'tempfile' @@ -826,27 +794,13 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + "BUILD_DATE" => BUILD_DATE } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") end end - def build_docker_from_dockerfiles(flavor) - env = { - "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), - "RELEASE" => ENV["RELEASE"], - "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS - } - Dir.chdir("docker") do |dir| - safe_system(env, "make build-from-dockerfiles_#{flavor}") - end - end - def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),