Skip to content

[9.1] (backport #18181) Remove redundant testing and circular dependency from docker acceptance testing #18254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: 9.1
Choose a base branch
from
+42 −48
Open

[9.1] (backport #18181) Remove redundant testing and circular dependency from docker acceptance testing #18254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .buildkite/scripts/dra/build_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ esac
rake artifact:docker || error "artifact:docker build failed."
rake artifact:docker_oss || error "artifact:docker_oss build failed."
rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."

# Generating public dockerfiles is the primary use case for NOT using local artifacts
export LOCAL_ARTIFACTS=false
rake artifact:dockerfiles || error "artifact:dockerfiles build failed."

STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)"
Expand Down
5 changes: 4 additions & 1 deletion ci/docker_acceptance_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ set -x
export JRUBY_OPTS="-J-Xmx1g"
export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"

# Use local artifacts for acceptance test Docker builds
export LOCAL_ARTIFACTS=true

if [ -n "$BUILD_JAVA_HOME" ]; then
GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME"
fi
Expand Down Expand Up @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then
elif [[ $SELECTED_TEST_SUITE == "full" ]]; then
echo "--- Building $SELECTED_TEST_SUITE docker images"
cd $LS_HOME
rake artifact:build_docker_full
rake artifact:docker
echo "--- Acceptance: Installing dependencies"
cd $QA_DIR
bundle install
Expand Down
38 changes: 20 additions & 18 deletions docker/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,19 +123,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="full" \
local_artifacts="false" \
local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline

build-from-dockerfiles_full: public-dockerfiles_full
cd $(ARTIFACTS_DIR)/docker && \
mkdir -p dockerfile_build_full && cd dockerfile_build_full && \
tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) .

public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
created_date="${BUILD_DATE}" \
Expand All @@ -144,19 +137,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="oss" \
local_artifacts="false" \
local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline

build-from-dockerfiles_oss: public-dockerfiles_oss
cd $(ARTIFACTS_DIR)/docker && \
mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \
tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) .

public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
created_date="${BUILD_DATE}" \
Expand All @@ -165,18 +151,34 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="wolfi" \
local_artifacts="false" \
local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline

<<<<<<< HEAD
build-from-dockerfiles_wolfi: public-dockerfiles_wolfi
cd $(ARTIFACTS_DIR)/docker && \
mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \
tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \
sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \
docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) .
=======
public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
created_date="${BUILD_DATE}" \
elastic_version="${ELASTIC_VERSION}" \
arch="${ARCHITECTURE}" \
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="observability-sre" \
local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \
tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181))

public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
Expand All @@ -189,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="ironbank" \
local_artifacts="false" \
local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \
templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
cd $(ARTIFACTS_DIR)/ironbank && \
cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \
Expand Down
44 changes: 15 additions & 29 deletions rakelib/artifacts.rake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,11 @@ namespace "artifact" do

desc "Generate rpm, deb, tar and zip artifacts"
task "all" => ["prepare", "build"]
<<<<<<< HEAD
task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi"]
=======
task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"]
>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181))

desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies"
task "archives" => ["prepare", "generate_build_metadata"] do
Expand Down Expand Up @@ -371,11 +375,18 @@ namespace "artifact" do
build_dockerfile('oss')
end

<<<<<<< HEAD
namespace "dockerfile_oss" do
desc "Build Oss Docker image from Dockerfile context files"
task "docker" => ["archives_docker", "dockerfile_oss"] do
build_docker_from_dockerfiles('oss')
end
=======
desc "Generate Dockerfile for observability-sre images"
task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do
puts("[dockerfiles] Building observability-sre Dockerfile")
build_dockerfile('observability-sre')
>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181))
end

desc "Generate Dockerfile for full images"
Expand All @@ -384,26 +395,12 @@ namespace "artifact" do
build_dockerfile('full')
end

namespace "dockerfile_full" do
desc "Build Full Docker image from Dockerfile context files"
task "docker" => ["archives_docker", "dockerfile_full"] do
build_docker_from_dockerfiles('full')
end
end

desc "Generate Dockerfile for wolfi images"
task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do
puts("[dockerfiles] Building wolfi Dockerfiles")
build_dockerfile('wolfi')
end

namespace "dockerfile_wolfi" do
desc "Build Wolfi Docker image from Dockerfile context files"
task "docker" => ["archives_docker", "dockerfile_wolfi"] do
build_docker_from_dockerfiles('wolfi')
end
end

desc "Generate build context for ironbank"
task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do
puts("[dockerfiles] Building ironbank Dockerfiles")
Expand All @@ -429,6 +426,7 @@ namespace "artifact" do
Rake::Task["artifact:archives_oss"].invoke
end

<<<<<<< HEAD
task "build_docker_full" => [:generate_build_metadata] do
Rake::Task["artifact:docker"].invoke
Rake::Task["artifact:dockerfile_full"].invoke
Expand All @@ -447,6 +445,8 @@ namespace "artifact" do
Rake::Task["artifact:dockerfile_wolfi:docker"].invoke
end

=======
>>>>>>> a994c7cb (Remove redundant testing and circular dependency from docker acceptance testing (#18181))
task "generate_build_metadata" do
require 'time'
require 'tempfile'
Expand Down Expand Up @@ -826,27 +826,13 @@ namespace "artifact" do
"ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
"RELEASE" => ENV["RELEASE"],
"VERSION_QUALIFIER" => VERSION_QUALIFIER,
"BUILD_DATE" => BUILD_DATE,
"LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS
"BUILD_DATE" => BUILD_DATE
}
Dir.chdir("docker") do |dir|
safe_system(env, "make build-from-local-#{flavor}-artifacts")
end
end

def build_docker_from_dockerfiles(flavor)
env = {
"ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
"RELEASE" => ENV["RELEASE"],
"VERSION_QUALIFIER" => VERSION_QUALIFIER,
"BUILD_DATE" => BUILD_DATE,
"LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS
}
Dir.chdir("docker") do |dir|
safe_system(env, "make build-from-dockerfiles_#{flavor}")
end
end

def build_dockerfile(flavor)
env = {
"ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"),
Expand Down
Loading