[8.18] (backport #17995) Build artifacts only for those that match host arch (#18072)

* Build artifacts only for those that match host arch (#17995)

* Build artifacts only for those that match host arch

Previously when doing platform specific builds (x86_64, arm) BOTH sets of
artifacts were produced regarless of host. This increases CI run times and
is needless resource consumption. This commit makes the rake tasks compile
artifacts for only the host arch it is being run on.

* set ARCH for building observabilitysre image

* Parallelize system package steps across different build hosts

This follows the pattern for prepping docker artifacts by preparing artifacts
for a target architecture on a specific build host. This allows us to process
x86 and arm packages in parallel across builders. Note that there is no
requirement today for the system packages to be built on an arch that matches
the target, spearating out to a separate builder opens up that opportunity for
us though.

(cherry picked from commit f2e4948)

# Conflicts:
#	.buildkite/pull_request_pipeline.yml
#	rakelib/artifacts.rake

* fix merge conflicts

* Ensure acceptance test artifacts are being built for correct arch (#18079)

* Ensure acceptance test artifacts are being built for correct arch

With the improvment in #17995 we only
build packages that match the architecture of the host. This is specified with
an enviornment variable. Previously the acceptance tests did not need to
specifcy the architecture when building packages. This commit updates the
builder to build packages with the correct arch.

* Ensure ARCH is set for docker acceptance tests

* fix merge conflict

---------

Co-authored-by: Cas Donoghue <[email protected]>

Author: mergify[bot]

.buildkite/aarch64_pipeline.yml

@@ -133,7 +133,7 @@ steps:
       - label: "Docker [{{matrix}}] flavor acceptance"
         command:
           set -euo pipefail
-          
+          export ARCH="aarch64"
           source .buildkite/scripts/common/vm-agent.sh && ci/docker_acceptance_tests.sh {{matrix}}
         retry:
           automatic:

.buildkite/scripts/dra/generatesteps.py

@@ -17,8 +17,8 @@ def to_bk_key_friendly_string(key):
 
 def package_x86_step(branch, workflow_type):
     step = f'''
-- label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts"
-  key: "logstash_build_packages_dra"
+- label: ":package: Build x86 packages / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_build_x86_packages_dra"
   agents:
     provider: gcp
     imageProject: elastic-images-prod
@@ -28,12 +28,37 @@ def package_x86_step(branch, workflow_type):
   command: |
     export WORKFLOW_TYPE="{workflow_type}"
     export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    export ARCH="x86_64"
     eval "$(rbenv init -)"
     .buildkite/scripts/dra/build_packages.sh
   artifact_paths:
     - "**/*.hprof"
 '''
+    return step
 
+def package_arm_step(branch, workflow_type):
+  # Note: this is currently using an x86 host. We can use a true arm host later 
+  # if we want. System packages dont rely on matching host arch to target artifact arch
+  # This just parallelizes the packaging for each across different build hosts for shorter
+  # total time
+    step = f'''
+- label: ":package: Build arm packages / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_build_arm_packages_dra"
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-16"
+    diskSizeGb: 200
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    ARCH="aarch64"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build_packages.sh
+  artifact_paths:
+    - "**/*.hprof"
+'''
     return step
 
 def package_x86_docker_step(branch, workflow_type):
@@ -106,6 +131,7 @@ def publish_dra_step(branch, workflow_type, depends_on):
 def build_steps_to_yaml(branch, workflow_type):
     steps = []
     steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type)))
+    steps.extend(yaml.safe_load(package_arm_step(branch, workflow_type)))
     steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type)))
     steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type)))
 

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -125,6 +125,7 @@ def acceptance_linux_steps() -> list[typing.Any]:
 source .buildkite/scripts/common/vm-agent.sh
 echo "--- Building all artifacts"
 ./gradlew clean bootstrap
+export ARCH="x86_64"
 rake artifact:deb artifact:rpm
 """),
         "artifact_paths": [
@@ -162,6 +163,7 @@ def acceptance_docker_steps()-> list[typing.Any]:
             "command": LiteralScalarString(f"""#!/usr/bin/env bash
 set -euo pipefail
 source .buildkite/scripts/common/vm-agent.sh
+export ARCH="x86_64"
 ci/docker_acceptance_tests.sh {flavor}"""),
             "retry": {"automatic": [{"limit": 3}]},
         })

rakelib/artifacts.rake

@@ -22,7 +22,13 @@ namespace "artifact" do
   VERSION_QUALIFIER = ENV["VERSION_QUALIFIER"].to_s.strip.empty? ? nil : ENV["VERSION_QUALIFIER"]
   LOCAL_ARTIFACTS = ENV["LOCAL_ARTIFACTS"] || "true"
   PACKAGE_SUFFIX = SNAPSHOT_BUILD ? "-SNAPSHOT" : ""
-
+  # Normalize ARCH environment variable to either x86_64 or arm64, defaulting to arm64
+  ARCH = case ENV["ARCH"]
+         when "x86_64"
+           "x86_64"
+         else
+           "arm64" # default for aarch64, arm64, or any other value (including nil)
+         end
   ## TODO: Install new service files
   def package_files
     res = [
@@ -172,9 +178,11 @@ namespace "artifact" do
     #with bundled JDKs
     license_details = ['ELASTIC-LICENSE']
     @bundles_jdk = true
-    create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
-
+    if ARCH == "x86_64"
+      create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
+    else
+      create_archive_pack(license_details, "arm64", "linux", "darwin")
+    end
     #without JDK
     safe_system("./gradlew bootstrap") #force the build of Logstash jars
     @bundles_jdk = false
@@ -186,8 +194,7 @@ namespace "artifact" do
   task "archives_docker" => ["prepare", "generate_build_metadata"] do
     license_details = ['ELASTIC-LICENSE']
     @bundles_jdk = true
-    create_archive_pack(license_details, "x86_64", "linux", "darwin")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
+    create_archive_pack(license_details, ARCH, "linux", "darwin")
     safe_system("./gradlew bootstrap") # force the build of Logstash jars
   end
 
@@ -237,8 +244,11 @@ namespace "artifact" do
     #with bundled JDKs
     @bundles_jdk = true
     license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths]
-    create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
+    if ARCH == "x86_64"
+      create_archive_pack(license_details, "x86_64", "linux", "windows", "darwin")
+    else
+      create_archive_pack(license_details, "arm64", "linux", "darwin")
+    end
 
     #without JDK
     @bundles_jdk = false
@@ -252,20 +262,16 @@ namespace "artifact" do
     #with bundled JDKs
     @bundles_jdk = true
     license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths]
-    create_archive_pack(license_details, "x86_64", "linux", "darwin")
-    create_archive_pack(license_details, "arm64", "linux", "darwin")
+    create_archive_pack(license_details, ARCH, "linux", "darwin")
     safe_system("./gradlew bootstrap") # force the build of Logstash jars
   end
 
   desc "Build an RPM of logstash with all dependencies"
   task "rpm" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
-    puts("[artifact:rpm] building rpm package x86_64")
-    package_with_jdk("centos", "x86_64")
-
-    puts("[artifact:rpm] building rpm package arm64")
-    package_with_jdk("centos", "arm64")
+    puts("[artifact:rpm] building rpm package #{ARCH}")
+    package_with_jdk("centos", ARCH)
 
     #without JDKs
     @bundles_jdk = false
@@ -277,11 +283,8 @@ namespace "artifact" do
   task "rpm_oss" => ["prepare-oss", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
-    puts("[artifact:rpm] building rpm OSS package x86_64")
-    package_with_jdk("centos", "x86_64", :oss)
-
-    puts("[artifact:rpm] building rpm OSS package arm64")
-    package_with_jdk("centos", "arm64", :oss)
+    puts("[artifact:rpm] building rpm OSS package #{ARCH}")
+    package_with_jdk("centos", ARCH, :oss)
 
     #without JDKs
     @bundles_jdk = false
@@ -293,11 +296,8 @@ namespace "artifact" do
   task "deb" => ["prepare", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
-    puts("[artifact:deb] building deb package for x86_64")
-    package_with_jdk("ubuntu", "x86_64")
-
-    puts("[artifact:deb] building deb package for OS: linux arm64")
-    package_with_jdk("ubuntu", "arm64")
+    puts("[artifact:deb] building deb package for #{ARCH}")
+    package_with_jdk("ubuntu", ARCH)
 
     #without JDKs
     @bundles_jdk = false
@@ -309,11 +309,8 @@ namespace "artifact" do
   task "deb_oss" => ["prepare-oss", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
-    puts("[artifact:deb_oss] building deb OSS package x86_64")
-    package_with_jdk("ubuntu", "x86_64", :oss)
-
-    puts("[artifact:deb_oss] building deb OSS package arm64")
-    package_with_jdk("ubuntu", "arm64", :oss)
+    puts("[artifact:deb_oss] building deb OSS package #{ARCH}")
+    package_with_jdk("ubuntu", ARCH, :oss)
 
     #without JDKs
     @bundles_jdk = false