ci: pin elastic-agent version (#8736)

* ci: pin elastic-agent version

* ci: override BEAT_VERSION

* ci: fix mage cloud:load and cloud:push

Author: pkoutsovasilis

.buildkite/bk.integration-fips.pipeline.yml

@@ -7,6 +7,8 @@ env:
   IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1750467641"
   IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1750467641"
   IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1750467641"
+  # Remove AGENT_PACKAGE_VERSION and AGENT_VERSION pinning once 9.2.0 DRA and stack are released
+  AGENT_VERSION: "9.1.0-SNAPSHOT"
 
 steps:
   - label: Build and push custom elastic-agent image

.buildkite/bk.integration.pipeline.yml

@@ -12,6 +12,9 @@ env:
   IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1750467641"
   IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1750467641"
   IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1750467641"
+  # Remove AGENT_PACKAGE_VERSION and AGENT_VERSION pinning once 9.2.0 DRA and stack are released
+  AGENT_VERSION: "9.1.0-SNAPSHOT"
+
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins

.buildkite/integration.pipeline.yml

@@ -2,6 +2,9 @@
 
 env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  # Remove AGENT_PACKAGE_VERSION and AGENT_VERSION pinning once 9.2.0 DRA and stack are released
+  AGENT_PACKAGE_VERSION: "9.1.0"
+  BEAT_VERSION: "9.1.0"
 
 steps:
   - group: "Integration tests: packaging"

.buildkite/scripts/buildkite-integration-tests.ps1

@@ -25,9 +25,11 @@ go install gotest.tools/gotestsum
 gotestsum --version
 
 $env:TEST_BINARY_NAME = "elastic-agent"
-# Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
-$AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' })
-$env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT"
+if (-not $env:AGENT_VERSION) {
+    # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
+    $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' })
+    $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT"
+}
 
 Write-Output "~~~ Agent version: $env:AGENT_VERSION"
 $env:SNAPSHOT = $true

.buildkite/scripts/buildkite-integration-tests.sh

@@ -37,9 +37,11 @@ echo "~~~ Running integration tests as $USER"
 
 make install-gotestsum
 
-# Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
-AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)
-AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+if [[ -z "${AGENT_VERSION:-}" ]]; then
+  # Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
+  AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)
+  AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+fi
 
 export AGENT_VERSION
 echo "~~~ Agent version: ${AGENT_VERSION}"

.buildkite/scripts/common.sh

@@ -9,8 +9,11 @@ fi
 if [[ -z "${SETUP_GVM_VERSION-""}" ]]; then
     SETUP_GVM_VERSION="v0.5.2" # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
 fi
-BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
-export BEAT_VERSION
+
+if [[ -z "${BEAT_VERSION-""}" ]]; then
+  BEAT_VERSION=$(grep -oE '[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+[0-9]+)?' "${WORKSPACE}/version/version.go")
+  export BEAT_VERSION
+fi
 
 getOSOptions() {
   case $(uname | tr '[:upper:]' '[:lower:]') in

magefile.go

@@ -996,11 +996,10 @@ func (Cloud) Image(ctx context.Context) {
 // Looks in build/distributions for an elastic-agent-cloud*.docker.tar.gz artifact and imports it as docker.elastic.co/beats-ci/elastic-agent-cloud:$VERSION
 // DOCKER_IMPORT_SOURCE - override source for import
 func (Cloud) Load() error {
-	snapshot := os.Getenv(snapshotEnv)
-	defer os.Setenv(snapshotEnv, snapshot)
-	os.Setenv(snapshotEnv, "true")
-
-	version := getVersion()
+	agentVersion, err := mage.AgentPackageVersion()
+	if err != nil {
+		return fmt.Errorf("failed to get agent package version: %w", err)
+	}
 
 	// Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant
 	fips := os.Getenv(fipsEnv)
@@ -1009,12 +1008,14 @@ func (Cloud) Load() error {
 	if err != nil {
 		fipsVal = false
 	}
-	os.Setenv(fipsEnv, strconv.FormatBool(fipsVal))
+	if err := os.Setenv(fipsEnv, strconv.FormatBool(fipsVal)); err != nil {
+		return fmt.Errorf("failed to set fips env var: %w", err)
+	}
 	devtools.FIPSBuild = fipsVal
 
-	source := "build/distributions/elastic-agent-cloud-" + version + "-linux-" + runtime.GOARCH + ".docker.tar.gz"
+	source := "build/distributions/elastic-agent-cloud-" + agentVersion + "-SNAPSHOT-linux-" + runtime.GOARCH + ".docker.tar.gz"
 	if fipsVal {
-		source = "build/distributions/elastic-agent-cloud-fips-" + version + "-linux-" + runtime.GOARCH + ".docker.tar.gz"
+		source = "build/distributions/elastic-agent-cloud-fips-" + agentVersion + "-SNAPSHOT-linux-" + runtime.GOARCH + ".docker.tar.gz"
 	}
 	if envSource, ok := os.LookupEnv("DOCKER_IMPORT_SOURCE"); ok && envSource != "" {
 		source = envSource
@@ -1026,20 +1027,9 @@ func (Cloud) Load() error {
 // Push builds a cloud image tags it correctly and pushes to remote image repo.
 // Previous login to elastic registry is required!
 func (Cloud) Push() error {
-	snapshot := os.Getenv(snapshotEnv)
-	defer os.Setenv(snapshotEnv, snapshot)
-
-	os.Setenv(snapshotEnv, "true")
-
-	version := getVersion()
-	var tag string
-	if envTag, isPresent := os.LookupEnv("CUSTOM_IMAGE_TAG"); isPresent && len(envTag) > 0 {
-		tag = envTag
-	} else {
-		commit := dockerCommitHash()
-		time := time.Now().Unix()
-
-		tag = fmt.Sprintf("%s-%s-%d", version, commit, time)
+	agentVersion, err := mage.AgentPackageVersion()
+	if err != nil {
+		return fmt.Errorf("failed to get agent package version: %w", err)
 	}
 
 	// Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant
@@ -1049,38 +1039,46 @@ func (Cloud) Push() error {
 	if err != nil {
 		fipsVal = false
 	}
-	os.Setenv(fipsEnv, strconv.FormatBool(fipsVal))
+	if err := os.Setenv(fipsEnv, strconv.FormatBool(fipsVal)); err != nil {
+		return fmt.Errorf("failed to set fips env var: %w", err)
+	}
 	devtools.FIPSBuild = fipsVal
 
-	sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s", version)
+	sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s-SNAPSHOT", agentVersion)
 	if fipsVal {
-		sourceCloudImageName = fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud-fips:%s", version)
+		sourceCloudImageName = fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud-fips:%s-SNAPSHOT", agentVersion)
+	}
+	var targetTag string
+	if envTag, isPresent := os.LookupEnv("CUSTOM_IMAGE_TAG"); isPresent && len(envTag) > 0 {
+		targetTag = envTag
+	} else {
+		targetTag = fmt.Sprintf("%s-%s-%d", agentVersion, dockerCommitHash(), time.Now().Unix())
 	}
 	var targetCloudImageName string
 	if customImage, isPresent := os.LookupEnv("CI_ELASTIC_AGENT_DOCKER_IMAGE"); isPresent && len(customImage) > 0 {
-		targetCloudImageName = fmt.Sprintf("%s:%s", customImage, tag)
+		targetCloudImageName = fmt.Sprintf("%s:%s", customImage, targetTag)
 	} else {
-		targetCloudImageName = fmt.Sprintf(cloudImageTmpl, tag)
+		targetCloudImageName = fmt.Sprintf(cloudImageTmpl, targetTag)
 	}
 
 	fmt.Printf(">> Setting a docker image tag to %s\n", targetCloudImageName)
 	err = sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
 	if err != nil {
-		return fmt.Errorf("Failed setting a docker image tag: %w", err)
+		return fmt.Errorf("failed setting a docker image tag: %w", err)
 	}
 	fmt.Println(">> Docker image tag updated successfully")
 
 	fmt.Println(">> Pushing a docker image to remote registry")
 	err = sh.RunV("docker", "image", "push", targetCloudImageName)
 	if err != nil {
-		return fmt.Errorf("Failed pushing docker image: %w", err)
+		return fmt.Errorf("failed pushing docker image: %w", err)
 	}
 	fmt.Printf(">> Docker image pushed to remote registry successfully: %s\n", targetCloudImageName)
 
 	return nil
 }
 
-// Creates a new devmachine that will be auto-deleted in 6 hours.
+// Create a new devmachine that will be auto-deleted in 6 hours.
 // Example: MACHINE_IMAGE="family/platform-ingest-elastic-agent-ubuntu-2204" ZONE="us-central1-a" mage devmachine:create "pavel-dev-machine"
 // ZONE defaults to 'us-central1-a', MACHINE_IMAGE defaults to 'family/platform-ingest-elastic-agent-ubuntu-2204'
 func (Devmachine) Create(instanceName string) error {

testing/integration/ess/upgrade_broken_package_test.go

@@ -32,6 +32,7 @@ func TestUpgradeBrokenPackageVersion(t *testing.T) {
 		Local: false, // requires Agent installation
 		Sudo:  true,  // requires Agent installation
 	})
+	t.Skip("Skip this test until elastic agent version pinning is removed from Integration test runs on CI")
 
 	ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute))
 	defer cancel()