Cache binaries downloaded for packaging locally (#9133)

* Add an env variable to keep the download archive

* Only download artifacts which aren't present

* Document the KEEP_ARCHIVE env variable

* Update README.md

Co-authored-by: Panos Koutsovasilis <[email protected]>

* fixup! Only download artifacts which aren't present

---------

Co-authored-by: Panos Koutsovasilis <[email protected]>

Author: swiatekm

README.md

@@ -108,6 +108,11 @@ To build a local version of the agent for development, run the command below. Th
 DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64 PACKAGES=tar.gz mage -v package
 ```
 
+If you build the same agent package often (when running integration tests, for example),
+you can also set KEEP_ARCHIVE=true in your environment. The packaging step will then
+avoid deleting the binary archive after the package is generated, removing the need to
+re-download binaries on every invocation.
+
 The resulting package will be produced in the build/distributions directory. The version is controlled by the value in [version.go](version/version.go).
 To install the agent extract the package and run the install command:
 

dev-tools/mage/downloads/utils.go

@@ -31,13 +31,7 @@ type downloadRequest struct {
 // It writes to the destination file as it downloads it, without
 // loading the entire file into memory.
 func downloadFile(downloadRequest *downloadRequest) error {
-	targetFile, err := os.Create(downloadRequest.TargetPath)
-	if err != nil {
-		return fmt.Errorf("creating file: %w", err)
-	}
-	defer func() {
-		_ = targetFile.Close()
-	}()
+	stat, _ := os.Stat(downloadRequest.TargetPath)
 
 	exp := getExponentialBackoff(3)
 
@@ -47,6 +41,11 @@ func downloadFile(downloadRequest *downloadRequest) error {
 		if err != nil {
 			return fmt.Errorf("creating request: %w", err)
 		}
+		// if the target file already exists, add the If-Modified-Since header
+		if stat != nil {
+			req.Header.Add("If-Modified-Since", stat.ModTime().Format(http.TimeFormat))
+		}
+
 		resp, err := http.DefaultClient.Do(req)
 		if err != nil {
 			retryCount++
@@ -55,23 +54,36 @@ func downloadFile(downloadRequest *downloadRequest) error {
 		defer func() {
 			_ = resp.Body.Close()
 		}()
+
+		if resp.StatusCode == http.StatusNotModified {
+			return nil
+		}
+
+		targetFile, err := os.Create(downloadRequest.TargetPath)
+		if err != nil {
+			return fmt.Errorf("creating file: %w", err)
+		}
+		defer func() {
+			_ = targetFile.Close()
+		}()
+
 		_, err = io.Copy(targetFile, resp.Body)
 		if err != nil {
 			// try to drain the body before returning to ensure the connection can be reused
 			_, _ = io.Copy(io.Discard, resp.Body)
 			return fmt.Errorf("writing file %s: %w", targetFile.Name(), err)
 		}
 
+		_ = os.Chmod(targetFile.Name(), 0666)
+
 		return nil
 	}
 
-	err = backoff.Retry(download, exp)
+	err := backoff.Retry(download, exp)
 	if err != nil {
 		return err
 	}
 
-	_ = os.Chmod(targetFile.Name(), 0666)
-
 	return nil
 }
 

magefile.go

@@ -1199,12 +1199,16 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
 		log.Printf("dependencies extracted from package specs: %v", dependencies)
 	}
 
+	keepArchive := os.Getenv("KEEP_ARCHIVE") != ""
+
 	// download/copy all the necessary dependencies for packaging elastic-agent
 	archivePath, dropPath, dependencies := collectPackageDependencies(platforms, dependenciesVersion, packageTypes, dependencies)
 
 	// cleanup after build
-	defer os.RemoveAll(archivePath)
-	defer os.RemoveAll(dropPath)
+	if !keepArchive {
+		defer os.RemoveAll(archivePath)
+		defer os.RemoveAll(dropPath)
+	}
 	defer os.Unsetenv(agentDropPath)
 
 	// create flat dir