Verify download checksums during packaging (#9226) (#9555)

* Verify download checksums

* Add tests for checksum verification

(cherry picked from commit d5d0175)

Co-authored-by: Mikołaj Świątek <[email protected]>

Author: mergify[bot]

dev-tools/mage/downloads/utils.go

@@ -9,12 +9,17 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	devtools "github.com/elastic/elastic-agent/dev-tools/mage"
+
 	"github.com/cenkalti/backoff/v4"
 	"github.com/gofrs/uuid/v5"
 )
 
+var checksumFileRegex = regexp.MustCompile(`^([0-9a-f]{128})\s+(\w.*)$`)
+
 // downloadRequest struct contains download details ad path and URL
 type downloadRequest struct {
 	URL                 string
@@ -88,3 +93,32 @@ func downloadFile(downloadRequest *downloadRequest) error {
 
 	return nil
 }
+
+// verifyChecksum verifies a checksum file, with the content generated by the sha512sum program.
+// The format is the hex encoded checksum, followed by a space, and then the filename.
+// It is assumed that the files are in the same directory.
+func verifyChecksum(checksumFile string) error {
+	checksumFileContent, err := os.ReadFile(checksumFile)
+	if err != nil {
+		return fmt.Errorf("failed to read checksum file %s: %w", checksumFile, err)
+	}
+	strippedChecksumFileContent := strings.TrimSpace(string(checksumFileContent))
+	matches := checksumFileRegex.FindStringSubmatch(strippedChecksumFileContent)
+	if len(matches) != 3 {
+		return fmt.Errorf("checksum file %s has invalid format, expected `{checksum} {filename}`", checksumFile)
+	}
+	expectedChecksum := matches[1]
+	fileName := matches[2]
+
+	filePath := filepath.Join(filepath.Dir(checksumFile), fileName)
+	actualChecksum, err := devtools.GetSHA512Hash(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to compute checksum of file %s: %w", fileName, err)
+	}
+
+	if expectedChecksum != actualChecksum {
+		return fmt.Errorf("checksum of file %s does not match expected checksum of %s", fileName, actualChecksum)
+	}
+
+	return nil
+}

dev-tools/mage/downloads/utils_test.go

@@ -5,13 +5,17 @@
 package downloads
 
 import (
+	"crypto/sha512"
+	"encoding/hex"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -30,3 +34,59 @@ func TestDownloadFile(t *testing.T) {
 	assert.NotEmpty(t, dRequest.UnsanitizedFilePath)
 	defer os.Remove(filepath.Dir(dRequest.UnsanitizedFilePath))
 }
+
+func TestVerifyChecksum(t *testing.T) {
+	tmpDir := t.TempDir()
+	content := "hello world"
+	hashBytes := sha512.Sum512([]byte(content))
+	hashHex := hex.EncodeToString(hashBytes[:])
+
+	t.Run("valid checksum", func(t *testing.T) {
+		// Write the file to be verified
+		fileName := "testfile.txt"
+		require.NoError(t, os.WriteFile(filepath.Join(tmpDir, fileName), []byte(content), 0644))
+
+		// Write the checksum file
+		checksumContent := fmt.Sprintf("%s %s", hashHex, fileName)
+		checksumPath := filepath.Join(tmpDir, "checksum.txt")
+		require.NoError(t, os.WriteFile(checksumPath, []byte(checksumContent), 0644))
+
+		// Run test
+		err := verifyChecksum(checksumPath)
+		assert.NoError(t, err)
+	})
+
+	t.Run("missing checksum file", func(t *testing.T) {
+		err := verifyChecksum(filepath.Join(tmpDir, "missing.txt"))
+		assert.ErrorContains(t, err, "failed to read checksum file")
+	})
+
+	t.Run("malformed checksum content", func(t *testing.T) {
+		checksumPath := filepath.Join(tmpDir, "badchecksum.txt")
+		require.NoError(t, os.WriteFile(checksumPath, []byte("invalid-format-line"), 0644))
+
+		err := verifyChecksum(checksumPath)
+		assert.ErrorContains(t, err, "invalid format")
+	})
+
+	t.Run("missing target file", func(t *testing.T) {
+		checksumContent := fmt.Sprintf("%s %s", hashHex, "nonexistent.txt")
+		checksumPath := filepath.Join(tmpDir, "checksum_missing_target.txt")
+		require.NoError(t, os.WriteFile(checksumPath, []byte(checksumContent), 0644))
+
+		err := verifyChecksum(checksumPath)
+		assert.ErrorContains(t, err, "failed to open file for sha512 summing")
+	})
+
+	t.Run("checksum mismatch", func(t *testing.T) {
+		invalidContent := content + "x"
+		fileName := "file.txt"
+		require.NoError(t, os.WriteFile(filepath.Join(tmpDir, fileName), []byte(invalidContent), 0644))
+		checksumContent := fmt.Sprintf("%s %s", hashHex, fileName)
+		checksumPath := filepath.Join(tmpDir, "badhash.txt")
+		require.NoError(t, os.WriteFile(checksumPath, []byte(checksumContent), 0644))
+
+		err := verifyChecksum(checksumPath)
+		assert.ErrorContains(t, err, "does not match expected checksum")
+	})
+}

dev-tools/mage/downloads/versions.go

@@ -478,7 +478,14 @@ func FetchProjectBinaryForSnapshots(ctx context.Context, useCISnapshots bool, pr
 		return "", err
 	}
 	if downloadSHAFile && downloadShaURL != "" {
-		downloadLocation, err = handleDownload(downloadShaURL)
+		checksumFileLocation, err := handleDownload(downloadShaURL)
+		if err != nil {
+			return "", err
+		}
+		err = verifyChecksum(checksumFileLocation)
+		if err != nil {
+			return "", err
+		}
 	}
 	return downloadLocation, err
 }