Merge pull request #689 from jetstack/VC-43403-inventory-api-sha256

[VC-43403] Revert to using sha256 checksum to match latest snapshot-links API

Author: SgtCoDFish

pkg/internal/cyberark/dataupload/dataupload.go

@@ -3,8 +3,9 @@ package dataupload
 import (
 	"bytes"
 	"context"
-	"crypto/sha3"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -56,20 +57,33 @@ func NewCyberArkClient(trustedCAs *x509.CertPool, baseURL string, authenticateRe
 }
 
 // PostDataReadingsWithOptions PUTs the supplied payload to an [AWS presigned URL] which it obtains via the CyberArk inventory API.
-//
 // [AWS presigned URL]: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+//
+// A SHA256 checksum header is included in the request, to verify that the payload
+// has been received intact.
+// Read [Checking object integrity for data uploads in Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity-upload.html),
+// to learn more.
+//
+// TODO(wallrj): There is a bug in the AWS backend:
+// [S3 Presigned PutObjectCommand URLs ignore Sha256 Hash when uploading](https://github.com/aws/aws-sdk/issues/480)
+// ...which means that the `x-amz-checksum-sha256` request header is optional.
+// If you omit that header, it is possible to PUT any data.
+// There is a work around listed in that issue which we have shared with the
+// CyberArk API team.
 func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payload api.DataReadingsPost, opts Options) error {
 	if opts.ClusterName == "" {
 		return fmt.Errorf("programmer mistake: the cluster name (aka `cluster_id` in the config file) cannot be left empty")
 	}
 
 	encodedBody := &bytes.Buffer{}
-	checksum := sha3.New256()
-	if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(payload); err != nil {
+	hash := sha256.New()
+	if err := json.NewEncoder(io.MultiWriter(encodedBody, hash)).Encode(payload); err != nil {
 		return err
 	}
-
-	presignedUploadURL, err := c.retrievePresignedUploadURL(ctx, hex.EncodeToString(checksum.Sum(nil)), opts)
+	checksum := hash.Sum(nil)
+	checksumHex := hex.EncodeToString(checksum)
+	checksumBase64 := base64.StdEncoding.EncodeToString(checksum)
+	presignedUploadURL, err := c.retrievePresignedUploadURL(ctx, checksumHex, opts)
 	if err != nil {
 		return fmt.Errorf("while retrieving snapshot upload URL: %s", err)
 	}
@@ -79,7 +93,7 @@ func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payloa
 	if err != nil {
 		return err
 	}
-
+	req.Header.Set("X-Amz-Checksum-Sha256", checksumBase64)
 	version.SetUserAgent(req)
 
 	res, err := c.client.Do(req)
@@ -107,7 +121,7 @@ func (c *CyberArkClient) retrievePresignedUploadURL(ctx context.Context, checksu
 
 	request := struct {
 		ClusterID    string `json:"cluster_id"`
-		Checksum     string `json:"checksum_sha3"`
+		Checksum     string `json:"checksum_sha256"`
 		AgentVersion string `json:"agent_version"`
 	}{
 		ClusterID:    opts.ClusterName,

pkg/internal/cyberark/dataupload/mock.go

@@ -1,8 +1,8 @@
 package dataupload
 
 import (
-	"crypto/sha3"
-	"encoding/hex"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -76,7 +76,7 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r
 	decoder := json.NewDecoder(r.Body)
 	var req struct {
 		ClusterID    string `json:"cluster_id"`
-		Checksum     string `json:"checksum_sha3"`
+		Checksum     string `json:"checksum_sha256"`
 		AgentVersion string `json:"agent_version"`
 	}
 	decoder.DisallowUnknownFields()
@@ -112,12 +112,16 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r
 	// Write response body
 	w.WriteHeader(http.StatusOK)
 	w.Header().Set("Content-Type", "application/json")
-	presignedURL := mds.Server.URL + "/presigned-upload?checksum=" + req.Checksum
+	presignedURL := mds.Server.URL + "/presigned-upload"
 	_ = json.NewEncoder(w).Encode(struct {
 		URL string `json:"url"`
 	}{presignedURL})
 }
 
+// An example of a real checksum mismatch error from the AWS API when the
+// request body does not match the checksum in the request header.
+const amzExampleChecksumError = `<Error><Code>BadDigest</Code><Message>The SHA256 you specified did not match the calculated checksum.</Message><RequestId>GBDMP09BEZ929YBK</RequestId><HostId>sFTQb9JQpfJY/t+Ctn0anBmp4lKzEGES8ttmfAmFInuJIhvaV/U+20vYaGbdtlEnExZQRV/5xo6RQqq3xItM+px/Q2AEiv1G</HostId></Error>`
+
 func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Request, invalidJSON bool) {
 	if r.Method != http.MethodPut {
 		w.WriteHeader(http.StatusMethodNotAllowed)
@@ -137,11 +141,17 @@ func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	checksum := sha3.New256()
+	amzChecksum := r.Header.Get("X-Amz-Checksum-Sha256")
+	if amzChecksum == "" {
+		http.Error(w, "should set x-amz-checksum-sha256 header on all requests", http.StatusInternalServerError)
+		return
+	}
+
+	checksum := sha256.New()
 	_, _ = io.Copy(checksum, r.Body)
 
-	if r.URL.Query().Get("checksum") != hex.EncodeToString(checksum.Sum(nil)) {
-		http.Error(w, "checksum is invalid", http.StatusInternalServerError)
+	if amzChecksum != base64.StdEncoding.EncodeToString(checksum.Sum(nil)) {
+		http.Error(w, amzExampleChecksumError, http.StatusBadRequest)
 	}
 
 	w.WriteHeader(http.StatusOK)