diff --git a/pkg/internal/cyberark/dataupload/dataupload.go b/pkg/internal/cyberark/dataupload/dataupload.go index d8835e3b..e2b8a868 100644 --- a/pkg/internal/cyberark/dataupload/dataupload.go +++ b/pkg/internal/cyberark/dataupload/dataupload.go @@ -3,8 +3,9 @@ package dataupload import ( "bytes" "context" - "crypto/sha3" + "crypto/sha256" "crypto/x509" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -56,20 +57,33 @@ func NewCyberArkClient(trustedCAs *x509.CertPool, baseURL string, authenticateRe } // PostDataReadingsWithOptions PUTs the supplied payload to an [AWS presigned URL] which it obtains via the CyberArk inventory API. -// // [AWS presigned URL]: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// +// A SHA256 checksum header is included in the request, to verify that the payload +// has been received intact. +// Read [Checking object integrity for data uploads in Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity-upload.html), +// to learn more. +// +// TODO(wallrj): There is a bug in the AWS backend: +// [S3 Presigned PutObjectCommand URLs ignore Sha256 Hash when uploading](https://github.com/aws/aws-sdk/issues/480) +// ...which means that the `x-amz-checksum-sha256` request header is optional. +// If you omit that header, it is possible to PUT any data. +// There is a work around listed in that issue which we have shared with the +// CyberArk API team. func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payload api.DataReadingsPost, opts Options) error { if opts.ClusterName == "" { return fmt.Errorf("programmer mistake: the cluster name (aka `cluster_id` in the config file) cannot be left empty") } encodedBody := &bytes.Buffer{} - checksum := sha3.New256() - if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(payload); err != nil { + hash := sha256.New() + if err := json.NewEncoder(io.MultiWriter(encodedBody, hash)).Encode(payload); err != nil { return err } - - presignedUploadURL, err := c.retrievePresignedUploadURL(ctx, hex.EncodeToString(checksum.Sum(nil)), opts) + checksum := hash.Sum(nil) + checksumHex := hex.EncodeToString(checksum) + checksumBase64 := base64.StdEncoding.EncodeToString(checksum) + presignedUploadURL, err := c.retrievePresignedUploadURL(ctx, checksumHex, opts) if err != nil { return fmt.Errorf("while retrieving snapshot upload URL: %s", err) } @@ -79,7 +93,7 @@ func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payloa if err != nil { return err } - + req.Header.Set("X-Amz-Checksum-Sha256", checksumBase64) version.SetUserAgent(req) res, err := c.client.Do(req) @@ -107,7 +121,7 @@ func (c *CyberArkClient) retrievePresignedUploadURL(ctx context.Context, checksu request := struct { ClusterID string `json:"cluster_id"` - Checksum string `json:"checksum_sha3"` + Checksum string `json:"checksum_sha256"` AgentVersion string `json:"agent_version"` }{ ClusterID: opts.ClusterName, diff --git a/pkg/internal/cyberark/dataupload/mock.go b/pkg/internal/cyberark/dataupload/mock.go index f8a2530b..fc4c2331 100644 --- a/pkg/internal/cyberark/dataupload/mock.go +++ b/pkg/internal/cyberark/dataupload/mock.go @@ -1,8 +1,8 @@ package dataupload import ( - "crypto/sha3" - "encoding/hex" + "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "io" @@ -76,7 +76,7 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r decoder := json.NewDecoder(r.Body) var req struct { ClusterID string `json:"cluster_id"` - Checksum string `json:"checksum_sha3"` + Checksum string `json:"checksum_sha256"` AgentVersion string `json:"agent_version"` } decoder.DisallowUnknownFields() @@ -112,12 +112,16 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r // Write response body w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") - presignedURL := mds.Server.URL + "/presigned-upload?checksum=" + req.Checksum + presignedURL := mds.Server.URL + "/presigned-upload" _ = json.NewEncoder(w).Encode(struct { URL string `json:"url"` }{presignedURL}) } +// An example of a real checksum mismatch error from the AWS API when the +// request body does not match the checksum in the request header. +const amzExampleChecksumError = `BadDigestThe SHA256 you specified did not match the calculated checksum.GBDMP09BEZ929YBKsFTQb9JQpfJY/t+Ctn0anBmp4lKzEGES8ttmfAmFInuJIhvaV/U+20vYaGbdtlEnExZQRV/5xo6RQqq3xItM+px/Q2AEiv1G` + func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Request, invalidJSON bool) { if r.Method != http.MethodPut { w.WriteHeader(http.StatusMethodNotAllowed) @@ -137,11 +141,17 @@ func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Req return } - checksum := sha3.New256() + amzChecksum := r.Header.Get("X-Amz-Checksum-Sha256") + if amzChecksum == "" { + http.Error(w, "should set x-amz-checksum-sha256 header on all requests", http.StatusInternalServerError) + return + } + + checksum := sha256.New() _, _ = io.Copy(checksum, r.Body) - if r.URL.Query().Get("checksum") != hex.EncodeToString(checksum.Sum(nil)) { - http.Error(w, "checksum is invalid", http.StatusInternalServerError) + if amzChecksum != base64.StdEncoding.EncodeToString(checksum.Sum(nil)) { + http.Error(w, amzExampleChecksumError, http.StatusBadRequest) } w.WriteHeader(http.StatusOK)