Update the dataupload package to be compatible with the implementated inventory API

Signed-off-by: Richard Wall <[email protected]>

Author: wallrj

pkg/internal/cyberark/dataupload/dataupload.go

@@ -22,6 +22,11 @@ const (
 	// maxRetrievePresignedUploadURLBodySize is the maximum allowed size for a response body from the
 	// Retrieve Presigned Upload URL service.
 	maxRetrievePresignedUploadURLBodySize = 10 * 1024
+
+	// apiPathSnapshotLinks is the URL path of the snapshot-links endpoint of the inventory API.
+	// This endpoint returns an AWS presigned URL.
+	// TODO(wallrj): Link to CyberArk API documentation when it is published.
+	apiPathSnapshotLinks = "/api/ingestions/kubernetes/snapshot-links"
 )
 
 type CyberArkClient struct {
@@ -51,6 +56,9 @@ func NewCyberArkClient(trustedCAs *x509.CertPool, baseURL string, authenticateRe
 	}, nil
 }
 
+// PostDataReadingsWithOptions PUTs the supplied payload to an [AWS presigned URL] which it obtains via the CyberArk inventory API.
+//
+// [AWS presigned URL]: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payload api.DataReadingsPost, opts Options) error {
 	if opts.ClusterName == "" {
 		return fmt.Errorf("programmer mistake: the cluster name (aka `cluster_id` in the config file) cannot be left empty")
@@ -64,15 +72,15 @@ func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payloa
 
 	presignedUploadURL, err := c.retrievePresignedUploadURL(ctx, hex.EncodeToString(checksum.Sum(nil)), opts)
 	if err != nil {
-		return err
+		return fmt.Errorf("while retrieving snapshot upload URL: %s", err)
 	}
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, presignedUploadURL, encodedBody)
+	// The snapshot-links endpoint returns an AWS presigned URL which only supports the PUT verb.
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, presignedUploadURL, encodedBody)
 	if err != nil {
 		return err
 	}
 
-	req.Header.Set("Content-Type", "application/json")
 	version.SetUserAgent(req)
 
 	res, err := c.client.Do(req)
@@ -93,7 +101,7 @@ func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payloa
 }
 
 func (c *CyberArkClient) retrievePresignedUploadURL(ctx context.Context, checksum string, opts Options) (string, error) {
-	uploadURL, err := url.JoinPath(c.baseURL, "/api/data/kubernetes/upload")
+	uploadURL, err := url.JoinPath(c.baseURL, apiPathSnapshotLinks)
 	if err != nil {
 		return "", err
 	}
@@ -102,10 +110,12 @@ func (c *CyberArkClient) retrievePresignedUploadURL(ctx context.Context, checksu
 		ClusterID          string `json:"cluster_id"`
 		ClusterDescription string `json:"cluster_description"`
 		Checksum           string `json:"checksum_sha256"`
+		AgentVersion       string `json:"agent_version"`
 	}{
 		ClusterID:          opts.ClusterName,
 		ClusterDescription: opts.ClusterDescription,
 		Checksum:           checksum,
+		AgentVersion:       version.PreflightVersion,
 	}
 
 	encodedBody := &bytes.Buffer{}
@@ -120,7 +130,7 @@ func (c *CyberArkClient) retrievePresignedUploadURL(ctx context.Context, checksu
 
 	req.Header.Set("Content-Type", "application/json")
 	if err := c.authenticateRequest(req); err != nil {
-		return "", fmt.Errorf("failed to authenticate request")
+		return "", fmt.Errorf("failed to authenticate request: %s", err)
 	}
 	version.SetUserAgent(req)
 

pkg/internal/cyberark/dataupload/dataupload_test.go

@@ -75,7 +75,7 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) {
 			opts:         defaultOpts,
 			authenticate: setToken("fail-token"),
 			requireFn: func(t *testing.T, err error) {
-				require.ErrorContains(t, err, "received response with status code 500: should authenticate using the correct bearer token")
+				require.ErrorContains(t, err, "while retrieving snapshot upload URL: received response with status code 500: should authenticate using the correct bearer token")
 			},
 		},
 		{
@@ -84,16 +84,16 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) {
 			opts:         dataupload.Options{ClusterName: "invalid-json-retrieve-presigned", ClusterDescription: defaultOpts.ClusterDescription},
 			authenticate: setToken("success-token"),
 			requireFn: func(t *testing.T, err error) {
-				require.ErrorContains(t, err, "rejecting JSON response from server as it was too large or was truncated")
+				require.ErrorContains(t, err, "while retrieving snapshot upload URL: rejecting JSON response from server as it was too large or was truncated")
 			},
 		},
 		{
-			name:         "500 from server (PostData step)",
+			name:         "500 from server (RetrievePresignedUploadURL step)",
 			payload:      defaultPayload,
 			opts:         dataupload.Options{ClusterName: "invalid-response-post-data", ClusterDescription: defaultOpts.ClusterDescription},
 			authenticate: setToken("success-token"),
 			requireFn: func(t *testing.T, err error) {
-				require.ErrorContains(t, err, "received response with status code 500: mock error")
+				require.ErrorContains(t, err, "while retrieving snapshot upload URL: received response with status code 500: mock error")
 			},
 		},
 	}

pkg/internal/cyberark/dataupload/mock.go

@@ -37,7 +37,7 @@ func (mds *mockDataUploadServer) Close() {
 
 func (mds *mockDataUploadServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Path {
-	case "/api/data/kubernetes/upload":
+	case apiPathSnapshotLinks:
 		mds.handlePresignedUpload(w, r)
 		return
 	case "/presigned-upload":
@@ -123,7 +123,7 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r
 }
 
 func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Request, invalidJSON bool) {
-	if r.Method != http.MethodPost {
+	if r.Method != http.MethodPut {
 		w.WriteHeader(http.StatusMethodNotAllowed)
 		_, _ = w.Write([]byte(`{"message":"method not allowed"}`))
 		return
@@ -134,11 +134,6 @@ func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if r.Header.Get("Content-Type") != "application/json" {
-		http.Error(w, "should send JSON on all requests", http.StatusInternalServerError)
-		return
-	}
-
 	if invalidJSON {
 		w.WriteHeader(http.StatusOK)
 		w.Header().Set("Content-Type", "application/json")