fix: retry bulk update endpoint on error (#3492)

Author: sweatybridge

pkg/function/batch.go

@@ -84,10 +84,15 @@ OUTER:
 		policy.Reset()
 	}
 	if len(toUpdate) > 1 {
-		if resp, err := s.client.V1BulkUpdateFunctionsWithResponse(ctx, s.project, toUpdate); err != nil {
-			return errors.Errorf("failed to bulk update: %w", err)
-		} else if resp.JSON200 == nil {
-			return errors.Errorf("unexpected bulk update status %d: %s", resp.StatusCode(), string(resp.Body))
+		if err := backoff.Retry(func() error {
+			if resp, err := s.client.V1BulkUpdateFunctionsWithResponse(ctx, s.project, toUpdate); err != nil {
+				return errors.Errorf("failed to bulk update: %w", err)
+			} else if resp.JSON200 == nil {
+				return errors.Errorf("unexpected bulk update status %d: %s", resp.StatusCode(), string(resp.Body))
+			}
+			return nil
+		}, policy); err != nil {
+			return err
 		}
 	}
 	return nil

pkg/function/batch_test.go

@@ -42,6 +42,65 @@ func TestUpsertFunctions(t *testing.T) {
 		era.eszip = &MockBundler{}
 	})
 
+	t.Run("deploys with bulk update", func(t *testing.T) {
+		// Setup mock api
+		defer gock.OffAll()
+		gock.New(mockApiHost).
+			Get("/v1/projects/" + mockProject + "/functions").
+			Reply(http.StatusOK).
+			JSON([]api.FunctionResponse{{Slug: "test-a"}})
+		gock.New(mockApiHost).
+			Patch("/v1/projects/" + mockProject + "/functions/test-a").
+			Reply(http.StatusOK).
+			JSON(api.FunctionResponse{Slug: "test-a"})
+		gock.New(mockApiHost).
+			Post("/v1/projects/" + mockProject + "/functions/test-b").
+			Reply(http.StatusOK).
+			JSON(api.FunctionResponse{Slug: "test-b"})
+		gock.New(mockApiHost).
+			Put("/v1/projects/" + mockProject + "/functions").
+			ReplyError(errors.New("network error"))
+		gock.New(mockApiHost).
+			Put("/v1/projects/" + mockProject + "/functions").
+			Reply(http.StatusServiceUnavailable)
+		gock.New(mockApiHost).
+			Put("/v1/projects/" + mockProject + "/functions").
+			Reply(http.StatusOK).
+			JSON(api.V1BulkUpdateFunctionsResponse{})
+		// Run test
+		err := client.UpsertFunctions(context.Background(), config.FunctionConfig{
+			"test-a": {},
+			"test-b": {},
+		})
+		// Check error
+		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
+	})
+
+	t.Run("handles concurrent deploy", func(t *testing.T) {
+		// Setup mock api
+		defer gock.OffAll()
+		gock.New(mockApiHost).
+			Get("/v1/projects/" + mockProject + "/functions").
+			Reply(http.StatusOK).
+			JSON([]api.FunctionResponse{})
+		gock.New(mockApiHost).
+			Post("/v1/projects/" + mockProject + "/functions").
+			Reply(http.StatusBadRequest).
+			BodyString("Duplicated function slug")
+		gock.New(mockApiHost).
+			Patch("/v1/projects/" + mockProject + "/functions/test").
+			Reply(http.StatusOK).
+			JSON(api.FunctionResponse{Slug: "test"})
+		// Run test
+		err := client.UpsertFunctions(context.Background(), config.FunctionConfig{
+			"test": {},
+		})
+		// Check error
+		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
+	})
+
 	t.Run("retries on network failure", func(t *testing.T) {
 		// Setup mock api
 		defer gock.OffAll()
@@ -84,6 +143,7 @@ func TestUpsertFunctions(t *testing.T) {
 		})
 		// Check error
 		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
 	})
 
 	t.Run("retries on update failure", func(t *testing.T) {
@@ -109,5 +169,6 @@ func TestUpsertFunctions(t *testing.T) {
 		})
 		// Check error
 		assert.NoError(t, err)
+		assert.Empty(t, apitest.ListUnmatchedRequests())
 	})
 }