Check app state before retrying on RESOURCE_ALREADY_EXISTS

Author: varundeepsaini

bundle/direct/dresources/all_test.go

@@ -719,3 +719,161 @@ func jsonDump(obj any) string {
 	}
 	return string(bytes)
 }
+
+// TestAppDoCreate_RetriesWhenAppIsDeleting verifies that DoCreate retries when
+// an app already exists but is in DELETING state.
+func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) {
+	server := testserver.New(t)
+
+	createCallCount := 0
+	getCallCount := 0
+
+	server.Handle("POST", "/api/2.0/apps", func(req testserver.Request) any {
+		createCallCount++
+		if createCallCount == 1 {
+			return testserver.Response{
+				StatusCode: 409,
+				Body: map[string]string{
+					"error_code": "RESOURCE_ALREADY_EXISTS",
+					"message":    "An app with the same name already exists.",
+				},
+			}
+		}
+		return apps.App{
+			Name: "test-app",
+			ComputeStatus: &apps.ComputeStatus{
+				State: apps.ComputeStateActive,
+			},
+		}
+	})
+
+	server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
+		getCallCount++
+		return apps.App{
+			Name: req.Vars["name"],
+			ComputeStatus: &apps.ComputeStatus{
+				State: apps.ComputeStateDeleting,
+			},
+		}
+	})
+
+	testserver.AddDefaultHandlers(server)
+
+	client, err := databricks.NewWorkspaceClient(&databricks.Config{
+		Host:  server.URL,
+		Token: "testtoken",
+	})
+	require.NoError(t, err)
+
+	r := (&ResourceApp{}).New(client)
+	ctx := context.Background()
+	name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"})
+
+	require.NoError(t, err)
+	assert.Equal(t, "test-app", name)
+	assert.Equal(t, 2, createCallCount, "expected Create to be called twice (1 retry)")
+	assert.Equal(t, 1, getCallCount, "expected Get to be called once to check app state")
+}
+
+// TestAppDoCreate_RetriesWhenGetReturnsNotFound verifies that DoCreate retries
+// when the app was just deleted between the create call and the get call.
+func TestAppDoCreate_RetriesWhenGetReturnsNotFound(t *testing.T) {
+	server := testserver.New(t)
+
+	createCallCount := 0
+	getCallCount := 0
+
+	server.Handle("POST", "/api/2.0/apps", func(req testserver.Request) any {
+		createCallCount++
+		if createCallCount == 1 {
+			return testserver.Response{
+				StatusCode: 409,
+				Body: map[string]string{
+					"error_code": "RESOURCE_ALREADY_EXISTS",
+					"message":    "An app with the same name already exists.",
+				},
+			}
+		}
+		return apps.App{
+			Name: "test-app",
+			ComputeStatus: &apps.ComputeStatus{
+				State: apps.ComputeStateActive,
+			},
+		}
+	})
+
+	server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
+		getCallCount++
+		return testserver.Response{
+			StatusCode: 404,
+			Body: map[string]string{
+				"error_code": "RESOURCE_DOES_NOT_EXIST",
+				"message":    "App not found.",
+			},
+		}
+	})
+
+	testserver.AddDefaultHandlers(server)
+
+	client, err := databricks.NewWorkspaceClient(&databricks.Config{
+		Host:  server.URL,
+		Token: "testtoken",
+	})
+	require.NoError(t, err)
+
+	r := (&ResourceApp{}).New(client)
+	ctx := context.Background()
+	name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"})
+
+	require.NoError(t, err)
+	assert.Equal(t, "test-app", name)
+	assert.Equal(t, 2, createCallCount, "expected Create to be called twice")
+	assert.Equal(t, 1, getCallCount, "expected Get to be called once to check app state")
+}
+
+// TestAppDoCreate_FailsWhenAppExistsAndNotDeleting verifies that DoCreate returns
+// a hard error when an app already exists but is NOT in DELETING state.
+func TestAppDoCreate_FailsWhenAppExistsAndNotDeleting(t *testing.T) {
+	server := testserver.New(t)
+
+	createCallCount := 0
+	getCallCount := 0
+
+	server.Handle("POST", "/api/2.0/apps", func(req testserver.Request) any {
+		createCallCount++
+		return testserver.Response{
+			StatusCode: 409,
+			Body: map[string]string{
+				"error_code": "RESOURCE_ALREADY_EXISTS",
+				"message":    "An app with the same name already exists.",
+			},
+		}
+	})
+
+	server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
+		getCallCount++
+		return apps.App{
+			Name: req.Vars["name"],
+			ComputeStatus: &apps.ComputeStatus{
+				State: apps.ComputeStateActive,
+			},
+		}
+	})
+
+	testserver.AddDefaultHandlers(server)
+
+	client, err := databricks.NewWorkspaceClient(&databricks.Config{
+		Host:  server.URL,
+		Token: "testtoken",
+	})
+	require.NoError(t, err)
+
+	r := (&ResourceApp{}).New(client)
+	ctx := context.Background()
+	_, _, err = r.DoCreate(ctx, &apps.App{Name: "test-app"})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "already exists")
+	assert.Equal(t, 1, createCallCount, "expected Create to be called only once")
+	assert.Equal(t, 1, getCallCount, "expected Get to be called once to check app state")
+}

bundle/direct/dresources/app.go

@@ -38,12 +38,25 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, *
 		ForceSendFields: nil,
 	}
 
-	retrier := retries.New[apps.App](retries.WithTimeout(time.Minute), retries.WithRetryFunc(shouldRetry))
+	retrier := retries.New[apps.App](retries.WithTimeout(15*time.Minute), retries.WithRetryFunc(shouldRetry))
 	app, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) {
 		waiter, err := r.client.Apps.Create(ctx, request)
 		if err != nil {
 			if errors.Is(err, apierr.ErrResourceAlreadyExists) {
-				return nil, retries.Continues("app already exists, retrying")
+				// Check if the app is in DELETING state - only then should we retry
+				existingApp, getErr := r.client.Apps.GetByName(ctx, config.Name)
+				if getErr != nil {
+					// If we can't get the app (e.g., it was just deleted), retry the create
+					if apierr.IsMissing(getErr) {
+						return nil, retries.Continues("app was deleted, retrying create")
+					}
+					return nil, retries.Halt(err)
+				}
+				if existingApp.ComputeStatus != nil && existingApp.ComputeStatus.State == apps.ComputeStateDeleting {
+					return nil, retries.Continues("app is deleting, retrying create")
+				}
+				// App exists and is not being deleted - this is a hard error
+				return nil, retries.Halt(err)
 			}
 			return nil, retries.Halt(err)
 		}