Add KillCaller feature to test server for CLI crash recovery testing (#4146)

Closes: #4140 
## Changes

Add `KillCaller` option to test server that terminates the CLI process
when specific endpoints are hit. This enables testing crash recovery
scenarios like `bundle deploy` interrupted mid-execution.

`KillCaller` specifies how many times to kill before allowing normal
responses, enabling retry/recovery testing.

```toml
[[Server]]
Pattern = "GET /api/2.0/workspace/list"
KillCaller = 2  # Kill first 2 calls, then allow
```

## Why

Enables testing CLI crash recovery scenarios (e.g., `bundle deploy`
interrupted mid-execution).

## Tests

Added `acceptance/selftest/kill_caller/` tests.

---------

Signed-off-by: Varun Deep Saini <[email protected]>
Co-authored-by: Denis Bilenko <[email protected]>

Author: varundeepsaini

acceptance/internal/config.go

@@ -142,6 +142,12 @@ type ServerStub struct {
 	// Configure as "1ms", "2s", "3m", etc.
 	// See [time.ParseDuration] for details.
 	Delay time.Duration
+
+	// Number of times to kill the caller process before returning normal responses.
+	// 0 = never kill (default), 1 = kill once then allow, 2 = kill twice then allow, etc.
+	// Useful for testing crash recovery scenarios where first deploy crashes but retry succeeds.
+	// Requires DATABRICKS_CLI_TEST_PID=1 to be set in the test environment.
+	KillCaller int
 }
 
 // FindConfigs finds all the config relevant for this test,

acceptance/internal/prepare_server.go

@@ -184,13 +184,23 @@ func startLocalServer(t *testing.T,
 		s.ResponseCallback = logResponseCallback(t)
 	}
 
+	// Track remaining kill counts per pattern (for KillCaller > 0)
+	killCounters := make(map[string]int)
+	killCountersMu := &sync.Mutex{}
+
 	for ind := range stubs {
 		// We want later stubs takes precedence, because then leaf configs take precedence over parent directory configs
 		// In gorilla/mux earlier handlers take precedence, so we need to reverse the order
 		stub := stubs[len(stubs)-1-ind]
 		require.NotEmpty(t, stub.Pattern)
 		items := strings.Split(stub.Pattern, " ")
 		require.Len(t, items, 2)
+
+		// Initialize kill counter for this pattern
+		if stub.KillCaller > 0 {
+			killCounters[stub.Pattern] = stub.KillCaller
+		}
+
 		s.Handle(items[0], items[1], func(req testserver.Request) any {
 			if stub.Delay > 0 {
 				ctx := req.Context
@@ -209,6 +219,11 @@ func startLocalServer(t *testing.T,
 				}
 			}
 
+			if shouldKillCaller(stub, killCounters, killCountersMu) {
+				killCaller(t, stub.Pattern, req.Headers)
+				return testserver.Response{StatusCode: http.StatusOK}
+			}
+
 			return stub.Response
 		})
 	}
@@ -218,6 +233,42 @@ func startLocalServer(t *testing.T,
 	return s.URL
 }
 
+func shouldKillCaller(stub ServerStub, killCounters map[string]int, mu *sync.Mutex) bool {
+	if stub.KillCaller <= 0 {
+		return false
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if killCounters[stub.Pattern] <= 0 {
+		return false
+	}
+	killCounters[stub.Pattern]--
+	return true
+}
+
+func killCaller(t *testing.T, pattern string, headers http.Header) {
+	pid := testserver.ExtractPidFromHeaders(headers)
+	if pid == 0 {
+		t.Errorf("KillCaller configured but test-pid not found in User-Agent")
+		return
+	}
+
+	process, err := os.FindProcess(pid)
+	if err != nil {
+		t.Errorf("Failed to find process %d: %s", pid, err)
+		return
+	}
+
+	// Use process.Kill() for cross-platform compatibility.
+	// On Unix, this sends SIGKILL. On Windows, this calls TerminateProcess.
+	if err := process.Kill(); err != nil {
+		t.Errorf("Failed to kill process %d: %s", pid, err)
+		return
+	}
+
+	t.Logf("KillCaller: killed PID %d (pattern: %s)", pid, pattern)
+}
+
 func startProxyServer(t *testing.T,
 	logRequests bool,
 	includeHeaders []string,

acceptance/selftest/kill_caller/currentuser/out.test.toml

@@ -0,0 +1,6 @@
+
+>>> errcode [CLI] current-user me
+[PROCESS_KILLED]
+
+Exit code: [KILLED]
+Script continued after kill

acceptance/selftest/kill_caller/currentuser/output.txt

@@ -0,0 +1,2 @@
+trace errcode $CLI current-user me
+echo "Script continued after kill"

acceptance/selftest/kill_caller/currentuser/script

@@ -0,0 +1,4 @@
+# Kill the CLI when it calls /Me endpoint (once, then allow)
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/Me"
+KillCaller = 1

acceptance/selftest/kill_caller/currentuser/test.toml

@@ -0,0 +1,29 @@
+
+>>> errcode [CLI] current-user me
+[PROCESS_KILLED]
+
+Exit code: [KILLED]
+Me attempt 1 done
+
+>>> errcode [CLI] current-user me
+[PROCESS_KILLED]
+
+Exit code: [KILLED]
+Me attempt 2 done
+
+>>> [CLI] current-user me
+{
+  "id":"123",
+  "userName":"[email protected]"
+}
+Me attempt 3 done - success!
+
+>>> errcode [CLI] workspace list /
+[PROCESS_KILLED]
+
+Exit code: [KILLED]
+Workspace attempt 1 done
+
+>>> [CLI] workspace list /
+ID  Type  Language  Path
+Workspace attempt 2 done - success!

acceptance/selftest/kill_caller/multi_pattern/out.test.toml

@@ -0,0 +1,16 @@
+# Test pattern 1: /Me endpoint (kills first 2, then allows)
+trace errcode $CLI current-user me
+echo "Me attempt 1 done"
+
+trace errcode $CLI current-user me
+echo "Me attempt 2 done"
+
+trace $CLI current-user me
+echo "Me attempt 3 done - success!"
+
+# Test pattern 2: /workspace/list endpoint (kills first 1, then allows)
+trace errcode $CLI workspace list /
+echo "Workspace attempt 1 done"
+
+trace $CLI workspace list /
+echo "Workspace attempt 2 done - success!"

acceptance/selftest/kill_caller/multi_pattern/output.txt

@@ -0,0 +1,17 @@
+# Test that multiple patterns can have independent KillCaller counts
+# Pattern 1: Kill first 2 requests to /Me endpoint
+# Pattern 2: Kill first 1 request to /workspace/list endpoint
+
+[[Server]]
+Pattern = "GET /api/2.0/preview/scim/v2/Me"
+KillCaller = 2
+Response.Body = '''
+{
+    "id": "123",
+    "userName": "[email protected]"
+}
+'''
+
+[[Server]]
+Pattern = "GET /api/2.0/workspace/list"
+KillCaller = 1