Skip to content

Commit 2db9670

Browse files
shreyas-goenkashreyas-goenka
committed
Gracefully exit when terminated mid deploy / destroy
1 parent 18cab43 commit 2db9670

File tree

12 files changed

+247
-7
lines changed

12 files changed

+247
-7
lines changed

‎acceptance/bundle/deploy/signal-cleanup/databricks.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
bundle:
2+
name: signal-test
3+
4+
resources:
5+
jobs:
6+
job1:
7+
name: job1
8+
9+
job2:
10+
name: job2 (deploy after ${resources.jobs.job1.id})

‎acceptance/bundle/deploy/signal-cleanup/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/deploy/signal-cleanup/output.txt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
2+
=== Wait until the deployment has started.Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/signal-test/default/files...
3+
Deploying resources...
4+
Deployment in progress, sending interrupt signal...
5+
6+
>>> kill -INT [PID]
7+
8+
>>> wait [PID]
9+
Operation interrupted. Gracefully shutting down...
10+
11+
Exit code: 130
12+
13+
>>> cat out.requests.txt
14+
{
15+
"method": "POST",
16+
"path": "/api/2.0/workspace/delete",
17+
"body": {
18+
"path": "/Workspace/Users/[USERNAME]/.bundle/signal-test/default/state/deploy.lock"
19+
}
20+
}
21+
22+
=== A creation request for job1 should be recorded in the requests file. No request for job2 should exist since the process was terminated mid deployment.
23+
>>> cat out.requests.txt
24+
{
25+
"method": "POST",
26+
"path": "/api/2.2/jobs/create",
27+
"body": {
28+
"deployment": {
29+
"kind": "BUNDLE",
30+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/signal-test/default/state/metadata.json"
31+
},
32+
"edit_mode": "UI_LOCKED",
33+
"format": "MULTI_TASK",
34+
"max_concurrent_runs": 1,
35+
"name": "job1",
36+
"queue": {
37+
"enabled": true
38+
}
39+
}
40+
}

‎acceptance/bundle/deploy/signal-cleanup/script‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
# Start deployment in background, redirecting stderr to capture when deployment starts
4+
$CLI bundle deploy 2>&1 &
5+
DEPLOY_PID=$!
6+
7+
# Wait for deployment to start by monitoring the requests file
8+
# Once we see the job creation request starting, we know deployment is in progress
9+
title "Wait until the deployment has started."
10+
for i in {1..30}; do
11+
if [ -f out.requests.txt ] && jq -e 'select(.method == "POST" and (.path | contains("/api/2.2/jobs/create")))' out.requests.txt >/dev/null 2>&1; then
12+
echo "Deployment in progress, sending interrupt signal..."
13+
break
14+
fi
15+
sleep 0.1
16+
done
17+
18+
# Send interrupt signal
19+
trace kill -INT $DEPLOY_PID
20+
21+
# Wait for process to complete
22+
errcode trace wait $DEPLOY_PID
23+
24+
# A deletion request for deploy.lock should have been recorded in the requests file
25+
trace cat out.requests.txt | jq 'select(.method == "POST" and (.path | contains("workspace/delete")) and (.body.path | contains("deploy.lock")))'
26+
27+
title "A creation request for job1 should be recorded in the requests file. No request for job2 should exist since the process was terminated mid deployment."
28+
trace cat out.requests.txt | jq 'select(.method == "POST" and (.path | contains("/api/2.2/jobs/create")))'
29+
30+
rm out.requests.txt

‎acceptance/bundle/deploy/signal-cleanup/test.toml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
Local = true
2+
Cloud = false
3+
RecordRequests = true
4+
5+
# Add delay to first job creation to ensure we can interrupt during deployment
6+
[[Server]]
7+
Pattern = "POST /api/2.2/jobs/create"
8+
Response.StatusCode = 200
9+
Response.Body = '{"job_id": 1111}'
10+
11+
# Large time to ensure deployment gets stuck when trying to create the first job.
12+
Delay = "300s"
13+
14+
# Replace PID numbers in kill/wait commands
15+
[[Repls]]
16+
Old = "(kill -INT |wait )\\d+"
17+
New = "$1[PID]"

‎acceptance/bundle/destroy/signal-cleanup/databricks.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
bundle:
2+
name: signal-test
3+
4+
resources:
5+
jobs:
6+
job1:
7+
name: job1
8+
9+
job2:
10+
name: job2 (depends on ${resources.jobs.job1.id})

‎acceptance/bundle/destroy/signal-cleanup/out.test.toml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎acceptance/bundle/destroy/signal-cleanup/output.txt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/signal-test/default/files...
2+
Deploying resources...
3+
Updating deployment state...
4+
Deployment complete!
5+
6+
=== Wait until the destroy has started.The following resources will be deleted:
7+
delete job job1
8+
delete job job2
9+
10+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/signal-test/default
11+
12+
Destroy in progress, sending interrupt signal...
13+
14+
>>> kill -INT [PID]
15+
16+
>>> wait [PID]
17+
Operation interrupted. Gracefully shutting down...
18+
19+
Exit code: 130
20+
21+
>>> cat out.requests.txt
22+
23+
=== A deletion request for job2 should be recorded in the requests file. No request for job1 should exist since the process was terminated mid destroy.
24+
>>> cat out.requests.txt
25+
{
26+
"method": "POST",
27+
"path": "/api/2.2/jobs/delete",
28+
"body": {
29+
"job_id": [NUMID]
30+
}
31+
}
32+
33+
=== [CLI]

‎acceptance/bundle/destroy/signal-cleanup/script‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
# First deploy the bundle so we have something to destroy
4+
$CLI bundle deploy --auto-approve
5+
6+
# Start destroy in background, redirecting stderr to capture when destroy starts
7+
$CLI bundle destroy --auto-approve 2>&1 &
8+
DESTROY_PID=$!
9+
10+
# Wait for destroy to start by monitoring for job deletion request
11+
title "Wait until the destroy has started."
12+
for i in {1..30}; do
13+
if [ -f out.requests.txt ] && jq -e 'select(.method == "POST" and (.path | contains("/api/2.2/jobs/delete")))' out.requests.txt >/dev/null 2>&1; then
14+
echo "Destroy in progress, sending interrupt signal..."
15+
break
16+
fi
17+
sleep 0.1
18+
done
19+
20+
# Send interrupt signal
21+
trace kill -INT $DESTROY_PID
22+
23+
# Wait for process to complete
24+
errcode trace wait $DESTROY_PID
25+
26+
# A deletion request for destroy.lock should have been recorded in the requests file
27+
trace cat out.requests.txt | jq 'select(.method == "POST" and (.path | contains("workspace/delete")) and (.body.path | contains("destroy.lock")))'
28+
29+
title "A deletion request for job2 should be recorded in the requests file. No request for job1 should exist since the process was terminated mid destroy."
30+
trace cat out.requests.txt | jq 'select(.method == "POST" and (.path | contains("/api/2.2/jobs/delete")))'
31+
32+
rm out.requests.txt

‎acceptance/bundle/destroy/signal-cleanup/test.toml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
Local = true
2+
Cloud = false
3+
RecordRequests = true
4+
5+
# Test only terraform engine (signal handling is the same for direct)
6+
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]
7+
8+
# Add delay to first job deletion to ensure we can interrupt during destroy
9+
[[Server]]
10+
Pattern = "POST /api/2.2/jobs/delete"
11+
Response.StatusCode = 200
12+
Response.Body = '{}'
13+
14+
# Large time to ensure destroy gets stuck when deleting the first job.
15+
Delay = "300s"
16+
17+
# Replace PID numbers in kill/wait commands
18+
[[Repls]]
19+
Old = "(kill -INT |wait )\\d+"
20+
New = "$1[PID]"

0 commit comments

Comments
 (0)