Skip to content

Use a detached timeout for pause completion in API #2238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
matthewlouisbrockman wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Use a detached timeout for pause completion in API #2238

Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5937c2e
cleanup use of ctx without cancel in orch pause
matthewlouisbrockman Mar 26, 2026
b7dc232
use context.WithTimeout(context.WithoutCancel(ctx) around the snapsho…
matthewlouisbrockman Mar 27, 2026
999e201
undo simplification to sandboxes.go to keep the PR minimal
matthewlouisbrockman Mar 27, 2026
dfac5c7
Return 202 for long-running sandbox pauses while continuing pause in …
matthewlouisbrockman Mar 27, 2026
5f17d26
handle closing cleaner
matthewlouisbrockman Mar 27, 2026
5c8f238
add the 202 to the contract
matthewlouisbrockman Mar 27, 2026
023fd3c
add a todo on the shutdown safety
matthewlouisbrockman Mar 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions packages/api/internal/orchestrator/pause_instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (

type PauseQueueExhaustedError struct{}

const pauseCompletionTimeout = 15 * time.Minute

func (PauseQueueExhaustedError) Error() string {
return "The pause queue is exhausted"
}
Expand All @@ -38,6 +40,9 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node,
return err
}

ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), pauseCompletionTimeout)
defer cancel()

err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String())
if errors.Is(err, PauseQueueExhaustedError{}) {
telemetry.ReportCriticalError(ctx, "pause queue exhausted", err)
Expand All @@ -64,18 +69,18 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node,
return fmt.Errorf("error pausing sandbox: %w", err)
}

o.snapshotCache.Invalidate(context.WithoutCancel(ctx), sbx.SandboxID)
o.snapshotCache.Invalidate(ctx, sbx.SandboxID)

return nil
}

func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string) error {
childCtx, childSpan := tracer.Start(ctx, "snapshot-instance")
ctx, childSpan := tracer.Start(ctx, "snapshot-instance")
defer childSpan.End()

client, childCtx := node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID)
client, ctx := node.GetSandboxDeleteCtx(ctx, sbx.SandboxID, sbx.ExecutionID)
_, err := client.Sandbox.Pause(
childCtx, &orchestrator.SandboxPauseRequest{
ctx, &orchestrator.SandboxPauseRequest{
SandboxId: sbx.SandboxID,
TemplateId: templateID,
BuildId: buildID,
Expand Down
Loading