Skip to content

Commit e210a27

Browse files
andrewnestershreyas-goenkapietern
authored
Added databricks apps run-local command to run Databricks apps locally (#2555)
## Changes Added `databricks apps run-local` command to run Databricks apps locally. This command allows to start the Databricks app locally. It starts app proxy which is used to proxy request to the app itself and injects necessary Databricks app related headers. The command support the following options: * `--prepare-environment`: will setup Python virtual environment and install necessary libraries. It uses `uv` * `--debug`: starts a Python debugger (based on debugpy) which allows you to attach your IDE debugger to running app. * `--port`: allows to configure which port to start proxy on * `--entry-point`: allows to configure which YAML file to use for Databricks app configuration. ## Tests Added acceptance test <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. --> --------- Co-authored-by: shreyas-goenka <[email protected]> Co-authored-by: Pieter Noordhuis <[email protected]>
1 parent 98396b4 commit e210a27

File tree

17 files changed

+938
-0
lines changed

17 files changed

+938
-0
lines changed

NEXT_CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
* Upgrade Go SDK to 0.65.0 ([#2786](https://github.com/databricks/cli/pull/2786))
1111

1212
### CLI
13+
* Added `databricks apps run-local` command to run Databricks apps locally ([#2555](https://github.com/databricks/cli/pull/2555))
1314

1415
### Bundles
1516
* Raise an error when Unity Catalog volumes are used for paths other than artifacts ([#2754](https://github.com/databricks/cli/pull/2754))
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import os
2+
import signal
3+
from flask import Flask, request
4+
5+
app = Flask(__name__)
6+
7+
app.logger.warning("Python Flask app has started with: " + os.environ.get("TEST"))
8+
9+
10+
@app.route("/")
11+
def index():
12+
return dict(request.headers)
13+
14+
15+
@app.route("/shutdown")
16+
def shutdown():
17+
os._exit(signal.SIGTERM)
18+
return "Shutting down..."
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
command:
2+
- flask
3+
- run
4+
5+
env:
6+
- name: TEST
7+
value: "test"
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
command:
2+
- python
3+
- -c
4+
- "print('Hello, world')"
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
command:
2+
- python
3+
- -c
4+
- "print('Hello, world')"
5+
6+
env:
7+
- name: VALUE_FROM
8+
valueFrom: "value-from-secret"
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
2+
>>> errcode [CLI] apps run-local --entry-point value-from.yml
3+
Error: VALUE_FROM defined in value-from.yml with valueFrom property and can't be resolved locally. Please set VALUE_FROM environment variable in your terminal or using --env flag
4+
5+
Exit code: 1
6+
Running command: uv run python -c print('Hello, world')
7+
Hello, world
8+
9+
=== Starting the app in background...
10+
=== Waiting
11+
=== Checking app is running...
12+
>>> curl -s -o - http://127.0.0.1:8001
13+
{
14+
"Accept": "*/*",
15+
"Accept-Encoding": "gzip",
16+
"Host": "127.0.0.1:8000",
17+
"User-Agent": "curl/(version)",
18+
"X-Forwarded-Email": "[USERNAME]",
19+
"X-Forwarded-Host": "localhost",
20+
"X-Forwarded-Preferred-Username": "",
21+
"X-Forwarded-User": "[USERNAME]",
22+
"X-Real-Ip": "127.0.0.1",
23+
"X-Request-Id": "[UUID]"
24+
}
25+
26+
=== Sending shutdown request...
27+
>>> curl -s -o /dev/null http://127.0.0.1:8001/shutdown
28+
29+
=== Checking CLI command output...
30+
>>> grep To debug your app, attach a debugger to port 5678 ./out.run.txt
31+
To debug your app, attach a debugger to port 5678
32+
33+
>>> grep -o Python Flask app has started with: test ./out.run.txt
34+
Python Flask app has started with: test
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
cd app
2+
3+
trace errcode $CLI apps run-local --entry-point value-from.yml 2>&1
4+
5+
# We first run the command with different entry point which starts unblocking script
6+
# so we don't need to start it in background. It will install the dependencies as part of the command
7+
trace $CLI apps run-local --prepare-environment --entry-point test.yml 2>&1 | grep -w "Hello, world"
8+
9+
title "Starting the app in background..."
10+
$CLI apps run-local --prepare-environment --debug > ../out.run.txt 2>&1 &
11+
PID=$!
12+
# Ensure background process is killed on script exit
13+
trap 'kill $PID 2>/dev/null || true' EXIT
14+
cd ..
15+
16+
title Waiting for the app to start...
17+
# Use a loop to check for the startup message instead of tail/sed which can be unreliable on Windows
18+
# due to file locking, buffering issues, and different text processing behavior across Windows versions.
19+
# A simple grep loop is more robust across platforms.
20+
while [ -z "$(grep -o "Python Flask app has started with" out.run.txt 2>/dev/null)" ]; do
21+
sleep 1
22+
done
23+
24+
title "Checking app is running..."
25+
trace curl -s -o - http://127.0.0.1:8001 | jq
26+
27+
title "Sending shutdown request..."
28+
trace curl -s -o /dev/null http://127.0.0.1:8001/shutdown || true
29+
30+
title "Checking CLI command output..."
31+
32+
trace grep "To debug your app, attach a debugger to port 5678" ./out.run.txt
33+
trace grep -o "Python Flask app has started with: test" ./out.run.txt
34+
rm out.run.txt
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
RecordRequests = false
2+
3+
Ignore = [
4+
'.venv',
5+
'__pycache__'
6+
]
7+
8+
[[Repls]]
9+
Old='curl/[0-9]+\.[0-9]+\.[0-9]+'
10+
New='curl/(version)'

cmd/workspace/apps/run_local.go

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
package apps
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"os"
8+
"os/exec"
9+
"os/signal"
10+
"strings"
11+
"syscall"
12+
"time"
13+
14+
"github.com/databricks/cli/cmd/root"
15+
"github.com/databricks/cli/libs/appproxy"
16+
"github.com/databricks/cli/libs/apps"
17+
"github.com/databricks/cli/libs/auth"
18+
"github.com/databricks/cli/libs/cmdctx"
19+
"github.com/databricks/cli/libs/cmdio"
20+
"github.com/databricks/databricks-sdk-go"
21+
"github.com/spf13/cobra"
22+
)
23+
24+
// Databricks Apps send a SIGKILL signal 15 seconds after a SIGTERM
25+
// https://docs.databricks.com/aws/en/dev-tools/databricks-apps/app-development#important-guidelines-for-implementing-databricks-apps
26+
const SHUTDOWN_TIMEOUT = 15 * time.Second
27+
28+
func setupWorkspaceAndConfig(cmd *cobra.Command, entryPoint string) (*apps.Config, *apps.AppSpec, error) {
29+
ctx := cmd.Context()
30+
w := cmdctx.WorkspaceClient(ctx)
31+
workspaceId, err := w.CurrentWorkspaceID(ctx)
32+
if err != nil {
33+
return nil, nil, err
34+
}
35+
36+
cwd, err := os.Getwd()
37+
if err != nil {
38+
return nil, nil, err
39+
}
40+
41+
config := apps.NewConfig(w.Config.Host, workspaceId, cwd)
42+
if entryPoint != "" {
43+
config.AppSpecFiles = []string{entryPoint}
44+
}
45+
spec, err := apps.ReadAppSpecFile(config)
46+
if err != nil {
47+
return nil, nil, err
48+
}
49+
50+
return config, spec, nil
51+
}
52+
53+
func setupApp(cmd *cobra.Command, config *apps.Config, spec *apps.AppSpec, customEnv []string, prepareEnvironment bool) (apps.App, []string, error) {
54+
ctx := cmd.Context()
55+
cfg := cmdctx.ConfigUsed(ctx)
56+
app := apps.NewApp(config, spec)
57+
env := auth.ProcessEnv(cfg)
58+
if cfg.Profile != "" {
59+
env = append(env, "DATABRICKS_CONFIG_PROFILE="+cfg.Profile)
60+
}
61+
62+
appEnv, err := spec.LoadEnvVars(ctx, customEnv)
63+
if err != nil {
64+
return app, nil, err
65+
}
66+
env = append(env, appEnv...)
67+
68+
if prepareEnvironment {
69+
err := app.PrepareEnvironment()
70+
if err != nil {
71+
return app, nil, err
72+
}
73+
}
74+
75+
return app, env, nil
76+
}
77+
78+
func startAppProcess(cmd *cobra.Command, config *apps.Config, app apps.App, env []string, debug bool) (*exec.Cmd, error) {
79+
ctx := cmd.Context()
80+
specCommand, err := app.GetCommand(debug)
81+
if err != nil {
82+
return nil, err
83+
}
84+
85+
cmdio.LogString(ctx, "Running command: "+strings.Join(specCommand, " "))
86+
appCmd := exec.Command(specCommand[0], specCommand[1:]...)
87+
appCmd.Stdin = cmd.InOrStdin()
88+
appCmd.Stdout = cmd.OutOrStdout()
89+
appCmd.Stderr = cmd.ErrOrStderr()
90+
91+
appEnvs := apps.GetBaseEnvVars(config)
92+
for _, envVar := range appEnvs {
93+
env = append(env, envVar.String())
94+
}
95+
96+
appCmd.Env = env
97+
appCmd.Dir = config.AppPath
98+
99+
err = appCmd.Start()
100+
if err != nil {
101+
return nil, err
102+
}
103+
104+
return appCmd, nil
105+
}
106+
107+
func setupProxy(ctx context.Context, cmd *cobra.Command, config *apps.Config, w *databricks.WorkspaceClient, port int, debug bool) error {
108+
proxy, err := appproxy.New(ctx, config.AppURL)
109+
if err != nil {
110+
return err
111+
}
112+
113+
me, err := w.CurrentUser.Me(ctx)
114+
if err != nil {
115+
return err
116+
}
117+
118+
for key, value := range apps.GetXHeaders(me) {
119+
proxy.InjectHeader(key, value)
120+
}
121+
122+
proxyAddr := fmt.Sprintf("localhost:%d", port)
123+
go func() {
124+
cmdio.LogString(ctx, "To access your app go to http://"+proxyAddr)
125+
err := proxy.ListenAndServe(proxyAddr)
126+
if err != nil {
127+
cmd.PrintErrln(err)
128+
}
129+
}()
130+
131+
if debug {
132+
cmdio.LogString(ctx, "To debug your app, attach a debugger to port "+apps.DEBUG_PORT)
133+
}
134+
135+
return nil
136+
}
137+
138+
// SIGTERM (not supported on Windows) and SIGINT (Ctrl+C, supported cross-platform)
139+
// are caught to enable graceful shutdown of the app process.
140+
func handleGracefulShutdown(appCmd *exec.Cmd) error {
141+
done := make(chan error, 1)
142+
sigChan := make(chan os.Signal, 1)
143+
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
144+
145+
go func() {
146+
done <- appCmd.Wait()
147+
}()
148+
149+
select {
150+
case err := <-done:
151+
return err
152+
case <-sigChan:
153+
if err := appCmd.Process.Signal(os.Interrupt); err != nil {
154+
return fmt.Errorf("failed to send interrupt signal: %w", err)
155+
}
156+
157+
select {
158+
case err := <-done:
159+
return err
160+
case <-time.After(SHUTDOWN_TIMEOUT):
161+
if err := appCmd.Process.Kill(); err != nil {
162+
return fmt.Errorf("failed to kill process: %w", err)
163+
}
164+
return errors.New("process killed after timeout")
165+
}
166+
}
167+
}
168+
169+
func newRunLocal() *cobra.Command {
170+
var (
171+
port int
172+
debug bool
173+
prepareEnvironment bool
174+
entryPoint string
175+
customEnv []string
176+
)
177+
178+
cmd := &cobra.Command{}
179+
180+
cmd.Use = "run-local"
181+
cmd.Short = `Run an app locally`
182+
cmd.Long = `Run an app locally.
183+
184+
This command starts an app locally.`
185+
186+
cmd.Flags().IntVar(&port, "port", 8001, "Port on which to run the app")
187+
cmd.Flags().BoolVar(&debug, "debug", false, "Enable debug mode")
188+
cmd.Flags().BoolVar(&prepareEnvironment, "prepare-environment", false, "Prepares the environment for running the app. Requires 'uv' to be installed.")
189+
cmd.Flags().StringSliceVar(&customEnv, "env", nil, "Set environment variables")
190+
cmd.Flags().StringVar(&entryPoint, "entry-point", "", "Specify the custom entry point with configuration (.yml file) for the app. Defaults to app.yml")
191+
192+
cmd.PreRunE = root.MustWorkspaceClient
193+
194+
cmd.RunE = func(cmd *cobra.Command, args []string) error {
195+
ctx := cmd.Context()
196+
w := cmdctx.WorkspaceClient(ctx)
197+
198+
config, spec, err := setupWorkspaceAndConfig(cmd, entryPoint)
199+
if err != nil {
200+
return err
201+
}
202+
203+
app, env, err := setupApp(cmd, config, spec, customEnv, prepareEnvironment)
204+
if err != nil {
205+
return err
206+
}
207+
208+
appCmd, err := startAppProcess(cmd, config, app, env, debug)
209+
if err != nil {
210+
return err
211+
}
212+
213+
err = setupProxy(ctx, cmd, config, w, port, debug)
214+
if err != nil {
215+
return err
216+
}
217+
218+
return handleGracefulShutdown(appCmd)
219+
}
220+
221+
cmd.ValidArgsFunction = cobra.NoFileCompletions
222+
223+
return cmd
224+
}
225+
226+
func init() {
227+
cmdOverrides = append(cmdOverrides, func(cmd *cobra.Command) {
228+
cmd.AddCommand(newRunLocal())
229+
})
230+
}

libs/apps/apps.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package apps
2+
3+
type App interface {
4+
PrepareEnvironment() error
5+
GetCommand(bool) ([]string, error)
6+
}
7+
8+
func NewApp(config *Config, spec *AppSpec) App {
9+
// We only support python apps for now, but later we can add more types
10+
// based on AppSpec
11+
return NewPythonApp(config, spec)
12+
}

0 commit comments

Comments
 (0)