Skip to content

Commit 26d9692

Browse files
cicoylecicoyle
authored
Merge pull request #1580 from sicoyle/fix-python-hangs
fix(python apps): fork child + exec appcmd in child to avoid issues
2 parents 115e63c + 88883d9 commit 26d9692

18 files changed

+570
-124
lines changed

‎cmd/run.go‎

Lines changed: 48 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ package cmd
1515

1616
import (
1717
"bufio"
18+
"errors"
1819
"fmt"
1920
"io"
2021
"os"
22+
"os/exec"
2123
"path/filepath"
2224
"runtime"
2325
"slices"
@@ -160,9 +162,9 @@ dapr run --run-file /path/to/directory -k
160162
fmt.Println(print.WhiteBold("WARNING: no application command found."))
161163
}
162164

163-
daprDirPath, err := standalone.GetDaprRuntimePath(cmdruntime.GetDaprRuntimePath())
164-
if err != nil {
165-
print.FailureStatusEvent(os.Stderr, "Failed to get Dapr install directory: %v", err)
165+
daprDirPath, pathErr := standalone.GetDaprRuntimePath(cmdruntime.GetDaprRuntimePath())
166+
if pathErr != nil {
167+
print.FailureStatusEvent(os.Stderr, "Failed to get Dapr install directory: %v", pathErr)
166168
os.Exit(1)
167169
}
168170

@@ -227,7 +229,7 @@ dapr run --run-file /path/to/directory -k
227229
sharedRunConfig.SchedulerHostAddress = &addr
228230
}
229231
}
230-
output, err := runExec.NewOutput(&standalone.RunConfig{
232+
appConfig := &standalone.RunConfig{
231233
AppID: appID,
232234
AppChannelAddress: appChannelAddress,
233235
AppPort: appPort,
@@ -239,7 +241,8 @@ dapr run --run-file /path/to/directory -k
239241
UnixDomainSocket: unixDomainSocket,
240242
InternalGRPCPort: internalGRPCPort,
241243
SharedRunConfig: *sharedRunConfig,
242-
})
244+
}
245+
output, err := runExec.NewOutput(appConfig)
243246
if err != nil {
244247
print.FailureStatusEvent(os.Stderr, err.Error())
245248
os.Exit(1)
@@ -280,6 +283,8 @@ dapr run --run-file /path/to/directory -k
280283

281284
output.DaprCMD.Stdout = os.Stdout
282285
output.DaprCMD.Stderr = os.Stderr
286+
// Set process group so sidecar survives when we exec the app process.
287+
setDaprProcessGroupForRun(output.DaprCMD)
283288

284289
err = output.DaprCMD.Start()
285290
if err != nil {
@@ -355,53 +360,26 @@ dapr run --run-file /path/to/directory -k
355360
return
356361
}
357362

358-
stdErrPipe, pipeErr := output.AppCMD.StderrPipe()
359-
if pipeErr != nil {
360-
print.FailureStatusEvent(os.Stderr, "Error creating stderr for App: "+err.Error())
363+
command := args[0]
364+
var binary string
365+
binary, err = exec.LookPath(command)
366+
if err != nil {
367+
print.FailureStatusEvent(os.Stderr, fmt.Sprintf("Failed to find command %s: %v", command, err))
361368
appRunning <- false
362369
return
363370
}
364-
365-
stdOutPipe, pipeErr := output.AppCMD.StdoutPipe()
366-
if pipeErr != nil {
367-
print.FailureStatusEvent(os.Stderr, "Error creating stdout for App: "+err.Error())
368-
appRunning <- false
369-
return
371+
env := output.AppCMD.Env
372+
if len(env) == 0 {
373+
env = os.Environ()
370374
}
375+
env = append(env, fmt.Sprintf("DAPR_HTTP_PORT=%d", output.DaprHTTPPort))
376+
env = append(env, fmt.Sprintf("DAPR_GRPC_PORT=%d", output.DaprGRPCPort))
371377

372-
errScanner := bufio.NewScanner(stdErrPipe)
373-
outScanner := bufio.NewScanner(stdOutPipe)
374-
go func() {
375-
for errScanner.Scan() {
376-
fmt.Println(print.Blue("== APP == " + errScanner.Text()))
377-
}
378-
}()
379-
380-
go func() {
381-
for outScanner.Scan() {
382-
fmt.Println(print.Blue("== APP == " + outScanner.Text()))
383-
}
384-
}()
385-
386-
err = output.AppCMD.Start()
387-
if err != nil {
388-
print.FailureStatusEvent(os.Stderr, err.Error())
378+
if startErr := startAppProcessInBackground(output, binary, args, env, sigCh); startErr != nil {
379+
print.FailureStatusEvent(os.Stderr, startErr.Error())
389380
appRunning <- false
390381
return
391382
}
392-
393-
go func() {
394-
appErr := output.AppCMD.Wait()
395-
396-
if appErr != nil {
397-
output.AppErr = appErr
398-
print.FailureStatusEvent(os.Stderr, "The App process exited with error code: %s", appErr.Error())
399-
} else {
400-
print.SuccessStatusEvent(os.Stdout, "Exited App successfully")
401-
}
402-
sigCh <- os.Interrupt
403-
}()
404-
405383
appRunning <- true
406384
}()
407385

@@ -465,11 +443,16 @@ dapr run --run-file /path/to/directory -k
465443
if output.AppErr != nil {
466444
exitWithError = true
467445
print.FailureStatusEvent(os.Stderr, fmt.Sprintf("Error exiting App: %s", output.AppErr))
468-
} else if output.AppCMD != nil && (output.AppCMD.ProcessState == nil || !output.AppCMD.ProcessState.Exited()) {
446+
} else if output.AppCMD != nil && output.AppCMD.Process != nil && (output.AppCMD.ProcessState == nil || !output.AppCMD.ProcessState.Exited()) {
469447
err = output.AppCMD.Process.Kill()
470448
if err != nil {
471-
exitWithError = true
472-
print.FailureStatusEvent(os.Stderr, fmt.Sprintf("Error exiting App: %s", err))
449+
// If the process already exited on its own, treat this as a clean shutdown.
450+
if errors.Is(err, os.ErrProcessDone) {
451+
print.SuccessStatusEvent(os.Stdout, "Exited App successfully")
452+
} else {
453+
exitWithError = true
454+
print.FailureStatusEvent(os.Stderr, fmt.Sprintf("Error exiting App: %s", err))
455+
}
473456
} else {
474457
print.SuccessStatusEvent(os.Stdout, "Exited App successfully")
475458
}
@@ -788,6 +771,13 @@ func startDaprdAndAppProcesses(runConfig *standalone.RunConfig, commandDir strin
788771
return runState, nil
789772
}
790773

774+
if strings.TrimSpace(runConfig.Command[0]) == "" {
775+
noCmdErr := errors.New("exec: no command")
776+
print.StatusEvent(appErrorWriter, print.LogFailure, "Error starting app process: %s", noCmdErr.Error())
777+
_ = killDaprdProcess(runState)
778+
return nil, noCmdErr
779+
}
780+
791781
// Start App process.
792782
go startAppProcess(runConfig, runState, appRunning, sigCh, startErrChan)
793783

@@ -836,7 +826,7 @@ func stopDaprdAndAppProcesses(runState *runExec.RunExec) bool {
836826
if appErr != nil {
837827
exitWithError = true
838828
print.StatusEvent(runState.AppCMD.ErrorWriter, print.LogFailure, "Error exiting App: %s", appErr)
839-
} else if runState.AppCMD.Command != nil && (runState.AppCMD.Command.ProcessState == nil || !runState.AppCMD.Command.ProcessState.Exited()) {
829+
} else if runState.AppCMD.Command != nil && runState.AppCMD.Command.Process != nil && (runState.AppCMD.Command.ProcessState == nil || !runState.AppCMD.Command.ProcessState.Exited()) {
840830
err = killAppProcess(runState)
841831
if err != nil {
842832
exitWithError = true
@@ -1009,11 +999,20 @@ func killDaprdProcess(runE *runExec.RunExec) error {
1009999

10101000
// killAppProcess is used to kill the App process and return error on failure.
10111001
func killAppProcess(runE *runExec.RunExec) error {
1012-
if runE.AppCMD.Command == nil {
1002+
if runE.AppCMD.Command == nil || runE.AppCMD.Command.Process == nil {
1003+
return nil
1004+
}
1005+
// Check if the process has already exited on its own.
1006+
if runE.AppCMD.Command.ProcessState != nil && runE.AppCMD.Command.ProcessState.Exited() {
1007+
// Process already exited, no need to kill it.
10131008
return nil
10141009
}
10151010
err := runE.AppCMD.Command.Process.Kill()
10161011
if err != nil {
1012+
// If the process already exited on its own
1013+
if errors.Is(err, os.ErrProcessDone) {
1014+
return nil
1015+
}
10171016
print.StatusEvent(runE.DaprCMD.ErrorWriter, print.LogFailure, "Error exiting App: %s", err)
10181017
return err
10191018
}

‎cmd/run_unix.go‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
//go:build !windows
2+
3+
/*
4+
Copyright 2026 The Dapr Authors
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
*/
15+
16+
package cmd
17+
18+
import (
19+
"fmt"
20+
"os"
21+
"os/exec"
22+
"syscall"
23+
24+
"github.com/dapr/cli/pkg/print"
25+
runExec "github.com/dapr/cli/pkg/runexec"
26+
)
27+
28+
// setDaprProcessGroupForRun sets the process group on the daprd command so the
29+
// sidecar can be managed independently (e.g. when the app is started via exec).
30+
func setDaprProcessGroupForRun(cmd *exec.Cmd) {
31+
if cmd == nil {
32+
return
33+
}
34+
cmd.SysProcAttr = &syscall.SysProcAttr{
35+
Setpgid: true,
36+
}
37+
}
38+
39+
// startAppProcessInBackground starts the app process using ForkExec.
40+
// This prevents the child from seeing a fork, avoiding Python async/threading issues,
41+
// and sets output.AppCMD.Process.
42+
// It then runs a goroutine that waits and signals sigCh.
43+
func startAppProcessInBackground(output *runExec.RunOutput, binary string, args []string, env []string, sigCh chan os.Signal) error {
44+
if output.AppCMD == nil || output.AppCMD.Process != nil {
45+
return fmt.Errorf("app command is nil")
46+
}
47+
48+
procAttr := &syscall.ProcAttr{
49+
Env: env,
50+
// stdin, stdout, and stderr inherit directly from the parent
51+
// This prevents Python from detecting pipes because if the app is Python then it will detect the pipes and think
52+
// it's a fork and will cause random hangs due to async python in durabletask-python.
53+
Files: []uintptr{0, 1, 2},
54+
Sys: &syscall.SysProcAttr{
55+
Setpgid: true,
56+
},
57+
}
58+
59+
// Use ForkExec to fork a child, then exec python in the child.
60+
// NOTE: This is needed bc forking a python app with async python running (i.e., everything in durabletask-python) will cause random hangs, no matter the python version.
61+
// Doing this this way makes python not see the fork, starts via exec, so it doesn't cause random hangs due to when forking async python apps where locks and such get corrupted in forking.
62+
argv := append([]string{binary}, args[1:]...)
63+
pid, err := syscall.ForkExec(binary, argv, procAttr)
64+
if err != nil {
65+
return fmt.Errorf("failed to fork/exec app: %w", err)
66+
}
67+
output.AppCMD.Process = &os.Process{Pid: pid}
68+
69+
go func() {
70+
var waitStatus syscall.WaitStatus
71+
_, err := syscall.Wait4(pid, &waitStatus, 0, nil)
72+
if err != nil {
73+
output.AppErr = err
74+
print.FailureStatusEvent(os.Stderr, "The App process exited with error: %s", err.Error())
75+
} else if waitStatus.Signaled() {
76+
output.AppErr = fmt.Errorf("app terminated by signal: %s", waitStatus.Signal())
77+
print.FailureStatusEvent(os.Stderr, "The App process was terminated by signal: %s", waitStatus.Signal())
78+
} else if waitStatus.Exited() && waitStatus.ExitStatus() != 0 {
79+
output.AppErr = fmt.Errorf("app exited with status %d", waitStatus.ExitStatus())
80+
print.FailureStatusEvent(os.Stderr, "The App process exited with error code: %d", waitStatus.ExitStatus())
81+
} else {
82+
print.SuccessStatusEvent(os.Stdout, "Exited App successfully")
83+
}
84+
sigCh <- os.Interrupt
85+
}()
86+
return nil
87+
}

‎cmd/run_windows.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
//go:build windows
2+
3+
/*
4+
Copyright 2026 The Dapr Authors
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
*/
15+
16+
package cmd
17+
18+
import (
19+
"fmt"
20+
"os"
21+
"os/exec"
22+
23+
"github.com/dapr/cli/pkg/print"
24+
runExec "github.com/dapr/cli/pkg/runexec"
25+
)
26+
27+
// setDaprProcessGroupForRun is a no-op on Windows (SysProcAttr.Setpgid does not exist).
28+
func setDaprProcessGroupForRun(cmd *exec.Cmd) {
29+
// no-op on Windows
30+
_ = cmd
31+
}
32+
33+
// startAppProcessInBackground starts the app process using exec.Command,
34+
// sets output.AppCMD to the new command, and runs a goroutine that waits and signals sigCh.
35+
func startAppProcessInBackground(output *runExec.RunOutput, binary string, args []string, env []string, sigCh chan os.Signal) error {
36+
cmdArgs := args[1:]
37+
if output.AppCMD == nil {
38+
output.AppCMD = exec.Command(binary, cmdArgs...)
39+
} else {
40+
output.AppCMD.Path = binary
41+
output.AppCMD.Args = append([]string{binary}, cmdArgs...)
42+
}
43+
output.AppCMD.Env = env
44+
output.AppCMD.Stdin = os.Stdin
45+
output.AppCMD.Stdout = os.Stdout
46+
output.AppCMD.Stderr = os.Stderr
47+
48+
if err := output.AppCMD.Start(); err != nil {
49+
return fmt.Errorf("failed to start app: %w", err)
50+
}
51+
52+
go func() {
53+
waitErr := output.AppCMD.Wait()
54+
if waitErr != nil {
55+
output.AppErr = waitErr
56+
print.FailureStatusEvent(os.Stderr, "The App process exited with error: %s", waitErr.Error())
57+
} else if output.AppCMD.ProcessState != nil && !output.AppCMD.ProcessState.Success() {
58+
output.AppErr = fmt.Errorf("app exited with status %d", output.AppCMD.ProcessState.ExitCode())
59+
print.FailureStatusEvent(os.Stderr, "The App process exited with error code: %d", output.AppCMD.ProcessState.ExitCode())
60+
} else {
61+
print.SuccessStatusEvent(os.Stdout, "Exited App successfully")
62+
}
63+
sigCh <- os.Interrupt
64+
}()
65+
return nil
66+
}

‎pkg/runexec/runexec_test.go‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"fmt"
1818
"os"
1919
"regexp"
20+
"runtime"
2021
"strings"
2122
"testing"
2223

@@ -27,6 +28,8 @@ import (
2728
"github.com/dapr/cli/pkg/standalone"
2829
)
2930

31+
const windowsOsType = "windows"
32+
3033
func assertArgumentEqual(t *testing.T, key string, expectedValue string, args []string) {
3134
var value string
3235
for index, arg := range args {
@@ -205,8 +208,23 @@ func TestRun(t *testing.T) {
205208
assert.NoError(t, err)
206209

207210
assertCommonArgs(t, basicConfig, output)
208-
assert.Equal(t, "MyCommand", output.AppCMD.Args[0])
209-
assert.Equal(t, "--my-arg", output.AppCMD.Args[1])
211+
require.NotNil(t, output.AppCMD)
212+
if runtime.GOOS == windowsOsType {
213+
// On Windows the app is run directly (no shell).
214+
require.GreaterOrEqual(t, len(output.AppCMD.Args), 2)
215+
assert.Equal(t, "MyCommand", output.AppCMD.Args[0])
216+
assert.Equal(t, "--my-arg", output.AppCMD.Args[1])
217+
} else {
218+
// On Unix the app command is executed via a shell wrapper
219+
require.GreaterOrEqual(t, len(output.AppCMD.Args), 5)
220+
assert.Equal(t, "sh", output.AppCMD.Args[0])
221+
assert.Equal(t, "-c", output.AppCMD.Args[1])
222+
assert.Equal(t, "exec \"$@\"", output.AppCMD.Args[2])
223+
assert.Equal(t, "sh", output.AppCMD.Args[3])
224+
assert.Equal(t, "MyCommand", output.AppCMD.Args[4])
225+
assert.Equal(t, "--my-arg", output.AppCMD.Args[5])
226+
}
227+
210228
assertArgumentEqual(t, "app-channel-address", "localhost", output.DaprCMD.Args)
211229
assertAppEnv(t, basicConfig, output)
212230
})

0 commit comments

Comments
 (0)