From 4d186d27b58fee8fe2b95421cdfc5dda50dd9139 Mon Sep 17 00:00:00 2001 From: Jahvon Dockery Date: Sat, 5 Jul 2025 21:25:52 -0400 Subject: [PATCH 1/3] fix: deadlock when nesting serial/parallel exec fixes a cache read deadlock when there is a serial or parallel exec includes in another serial or parallel exec --- internal/runner/parallel/parallel.go | 22 +++++++++++++--------- internal/runner/serial/serial.go | 21 ++++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/internal/runner/parallel/parallel.go b/internal/runner/parallel/parallel.go index 392e9fa1..7e4e811c 100644 --- a/internal/runner/parallel/parallel.go +++ b/internal/runner/parallel/parallel.go @@ -51,11 +51,18 @@ func (r *parallelRunner) Exec( if err != nil { return err } - defer str.Close() if err := str.CreateBucket(store.EnvironmentBucket()); err != nil { return err } - return handleExec(ctx, e, eng, parallelSpec, inputEnv, str) + cacheData, err := str.GetAll() + if err != nil { + return err + } + if err := str.Close(); err != nil { + ctx.Logger.Error(err, "unable to close store") + } + + return handleExec(ctx, e, eng, parallelSpec, inputEnv, cacheData) } return fmt.Errorf("no parallel executables to run") @@ -65,8 +72,9 @@ func (r *parallelRunner) Exec( func handleExec( ctx *context.Context, parent *executable.Executable, eng engine.Engine, - parallelSpec *executable.ParallelExecutableType, promptedEnv map[string]string, - str store.Store, + parallelSpec *executable.ParallelExecutableType, + promptedEnv map[string]string, + cacheData map[string]string, ) error { groupCtx, cancel := stdCtx.WithCancel(ctx.Ctx) defer cancel() @@ -77,11 +85,7 @@ func handleExec( } group.SetLimit(limit) - dm, err := str.GetAll() - if err != nil { - return err - } - dataMap := expr.ExpressionEnv(ctx, parent, dm, promptedEnv) + dataMap := expr.ExpressionEnv(ctx, parent, cacheData, promptedEnv) var execs []engine.Exec for i, refConfig := range parallelSpec.Execs { diff --git a/internal/runner/serial/serial.go b/internal/runner/serial/serial.go index 1f9ba720..e3695f6c 100644 --- a/internal/runner/serial/serial.go +++ b/internal/runner/serial/serial.go @@ -51,29 +51,32 @@ func (r *serialRunner) Exec( if err != nil { return err } - defer str.Close() if err := str.CreateBucket(store.EnvironmentBucket()); err != nil { return err } - return handleExec(ctx, e, eng, serialSpec, inputEnv, str) + cacheData, err := str.GetAll() + if err != nil { + return err + } + if err := str.Close(); err != nil { + ctx.Logger.Error(err, "unable to close store") + } + + return handleExec(ctx, e, eng, serialSpec, inputEnv, cacheData) } return fmt.Errorf("no serial executables to run") } -//nolint:funlen,gocognit +//nolint:gocognit func handleExec( ctx *context.Context, parent *executable.Executable, eng engine.Engine, serialSpec *executable.SerialExecutableType, promptedEnv map[string]string, - str store.Store, + cacheData map[string]string, ) error { - dm, err := str.GetAll() - if err != nil { - return err - } - dataMap := expr.ExpressionEnv(ctx, parent, dm, promptedEnv) + dataMap := expr.ExpressionEnv(ctx, parent, cacheData, promptedEnv) var execs []engine.Exec for i, refConfig := range serialSpec.Execs { From 6b750d133e3ca0d92c15c5fe0cc6c9ba69646c95 Mon Sep 17 00:00:00 2001 From: Jahvon Dockery Date: Sat, 5 Jul 2025 21:27:14 -0400 Subject: [PATCH 2/3] fix: cross-workspace nameless exec dup issue when the same nameless exec is used across workspaces, it sometimes may not be the exec returned. this fixes the issue by including the ws in the returned id --- types/executable/executable.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/types/executable/executable.go b/types/executable/executable.go index 3c0f8b06..333bd4a8 100644 --- a/types/executable/executable.go +++ b/types/executable/executable.go @@ -518,15 +518,14 @@ func NewExecutableID(workspace, namespace, name string) string { switch { case ws == "": return "" // TODO: return error or log warning + case ns == "" && name == "": + return ws + "/" // nameless executable case ns != "" && name == "": return fmt.Sprintf("%s:", ns) case ns != "": return fmt.Sprintf("%s/%s:%s", ws, ns, name) - case name != "": + default: return fmt.Sprintf("%s/%s", ws, name) - default: // ws != "" && ns == "" && name == "" - // for now, exclude the workspace from the string (until we can indicate that it's root / not named in the tui) - return "" } } From 43569e9c8c53a0ed97d8d1ed597e9dde1aabde56 Mon Sep 17 00:00:00 2001 From: Jahvon Dockery Date: Sat, 5 Jul 2025 21:45:29 -0400 Subject: [PATCH 3/3] ci: update project executables --- .execs/build.flow | 20 ++++++++------------ .execs/container.flow | 1 + .execs/helpers.flow | 14 ++++++++------ .execs/test.flow | 16 +++++++++++++--- .execs/validate.flow | 21 ++++++++++++++------- .github/workflows/ci.yaml | 6 +++--- docs/development.md | 13 ++++++++----- 7 files changed, 55 insertions(+), 36 deletions(-) diff --git a/.execs/build.flow b/.execs/build.flow index 08e3b83a..b41dc72b 100644 --- a/.execs/build.flow +++ b/.execs/build.flow @@ -8,10 +8,8 @@ executables: failFast: false execs: - ref: generate cli - - ref: generate completions - ref: generate docs - - ref: generate frontend - - ref: generate backend + - ref: generate tauri - verb: build name: binary @@ -31,20 +29,17 @@ executables: go build -o ${BIN_PATH}/${BIN_NAME} echo "flow built at ${BIN_PATH}/${BIN_NAME}" - - verb: generate - name: completions - tags: [docs] - exec: - dir: // - cmd: ./scripts/completions.sh - - verb: generate name: docs aliases: [documentation] + description: Generate docsify documentation for the project. tags: [docs] - exec: + parallel: + failFast: false dir: // - cmd: go run ./tools/docsgen/. + execs: + - cmd: go run ./tools/docsgen/. + - cmd: ./scripts/completions.sh - verb: generate name: cli @@ -63,6 +58,7 @@ executables: tags: [desktop] description: Generate code for the Tauri frontend and backend. parallel: + failFast: false execs: - ref: generate frontend - ref: generate backend diff --git a/.execs/container.flow b/.execs/container.flow index a431e03c..f0f83b29 100644 --- a/.execs/container.flow +++ b/.execs/container.flow @@ -22,6 +22,7 @@ executables: GOOS=linux GOARCH=amd64 go build -o flow echo "building container image..." $BUILDER build -t $IMAGE_REPO:$IMAGE_TAG . + rm flow - verb: run name: container diff --git a/.execs/helpers.flow b/.execs/helpers.flow index 49e6c51e..aac8e5cd 100644 --- a/.execs/helpers.flow +++ b/.execs/helpers.flow @@ -2,12 +2,14 @@ tags: [development, helper] executables: - verb: clean - name: ci - aliases: [codecov] - tags: [coverage, ci] - description: Clean up coverage files + name: tmp + aliases: [artifacts, ci] + description: Remove the temporary files created by executable runs exec: dir: // cmd: | - echo "Cleaning CI temp files..." - rm -f *.sarif *.out + echo "Cleaning coverage files..." + rm -f *.sarif *.out || true + echo "Clearing the bin directory..." + rm -rf .bin || true + rm ./flow || true diff --git a/.execs/test.flow b/.execs/test.flow index 133634bc..76244754 100644 --- a/.execs/test.flow +++ b/.execs/test.flow @@ -49,8 +49,13 @@ executables: execs: - cmd: | set -e - echo "Running Go unit tests with coverage..." - go test -race -coverprofile=unit-coverage.out -covermode=atomic -tags=unit ./... + echo "Running Go unit tests..." + if [ "$CI" = "true" ]; then + echo "Running Go unit tests with coverage..." + go test -race -coverprofile=unit-coverage.out -covermode=atomic -tags=unit ./... + else + go test -race -tags=unit ./... + fi echo "Unit tests completed" retries: 3 @@ -64,7 +69,12 @@ executables: - cmd: | set -e echo "Running Go E2E tests..." - go test -race -coverprofile=e2e-coverage.out -covermode=atomic -coverpkg=./... -tags=e2e ./tests/... + if [ "$CI" = "true" ]; then + echo "Running Go E2E tests with coverage..." + go test -race -coverprofile=e2e-coverage.out -covermode=atomic -coverpkg=./... -tags=e2e ./tests/... + else + go test -race -tags=e2e ./tests/... + fi echo "E2E tests completed" retries: 1 diff --git a/.execs/validate.flow b/.execs/validate.flow index 0b2d8ba4..e3682e56 100644 --- a/.execs/validate.flow +++ b/.execs/validate.flow @@ -37,11 +37,10 @@ executables: description: Run linters and formatters parallel: dir: // + failFast: false execs: - cmd: go fmt ./... - dir: // - cmd: go mod tidy - dir: // - cmd: | if ! command -v golangci-lint &> /dev/null; then echo "Installing golangci-lint..." @@ -49,7 +48,12 @@ executables: export PATH="$PATH:./bin" fi - golangci-lint run ./... --fix --output.sarif.path lint.sarif + if [ "$CI" = "true" ]; then + echo "Running golangci-lint with sarif output..." + golangci-lint run ./... --fix --output.sarif.path lint.sarif + else + golangci-lint run ./... --fix + fi - verb: lint name: ts @@ -67,12 +71,15 @@ executables: exec: dir: // cmd: | - echo "Running security scan..." - if ! command -v govulncheck &> /dev/null; then echo "Installing govulncheck..." go install golang.org/x/vuln/cmd/govulncheck@latest fi - govulncheck -format sarif ./... > govuln.sarif - echo "Security scan completed. Results saved to govuln.sarif" + if [ "$CI" = "true" ]; then + govulncheck -format sarif ./... > govuln.sarif + echo "Security scan completed. Results saved to govuln.sarif" + else + govulncheck ./... + echo "Security scan completed. No vulnerabilities found." + fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index acaaec96..237bd06f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: go-version: "^1.24" - uses: jahvon/flow-action@v1.0.0-beta1 with: - executable: 'lint go' + executable: 'lint go --param CI=true' timeout: '5m' flow-version: 'main' - name: Upload SARIF file @@ -43,7 +43,7 @@ jobs: go-version: "^1.24" - uses: jahvon/flow-action@v1.0.0-beta1 with: - executable: 'test unit' + executable: 'test unit --param CI=true' timeout: '5m' flow-version: 'main' id: unit-tests @@ -63,7 +63,7 @@ jobs: go-version: "^1.24" - uses: jahvon/flow-action@v1.0.0-beta1 with: - executable: 'test e2e' + executable: 'test e2e --param CI=true' timeout: '10m' flow-version: 'main' secrets: | diff --git a/docs/development.md b/docs/development.md index 3a938421..a728632a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -27,17 +27,20 @@ The `flow` project contains a few development executables that can be run locall workspace, you can run the following commands: ```sh -# Install local dependencies -flow install deps +# Install Go tool dependencies +flow install tools -# Build the project +# Build the CLI binary flow build binary # Validate code changes (runs tests, linters, codegen, etc) flow validate -# Run only tests -flow run tests +# Only generate code +flow generate + +# Only run tests +flow test all ``` ### Working with generated types