Fix persistent worker protocol bugs in scalafileextract (#150)

* Fix persistent worker protocol bugs in scalafileextract

Fixes three critical bugs that violated the Bazel persistent worker protocol:

1. **Stdout pollution**: The Node.js scalameta parser subprocess inherited
   os.Stdout, allowing any subprocess output to corrupt the persistent worker
   protocol stream. Per Bazel's spec, only WorkResponses should be written to
   stdout. Now redirects subprocess stdout to stderr.

2. **Missing stdout flush**: After writing WorkResponse messages, stdout was
   not explicitly flushed. Since Go's os.Stdout is buffered, responses could
   sit in the buffer causing Bazel to hang waiting for the worker. Now calls
   os.Stdout.Sync() after each response.

3. **Error handling**: When batchWork() failed, the worker terminated instead
   of reporting the error via WorkResponse. This crashed the entire worker
   process on any parse error, requiring Bazel to restart the worker and
   losing persistent worker performance benefits. Now reports errors via
   WorkResponse.ExitCode and .Output fields.

* Use self-hosted runner

* Use remotejdk

* use buffered writer for stdout

Author: pcj

.bazelrc

@@ -1,3 +1,6 @@
+# use remove java
+build --java_runtime_version=remotejdk_21
+
 # Keep until @io_bazel_rules_scala is upgraded
 build --incompatible_java_common_parameters=false
 

.github/workflows/ci.yaml

@@ -14,32 +14,15 @@ on:
 jobs:
   test:
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - uses: actions/checkout@v3
 
       - uses: bazelbuild/setup-bazelisk@v3
 
-      - name: Mount bazel action cache
-        uses: actions/cache@v4
-        if: always()
-        with:
-          path: "~/.cache/bazel"
-          key: bazel
-
-      - name: Mount bazel repo cache
-        uses: actions/cache@v4
-        if: always()
-        with:
-          path: "~/.cache/bazel-repo"
-          key: bazel-repo
-
       - name: bazel test
-        env:
-          # Bazelisk will download bazel to here, ensure it is cached between runs.
-          XDG_CACHE_HOME: ~/.cache/bazel-repo
         run: >-
           bazel
           --bazelrc=.github/workflows/ci.bazelrc

cmd/scalafileextract/scalafileextract.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"flag"
 	"fmt"
@@ -84,6 +85,9 @@ func run(args []string) error {
 }
 
 func persistentWork(cfg *Config) error {
+	// Use a buffered writer for stdout to ensure we can flush properly
+	stdout := bufio.NewWriter(os.Stdout)
+
 	for {
 		var req wppb.WorkRequest
 		if err := protobuf.ReadDelimitedFrom(&req, os.Stdin); err != nil {
@@ -106,12 +110,18 @@ func persistentWork(cfg *Config) error {
 		batchCfg.Cwd = cfg.Cwd
 
 		if err := batchWork(&batchCfg); err != nil {
-			return fmt.Errorf("performing persistent batch!: %v", err)
+			// Don't terminate the worker on batch errors; report via WorkResponse
+			resp.ExitCode = 1
+			resp.Output = fmt.Sprintf("performing persistent batch: %v", err)
 		}
 
-		if err := protobuf.WriteDelimitedTo(&resp, os.Stdout); err != nil {
+		if err := protobuf.WriteDelimitedTo(&resp, stdout); err != nil {
 			return fmt.Errorf("writing work response: %v", err)
 		}
+		// Flush stdout to ensure the WorkResponse is immediately available to Bazel
+		if err := stdout.Flush(); err != nil {
+			return fmt.Errorf("flushing work response: %v", err)
+		}
 	}
 	return nil
 }

pkg/parser/scalameta_parser.go

@@ -169,8 +169,11 @@ func (s *ScalametaParser) Start() error {
 		"NODE_PATH=" + processDir,
 		fmt.Sprintf("PORT=%d", s.httpPort),
 	}
-	cmd.Stdin = os.Stdin
-	cmd.Stdout = os.Stdout
+	// Don't inherit stdin from parent process
+	cmd.Stdin = nil
+	// Redirect stdout to stderr to avoid polluting the persistent worker protocol
+	// (only WorkResponses should go to stdout in persistent worker mode)
+	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	s.cmd = cmd
 

pkg/provider/semanticdb_provider.go

@@ -136,7 +136,7 @@ func (cr *SemanticdbProvider) CanProvide(dep *resolver.ImportLabel, expr build.E
 func (r *SemanticdbProvider) ParseScalaRule(kind string, from label.Label, dir string, srcs ...string) (*sppb.Rule, error) {
 	rule, err := r.delegate.ParseScalaRule(kind, from, dir, srcs...)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("semanticdb: %v", err)
 	}
 	for _, file := range rule.Files {
 		r.visitFile(from.Pkg, file)