Merge pull request #57 from WarpBuilds/fix/transparent-cache

Author: guptaankit015

.github/workflows/release-testing.yaml

@@ -3,7 +3,7 @@ name: Branch Build and Upload to R2
 on:
   push:
     branches:
-      - ah-docker-layer-caching
+      - fix/transparent-cache
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.WB_PACKAGES_DEV_R2_ACCESS_KEY_ID }}

pkg/app/app.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 	"time"
@@ -90,10 +91,11 @@ type ProxySettings struct {
 }
 
 type TransparentCacheSettings struct {
-	DerpPort       int  `json:"derp_port"`
-	OginyPort      int  `json:"oginy_port"`
-	AsurPort       int  `json:"asur_port"`
-	LoggingEnabled bool `json:"logging_enabled"`
+	DerpPort       int    `json:"derp_port"`
+	OginyPort      int    `json:"oginy_port"`
+	AsurPort       int    `json:"asur_port"`
+	LoggingEnabled bool   `json:"logging_enabled"`
+	CertDir        string `json:"cert_dir"`
 }
 
 func (t *TransparentCacheSettings) ApplyDefaults() {
@@ -106,6 +108,14 @@ func (t *TransparentCacheSettings) ApplyDefaults() {
 	if t.AsurPort == 0 {
 		t.AsurPort = 59993
 	}
+	if t.CertDir == "" {
+		// Default to ~/runner/certs, with fallback to /home/runner/certs
+		homeBase := os.Getenv("HOME")
+		if homeBase == "" {
+			homeBase = "/home/runner"
+		}
+		t.CertDir = filepath.Join(homeBase, "certs")
+	}
 }
 
 type RunnerSettings struct {
@@ -258,6 +268,7 @@ func NewApp(ctx context.Context, opts *ApplicationOptions) error {
 			settings.TransparentCache.AsurPort,
 			settings.Proxy.CacheBackendHost,
 			settings.Agent.RunnerVerificationToken,
+			settings.TransparentCache.CertDir,
 			settings.TransparentCache.LoggingEnabled,
 		); err != nil {
 			log.Logger().Errorf("failed to start transparent cache: %v", err)

pkg/manager/manager_github.go

@@ -147,7 +147,6 @@ func (m *ghManager) StartRunner(ctx context.Context, opts *StartRunnerOptions) (
 				})
 				if err != nil {
 					log.Logger().Errorf("error running post-end hook %s: %v", hook.HookID(), err)
-					return nil, err
 				}
 			}
 

pkg/manager/manager_github_cri.go

@@ -141,7 +141,6 @@ func (m *ghcriManager) StartRunner(ctx context.Context, opts *StartRunnerOptions
 				})
 				if err != nil {
 					log.Logger().Errorf("error running post-end hook %s: %v", hook.HookID(), err)
-					return nil, err
 				}
 			}
 

pkg/manager/manager_windows_github_cri.go

@@ -149,7 +149,6 @@ func (m *ghWindowsCriManager) StartRunner(ctx context.Context, opts *StartRunner
 				})
 				if err != nil {
 					log.Logger().Errorf("error running post-end hook %s: %v", hook.HookID(), err)
-					return nil, err
 				}
 			}
 

pkg/transparent-cache/oginy/oginy.go

@@ -18,7 +18,9 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -393,10 +395,90 @@ func resolveRealIP(hostname string) (string, error) {
 	return "", fmt.Errorf("no A record found for %s", hostname)
 }
 
+// detectOS detects the OS using runtime.GOOS
+// Returns: "ubuntu" (assumes Canonical Ubuntu for Linux), "darwin" (macOS), "windows" (Windows)
+func detectOS() string {
+	switch runtime.GOOS {
+	case "linux":
+		return "ubuntu"
+	case "darwin":
+		return "darwin"
+	case "windows":
+		return "windows"
+	default:
+		panic(fmt.Sprintf("unsupported OS: %s", runtime.GOOS))
+	}
+}
+
+// UpdateSystemCATrust adds the local CA to the system's trusted certificate store
+// This integrates the CA with the OS-level trust store, making it trusted system-wide
+// Fully implemented for Ubuntu (Canonical), shell methods only for macOS and Windows
+func UpdateSystemCATrust(caCertPath string) error {
+	switch detectOS() {
+	case "ubuntu":
+		// Fully implemented for Ubuntu (Canonical) - uses update-ca-certificates
+		return updateUbuntuCATrust(caCertPath)
+	case "darwin":
+		// macOS - shell method only (not fully implemented)
+		return updateDarwinCATrust(caCertPath)
+	case "windows":
+		// Windows - shell method only (not fully implemented)
+		return updateWindowsCATrust(caCertPath)
+	default:
+		log.Printf("System CA trust update not implemented for OS: %s", detectOS())
+		log.Printf("Supported OS: ubuntu (Canonical Ubuntu), darwin (macOS), windows")
+		return fmt.Errorf("unsupported OS: %s", detectOS())
+	}
+}
+
+func updateUbuntuCATrust(caCertPath string) error {
+	// Copy CA to system location
+	systemCALocation := "/usr/local/share/ca-certificates/warpbuild-local-ca.crt"
+
+	// Read the CA certificate
+	caData, err := os.ReadFile(caCertPath)
+	if err != nil {
+		return fmt.Errorf("read CA certificate: %v", err)
+	}
+
+	// Write to system location (requires root)
+	if err := os.WriteFile(systemCALocation, caData, 0644); err != nil {
+		log.Printf("Warning: Could not write CA to system location %s (need root): %v", systemCALocation, err)
+		log.Printf("To add CA to system trust store, run as root or manually:")
+		log.Printf("  sudo cp %s %s", caCertPath, systemCALocation)
+		log.Printf("  sudo update-ca-certificates")
+		return fmt.Errorf("write CA to system location: %v", err)
+	}
+
+	// Run update-ca-certificates
+	cmd := exec.Command("update-ca-certificates")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Printf("Warning: update-ca-certificates failed: %v", err)
+		log.Printf("Output: %s", string(output))
+		return fmt.Errorf("update-ca-certificates failed: %v", err)
+	}
+
+	log.Printf("Successfully added CA to Ubuntu system trust store")
+	log.Printf("CA is now trusted system-wide at: %s", systemCALocation)
+	return nil
+}
+
+func updateDarwinCATrust(caCertPath string) error {
+	return nil
+}
+
+func updateWindowsCATrust(caCertPath string) error {
+	return nil
+}
+
 // Start starts the OGINY TLS reverse proxy service
 // If port is > 0, it uses that port, otherwise defaults to 443
 // If loggingEnabled is true, all proxy traffic will be logged to files
-func Start(port int, loggingEnabled bool) error {
+// derpPort and asurPort are used to route requests to the correct backend services
+// certDir specifies the directory where certificates should be stored
+// OS is automatically detected from the system
+func Start(port int, derpPort int, asurPort int, certDir string, loggingEnabled bool) error {
 	// Ignore cfgPath since we're inlining the config
 	listenAddr := ":443"
 	if port > 0 {
@@ -437,17 +519,12 @@ func Start(port int, loggingEnabled bool) error {
 		}
 	}
 
-	// Set up certificate directory
-	var certDir string
+	// Determine certDir with priority: OGINY_CERT_DIR env var -> settings file value (defaults already applied)
 	if dir := os.Getenv("OGINY_CERT_DIR"); dir != "" {
 		certDir = dir
+		log.Printf("Using certificate directory from OGINY_CERT_DIR: %s", certDir)
 	} else {
-		// Use $HOME env var with fallback to /home
-		homeBase := os.Getenv("HOME")
-		if homeBase == "" {
-			homeBase = "/home"
-		}
-		certDir = filepath.Join(homeBase, "runner", "certs")
+		log.Printf("Using certificate directory from settings: %s", certDir)
 	}
 
 	// Create certificate directory if it doesn't exist
@@ -471,37 +548,10 @@ func Start(port int, loggingEnabled bool) error {
 		log.Printf("CA certificate generated at %s", caCertPath)
 	}
 
-	// Set environment variables for current process and children
-	os.Setenv("NODE_OPTIONS", "--use-openssl-ca")
-	os.Setenv("NODE_EXTRA_CA_CERTS", caCertPath)
-	os.Setenv("SSL_CERT_FILE", caCertPath)
-
-	// If running in GitHub Actions, write to GITHUB_ENV
-	if githubEnv := os.Getenv("GITHUB_ENV"); githubEnv != "" {
-		log.Printf("Detected GitHub Actions environment, writing to GITHUB_ENV")
-		if err := appendToFile(githubEnv, fmt.Sprintf("NODE_OPTIONS=--use-openssl-ca\nNODE_EXTRA_CA_CERTS=%s\n", caCertPath)); err != nil {
-			log.Printf("Warning: failed to write to GITHUB_ENV: %v", err)
-		}
-		// Write SSL_CERT_FILE in a separate call to avoid duplicate-check skipping
-		if err := appendToFile(githubEnv, fmt.Sprintf("SSL_CERT_FILE=%s\n", caCertPath)); err != nil {
-			log.Printf("Warning: failed to write SSL_CERT_FILE to GITHUB_ENV: %v", err)
-		}
-	}
-
-	// Also write to /etc/environment if we have permissions (for system-wide)
-	if err := appendToFile("/etc/environment", fmt.Sprintf("NODE_OPTIONS=\"--use-openssl-ca\"\nNODE_EXTRA_CA_CERTS=\"%s\"\n", caCertPath)); err != nil {
-		// This is expected to fail if not running as root
-		log.Printf("Note: Could not write to /etc/environment (need root): %v", err)
-		log.Printf("To set system-wide, run as root or manually add to /etc/environment:")
-		log.Printf("  NODE_OPTIONS=\"--use-openssl-ca\"")
-		log.Printf("  NODE_EXTRA_CA_CERTS=\"%s\"", caCertPath)
-		log.Printf("  SSL_CERT_FILE=\"%s\"", caCertPath)
-	} else {
-		log.Printf("Successfully updated /etc/environment")
-		// Write SSL_CERT_FILE in a separate call to avoid duplicate-check skipping
-		if err := appendToFile("/etc/environment", fmt.Sprintf("SSL_CERT_FILE=\"%s\"\n", caCertPath)); err != nil {
-			log.Printf("Warning: failed to append SSL_CERT_FILE to /etc/environment: %v", err)
-		}
+	// Update system CA trust store (OS-specific)
+	// This integrates the CA with the OS-level trust store, making it trusted system-wide
+	if err := UpdateSystemCATrust(caCertPath); err != nil {
+		log.Printf("Warning: Failed to update system CA trust store: %v", err)
 	}
 
 	// Generate certificates for each domain
@@ -515,13 +565,13 @@ func Start(port int, loggingEnabled bool) error {
 			serverName: "warpbuild.blob.core.windows.net",
 			certFile:   filepath.Join(certDir, "warpbuild.crt"),
 			keyFile:    filepath.Join(certDir, "warpbuild.key"),
-			targetURL:  "http://127.0.0.1:50053",
+			targetURL:  fmt.Sprintf("http://127.0.0.1:%d", asurPort),
 		},
 		{
 			serverName: resultsReceiverHost,
 			certFile:   filepath.Join(certDir, "results-receiver.crt"),
 			keyFile:    filepath.Join(certDir, "results-receiver.key"),
-			targetURL:  "http://127.0.0.1:50052",
+			targetURL:  fmt.Sprintf("http://127.0.0.1:%d", derpPort),
 		},
 	}
 

pkg/transparent-cache/transparent-cache.go

@@ -209,7 +209,7 @@ func setupNftables(hostMappings map[string]string, oginyPort int) error {
 	return nil
 }
 
-func Start(derpPort, oginyPort, asurPort int, cacheBackendHost, warpBuildRunnerVerificationToken string, loggingEnabled bool) error {
+func Start(derpPort, oginyPort, asurPort int, cacheBackendHost, warpBuildRunnerVerificationToken string, certDir string, loggingEnabled bool) error {
 	log.Println("========================================")
 	log.Println("Starting Transparent Cache Services")
 	log.Println("========================================")
@@ -248,7 +248,7 @@ func Start(derpPort, oginyPort, asurPort int, cacheBackendHost, warpBuildRunnerV
 		go func() {
 			defer wg.Done()
 			log.Printf("Starting OGINY TLS Reverse Proxy on port %d...", oginyPort)
-			if err := oginy.Start(oginyPort, loggingEnabled); err != nil {
+			if err := oginy.Start(oginyPort, derpPort, asurPort, certDir, loggingEnabled); err != nil {
 				errChan <- fmt.Errorf("OGINY service error: %v", err)
 			}
 		}()

test.go

@@ -65,7 +65,8 @@ func main() {
 
 	// Start the transparent cache services in a goroutine
 	go func() {
-		err := transparentcache.Start(*derpPort, *oginyPort, *asurPort, *backendURL, *authToken, *debug)
+		// Use empty string for certDir to use default behavior
+		err := transparentcache.Start(*derpPort, *oginyPort, *asurPort, *backendURL, *authToken, "/home/runner/certs", *debug)
 		errChan <- err
 	}()