feat: lowering perms to CAP_NET_ADMIN

Author: evgeniy-scherbina

.github/workflows/ci.yml

@@ -73,6 +73,36 @@ jobs:
     - name: Download and verify dependencies
       run: make deps
 
+      # Before (default):
+      #  - /etc/resolv.conf -> /run/systemd/resolve/stub-resolv.conf
+      #  - stub-resolv.conf points to 127.0.0.53 (systemd-resolved stub listener)
+      #  - systemd-resolved forwards to the real upstream file:
+      #      /run/systemd/resolve/resolv.conf
+      #  Flow: /etc/resolv.conf -> stub-resolv.conf (127.0.0.53) -> systemd-resolved -> /run/systemd/resolve/resolv.conf
+      #
+      # After (bind-mount):
+      #  - /etc/resolv.conf is bind-mounted to /run/systemd/resolve/resolv.conf
+      #  - processes read upstream nameservers directly from /run/systemd/resolve/resolv.conf
+      #  Flow: /etc/resolv.conf -> /run/systemd/resolve/resolv.conf
+      #
+      # This makes processes talk directly to the upstream DNS servers and
+      # bypasses the systemd-resolved *stub listener* (127.0.0.53).
+      #
+      # Important nuance: systemd-resolved itself is NOT stopped; it still runs and updates
+      # /run/systemd/resolve/resolv.conf. Because this is a bind (not a copy), updates to the
+      # upstream list are visible. Trade-off: we lose the stub’s features (caching,
+      # split-DNS/VPN per-interface behavior, DNSSEC/DoT/DoH mediation, mDNS/LLMNR).
+      #
+      # Reason: network namespaces have their own network stack (interfaces, routes and loopback).
+      # A process inside a network namespace resolves 127.0.0.53 against that namespace’s loopback, not the host’s,
+      # and systemd-resolved usually listens on the host loopback. As a result the stub at 127.0.0.53 is often
+      # unreachable from an isolated namespace and DNS lookups fail.
+      # Bind-mounting /run/systemd/resolve/resolv.conf over /etc/resolv.conf forces processes to use the upstream
+      # nameservers directly, avoiding that failure.
+    - name: Change DNS configuration
+      if: runner.os == 'Linux'
+      run: sudo mount --bind /run/systemd/resolve/resolv.conf /etc/resolv.conf
+
     - name: Run unit tests
       run: make unit-test
 

Makefile

@@ -62,7 +62,7 @@ e2e-test:
 		echo "E2E tests require Linux platform. Current platform: $$(uname)"; \
 		exit 1; \
 	fi
-	sudo $(shell which go) test -v -race ./e2e_tests
+	sudo $(shell which go) test -v -race ./e2e_tests -count=1
 	@echo "✓ E2E tests passed!"
 
 # Run tests with coverage (needs sudo for E2E tests)

boundary.go

@@ -55,8 +55,8 @@ func New(ctx context.Context, config Config) (*Boundary, error) {
 }
 
 func (b *Boundary) Start() error {
-	// Start the jailer (network isolation)
-	err := b.jailer.Start()
+	// Configure the jailer (network isolation)
+	err := b.jailer.ConfigureBeforeCommandExecution()
 	if err != nil {
 		return fmt.Errorf("failed to start jailer: %v", err)
 	}
@@ -78,6 +78,10 @@ func (b *Boundary) Command(command []string) *exec.Cmd {
 	return b.jailer.Command(command)
 }
 
+func (b *Boundary) ConfigureAfterCommandExecution(processPID int) error {
+	return b.jailer.ConfigureAfterCommandExecution(processPID)
+}
+
 func (b *Boundary) Close() error {
 	// Stop proxy server
 	if b.proxyServer != nil {

cli/cli.go

@@ -3,8 +3,10 @@ package cli
 import (
 	"context"
 	"fmt"
+	"log"
 	"log/slog"
 	"os"
+	"os/exec"
 	"os/signal"
 	"path/filepath"
 	"strings"
@@ -92,8 +94,39 @@ func BaseCommand() *serpent.Command {
 	}
 }
 
+func isChild() bool {
+	return os.Getenv("CHILD") == "true"
+}
+
 // Run executes the boundary command with the given configuration and arguments
 func Run(ctx context.Context, config Config, args []string) error {
+	if isChild() {
+		log.Printf("boundary CHILD process is started")
+
+		vethNetJail := os.Getenv("VETH_JAIL_NAME")
+		err := jail.SetupChildNetworking(vethNetJail)
+		if err != nil {
+			return fmt.Errorf("failed to setup child networking: %v", err)
+		}
+		log.Printf("child networking is successfully configured")
+
+		// Program to run
+		bin := args[0]
+		args = args[1:]
+
+		cmd := exec.Command(bin, args...)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			log.Printf("failed to run %s: %v", bin, err)
+			return err
+		}
+
+		return nil
+	}
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -191,15 +224,26 @@ func Run(ctx context.Context, config Config, args []string) error {
 	// Execute command in boundary
 	go func() {
 		defer cancel()
-		cmd := boundaryInstance.Command(args)
-		cmd.Stderr = os.Stderr
-		cmd.Stdout = os.Stdout
-		cmd.Stdin = os.Stdin
+		cmd := boundaryInstance.Command(os.Args)
+
+		logger.Debug("Executing command in boundary", "command", strings.Join(os.Args, " "))
+		err := cmd.Start()
+		if err != nil {
+			logger.Error("Command failed to start", "error", err)
+			return
+		}
+
+		err = boundaryInstance.ConfigureAfterCommandExecution(cmd.Process.Pid)
+		if err != nil {
+			logger.Error("configuration after command execution failed", "error", err)
+			return
+		}
 
-		logger.Debug("Executing command in boundary", "command", strings.Join(args, " "))
-		err := cmd.Run()
+		logger.Debug("waiting on a child process to finish")
+		err = cmd.Wait()
 		if err != nil {
 			logger.Error("Command execution failed", "error", err)
+			return
 		}
 	}()
 

e2e_tests/boundary_integration_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -37,29 +38,15 @@ func findProjectRoot(t *testing.T) string {
 	}
 }
 
-// getNamespaceName gets the single network namespace name
-// Fails if there are 0 or multiple namespaces
-func getNamespaceName(t *testing.T) string {
-	cmd := exec.Command("ip", "netns", "list")
+func getChildProcessPID(t *testing.T) int {
+	cmd := exec.Command("pgrep", "-f", "boundary-test", "-n")
 	output, err := cmd.Output()
-	require.NoError(t, err, "Failed to list network namespaces")
-
-	lines := strings.Split(string(output), "\n")
-	var namespaces []string
-
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
-		if line != "" {
-			// Extract namespace name (first field)
-			parts := strings.Fields(line)
-			if len(parts) > 0 {
-				namespaces = append(namespaces, parts[0])
-			}
-		}
-	}
+	require.NoError(t, err)
 
-	require.Len(t, namespaces, 1, "Expected exactly one network namespace, found %d: %v", len(namespaces), namespaces)
-	return namespaces[0]
+	pidStr := strings.TrimSpace(string(output))
+	pid, err := strconv.Atoi(pidStr)
+	require.NoError(t, err)
+	return pid
 }
 
 func TestBoundaryIntegration(t *testing.T) {
@@ -73,18 +60,19 @@ func TestBoundaryIntegration(t *testing.T) {
 	require.NoError(t, err, "Failed to build boundary binary")
 
 	// Create context for boundary process
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Second)
 	defer cancel()
 
 	// Start boundary process with sudo
 	boundaryCmd := exec.CommandContext(ctx, "/tmp/boundary-test",
 		"--allow", "dev.coder.com",
 		"--allow", "jsonplaceholder.typicode.com",
 		"--log-level", "debug",
-		"--", "bash", "-c", "sleep 10 && echo 'Test completed'")
+		//"--", "/bin/bash")
+		"--", "/bin/bash", "-c", "/usr/bin/sleep 12 && /usr/bin/echo 'Test completed'")
 
-	// Suppress output to prevent terminal corruption
-	boundaryCmd.Stdout = os.Stdout // Let it go to /dev/null
+	boundaryCmd.Stdin = os.Stdin
+	boundaryCmd.Stdout = os.Stdout
 	boundaryCmd.Stderr = os.Stderr
 
 	// Start the process
@@ -95,12 +83,18 @@ func TestBoundaryIntegration(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// Get the namespace name that boundary created
-	namespaceName := getNamespaceName(t)
+	//namespaceName := getNamespaceName(t)
+
+	pidInt := getChildProcessPID(t)
+	pid := fmt.Sprintf("%v", pidInt)
+
+	fmt.Printf("pidInt: %v\n", pidInt)
+	//time.Sleep(200 * time.Second)
 
 	// Test HTTP request through boundary (from inside the jail)
 	t.Run("HTTPRequestThroughBoundary", func(t *testing.T) {
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "nsenter", "-t", pid, "-n", "--",
 			"curl", "http://jsonplaceholder.typicode.com/todos/1")
 
 		// Capture stderr separately
@@ -128,7 +122,7 @@ func TestBoundaryIntegration(t *testing.T) {
 		certPath := fmt.Sprintf("%v/ca-cert.pem", configDir)
 
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "sudo", "nsenter", "-t", pid, "-n", "--",
 			"env", fmt.Sprintf("SSL_CERT_FILE=%v", certPath), "curl", "-s", "https://dev.coder.com/api/v2")
 
 		// Capture stderr separately
@@ -149,7 +143,7 @@ func TestBoundaryIntegration(t *testing.T) {
 	// Test blocked domain (from inside the jail)
 	t.Run("BlockedDomainTest", func(t *testing.T) {
 		// Run curl directly in the namespace using ip netns exec
-		curlCmd := exec.Command("sudo", "ip", "netns", "exec", namespaceName,
+		curlCmd := exec.Command("sudo", "sudo", "nsenter", "-t", pid, "-n", "--",
 			"curl", "-s", "http://example.com")
 
 		// Capture stderr separately

jail/jail.go

@@ -8,8 +8,9 @@ import (
 )
 
 type Jailer interface {
-	Start() error
+	ConfigureBeforeCommandExecution() error
 	Command(command []string) *exec.Cmd
+	ConfigureAfterCommandExecution(processPID int) error
 	Close() error
 }