refactor: directly download workspace (#1795)

Author: FabianKramm

Makefile

@@ -1,13 +1,14 @@
 GOOS := $(shell go env GOOS)
 GOARCH := $(shell go env GOARCH)
+SKIP_INSTALL := false
 
 # Platform host
 PLATFORM_HOST := localhost:8080
 
 # Build the CLI and Desktop
 .PHONY: build
 build:
-	BUILD_PLATFORMS=$(GOOS) BUILD_ARCHS=$(GOARCH) ./hack/rebuild.sh
+	SKIP_INSTALL=$(SKIP_INSTALL) BUILD_PLATFORMS=$(GOOS) BUILD_ARCHS=$(GOARCH) ./hack/rebuild.sh
 
 # Run the desktop app
 .PHONY: run-desktop
@@ -17,12 +18,15 @@ run-desktop: build
 # Run the daemon against loft host
 .PHONY: run-daemon
 run-daemon: build
-	devpod pro daemon start --host $(PLATFORM_HOST) 
+	devpod pro daemon start --host $(PLATFORM_HOST)
+
+# Namespace to use for the platform
+NAMESPACE := loft
 
 # Copy the devpod binary to the platform pod
 .PHONY: cp-to-platform
 cp-to-platform:
 	SKIP_INSTALL=true BUILD_PLATFORMS=linux BUILD_ARCHS=$(GOARCH) ./hack/rebuild.sh
-	POD=$$(kubectl get pod -n loft -l app=loft,release=loft -o jsonpath='{.items[0].metadata.name}'); \
+	POD=$$(kubectl get pod -n $(NAMESPACE) -l app=loft,release=loft -o jsonpath='{.items[0].metadata.name}'); \
 	echo "Copying ./test/devpod-linux-$(GOARCH) to pod $$POD"; \
-	kubectl cp -n loft ./test/devpod-linux-$(GOARCH) $$POD:/usr/local/bin/devpod 
+	kubectl cp -n $(NAMESPACE) ./test/devpod-linux-$(GOARCH) $$POD:/usr/local/bin/devpod 

cmd/agent/container/setup.go

@@ -4,10 +4,10 @@ package container
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -48,7 +48,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 var DockerlessImageConfigOutput = "/.dockerless/image.json"
@@ -585,37 +584,40 @@ func streamMount(ctx context.Context, workspaceInfo *provider2.ContainerWorkspac
 	// if we have a platform workspace socket we connect directly to it
 	if workspaceInfo.CLIOptions.Platform.Enabled {
 		// check if the runner proxy socket exists
-		logger.Infof("Waiting for runner proxy socket to be created")
-		var err error
-		waitErr := wait.PollUntilContextTimeout(ctx, time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) {
-			if _, err = os.Stat(filepath.Join(RootDir, ts.RunnerProxySocket)); err != nil {
-				return false, nil
-			}
-
-			return true, nil
-		})
-		if waitErr != nil {
-			return fmt.Errorf("finding runner proxy socket: %w", err)
-		}
-
 		httpClient := &http.Client{
 			Transport: &http.Transport{
-				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
-					return net.Dial("unix", filepath.Join(RootDir, ts.RunnerProxySocket))
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: true,
 				},
 			},
 		}
 
+		// build the url
 		logger.Infof("Download %s into DevContainer %s", m.Source, m.Target)
-		resp, err := httpClient.Get("http://runner-proxy/workspace-download?path=" + url.QueryEscape(m.Source))
+		url := fmt.Sprintf(
+			"https://%s/kubernetes/management/apis/management.loft.sh/v1/namespaces/%s/devpodworkspaceinstances/%s/download?path=%s",
+			ts.RemoveProtocol(workspaceInfo.CLIOptions.Platform.PlatformHost),
+			workspaceInfo.CLIOptions.Platform.InstanceNamespace,
+			workspaceInfo.CLIOptions.Platform.InstanceName,
+			url.QueryEscape(m.Source),
+		)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			return fmt.Errorf("create request: %w", err)
+		}
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", workspaceInfo.CLIOptions.Platform.AccessKey))
+
+		// send the request
+		resp, err := httpClient.Do(req)
 		if err != nil {
 			return fmt.Errorf("download workspace: %w", err)
 		}
 		defer resp.Body.Close()
 
 		// check if the response is ok
 		if resp.StatusCode != http.StatusOK {
-			return fmt.Errorf("download workspace: %s", resp.Status)
+			body, _ := io.ReadAll(resp.Body)
+			return fmt.Errorf("download workspace: body = %s, status = %s", string(body), resp.Status)
 		}
 
 		// create progress reader

go.mod

@@ -29,9 +29,9 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/joho/godotenv v1.5.1
 	github.com/julienschmidt/httprouter v1.3.0
-	github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.16
+	github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.19
 	github.com/loft-sh/analytics-client v0.0.0-20240219162240-2f4c64b2494e
-	github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.16
+	github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.19
 	github.com/loft-sh/apiserver v0.0.0-20250206205835-422f1d472459
 	github.com/loft-sh/log v0.0.0-20240219160058-26d83ffb46ac
 	github.com/loft-sh/programming-language-detection v0.0.5

go.sum

@@ -422,12 +422,12 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/loft-sh/admin-apis v0.0.0-20250221182517-7499d86167d2 h1:om1MqUdW84ZQc0GMGGgFfPI6xpTbrF+6DwKVq+76R44=
 github.com/loft-sh/admin-apis v0.0.0-20250221182517-7499d86167d2/go.mod h1:WHCqWfljfD1hkwk41hLeqBhW2yeLvWipB1sH6vfnR7U=
-github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.16 h1:a7iemRlezfhsmSWJYmhPpeMNIkRZFHhrJL538nLCqlM=
-github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.16/go.mod h1:UteiUdc6QeGt2uXIo5K5qlReRa366GbPY5QdDuPyFCs=
+github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.19 h1:SApHDj5CL+uuwL3Yu3d8RGzaMM4BAnBdGrWD5bS58yA=
+github.com/loft-sh/agentapi/v4 v4.3.0-devpod.alpha.19/go.mod h1:UteiUdc6QeGt2uXIo5K5qlReRa366GbPY5QdDuPyFCs=
 github.com/loft-sh/analytics-client v0.0.0-20240219162240-2f4c64b2494e h1:JcPnMaoczikvpasi8OJ47dCkWZjfgFubWa4V2SZo7h0=
 github.com/loft-sh/analytics-client v0.0.0-20240219162240-2f4c64b2494e/go.mod h1:FFWcGASyM2QlWTDTCG/WBVM/XYr8btqYt335TFNRCFg=
-github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.16 h1:tvVofFbGDND2ifiAQeA9gHCUhFR1GHBntlltSDs9fTk=
-github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.16/go.mod h1:QvDqPCWhP8VG8GH8OnbJ3ps8SmdDiF6f2qHDI2mvqBI=
+github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.19 h1:0451oc0fkDFtlOqW0sRHYW1E6gZ3DuO/u88gNxMYNk0=
+github.com/loft-sh/api/v4 v4.3.0-devpod.alpha.19/go.mod h1:tzvr5pIOmX9omKw0pjqTQTLj9n05WR8Llk2XuQ8sC+4=
 github.com/loft-sh/apiserver v0.0.0-20250206205835-422f1d472459 h1:6SrgBtT1S9ANsQMoO/O0Mq+hs9EbC5te5kPqOBfg5UI=
 github.com/loft-sh/apiserver v0.0.0-20250206205835-422f1d472459/go.mod h1:rung3jsKjaVAtykQN0vWmFHhx2A/umpRyAae8BJVSeE=
 github.com/loft-sh/log v0.0.0-20240219160058-26d83ffb46ac h1:Gz/7Lb7WgdgIv+KJz87ORA1zvQW52tUqKPGyunlp4dQ=

pkg/extract/extract.go

@@ -37,7 +37,7 @@ func Extract(origReader io.Reader, destFolder string, options ...Option) error {
 	}
 
 	// read ahead
-	bufioReader := bufio.NewReaderSize(origReader, 10*1024*1024)
+	bufioReader := bufio.NewReaderSize(origReader, 1024*1024)
 	testBytes, err := bufioReader.Peek(2) // read 2 bytes
 	if err != nil {
 		return err

pkg/ts/workspace_server.go

@@ -240,9 +240,6 @@ func (s *WorkspaceServer) startListeners(ctx context.Context, projectName, works
 		mux.HandleFunc("/git-credentials", func(w http.ResponseWriter, r *http.Request) {
 			s.gitCredentialsHandler(w, r, lc, transport, projectName, workspaceName)
 		})
-		mux.HandleFunc("/workspace-download", func(w http.ResponseWriter, r *http.Request) {
-			s.workspaceDownloadHandler(w, r, lc, transport, projectName, workspaceName)
-		})
 		if err := http.Serve(runnerProxyListener, mux); err != nil && err != http.ErrServerClosed {
 			s.log.Errorf("HTTP runner proxy server error: %v", err)
 		}
@@ -318,45 +315,6 @@ func (s *WorkspaceServer) gitCredentialsHandler(w http.ResponseWriter, r *http.R
 	proxy.ServeHTTP(w, r)
 }
 
-// httpPortForwardHandler is the HTTP reverse proxy handler for workspace.
-// It reconstructs the target URL using custom headers and forwards the request.
-func (s *WorkspaceServer) workspaceDownloadHandler(w http.ResponseWriter, r *http.Request, lc *tailscale.LocalClient, transport *http.Transport, projectName, workspaceName string) {
-	s.log.Infof("Received workspace download request from %s", r.RemoteAddr)
-
-	// create a new http client with a custom transport
-	discoveredRunner, err := s.discoverRunner(r.Context(), lc)
-	if err != nil {
-		http.Error(w, "failed to discover runner", http.StatusInternalServerError)
-		return
-	}
-
-	// get the path from the query
-	path := r.URL.Query().Get("path")
-	if path == "" {
-		http.Error(w, "missing path", http.StatusBadRequest)
-		return
-	}
-
-	// build the runner URL
-	runnerURL := fmt.Sprintf("http://%s.ts.loft/devpod/%s/%s/workspace-download?path=%s", discoveredRunner, projectName, workspaceName, url.QueryEscape(r.URL.Query().Get("path")))
-	parsedURL, err := url.Parse(runnerURL)
-	if err != nil {
-		http.Error(w, "failed to parse runner URL", http.StatusInternalServerError)
-		return
-	}
-
-	// Build the reverse proxy with a custom Director.
-	proxy := httputil.NewSingleHostReverseProxy(parsedURL)
-	proxy.Director = func(req *http.Request) {
-		dest := *parsedURL
-		req.URL = &dest
-		req.Host = dest.Host
-		req.Header.Set("Authorization", "Bearer "+s.config.AccessKey)
-	}
-	proxy.Transport = transport
-	proxy.ServeHTTP(w, r)
-}
-
 // httpPortForwardHandler is the HTTP reverse proxy handler for workspace.
 // It reconstructs the target URL using custom headers and forwards the request.
 func (s *WorkspaceServer) httpPortForwardHandler(w http.ResponseWriter, r *http.Request) {