feat: add http download restore method (#483)

* add http download method and tests for sftp storage type

* add http download method and tests for sftp storage type

* feat: add http download restore method and tests for cache-cli using sftp as storage type

Author: bogyo210

cache-cli/.htpasswd

@@ -0,0 +1 @@
+test:$apr1$kCL/ipwt$qOeC2mnzu9dJrmVYw2agO.

cache-cli/Dockerfile.sftp_server

@@ -1,9 +1,13 @@
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 
-RUN apt-get update && apt-get install -y openssh-server
+RUN apt-get update && apt-get install -y openssh-server nginx
 RUN mkdir -p /var/run/sshd
 
 COPY sshd_config /etc/ssh/sshd_config
+COPY nginx.conf /etc/nginx/nginx.conf
+COPY default.conf /etc/nginx/sites-enabled/default
+COPY wrapper.sh /wrapper.sh
+COPY .htpasswd /etc/nginx/.htpasswd
 
 RUN addgroup ftpaccess
 RUN adduser tester --ingroup ftpaccess --shell /bin/bash --disabled-password --gecos ''
@@ -13,5 +17,6 @@ RUN mkdir /etc/ssh/authorized_keys
 COPY id_rsa.pub /tmp/id_rsa.pub
 RUN cat /tmp/id_rsa.pub >> /etc/ssh/authorized_keys/tester
 
+EXPOSE 80
 EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]
+CMD ["/wrapper.sh"]

cache-cli/cmd/restore.go

@@ -103,7 +103,7 @@ func findMatchingKey(availableKeys []storage.CacheKey, match string) string {
 func downloadAndUnpackKey(storage storage.Storage, archiver archive.Archiver, metricsManager metrics.MetricsManager, key string) {
 	downloadStart := time.Now()
 	log.Infof("Downloading key '%s'...", key)
-	compressed, err := storage.Restore(key)
+	compressed, err := downloadKey(storage, key)
 	utils.Check(err)
 
 	downloadDuration := time.Since(downloadStart)
@@ -127,6 +127,29 @@ func downloadAndUnpackKey(storage storage.Storage, archiver archive.Archiver, me
 	}
 }
 
+func downloadKey(storage storage.Storage, key string) (*os.File, error) {
+	backend := os.Getenv("SEMAPHORE_CACHE_BACKEND")
+
+	// If this is not an sftp backend, then we are not in a cloud environment,
+	// and in there, there's no CDN variation, so just use the storage.
+	if backend != "sftp" {
+		return storage.Restore(key)
+	}
+
+	// Here, we are using sftp, so we know we are in a cloud job.
+	// But, not all cloud jobs should use this, so we only use it
+	// if the SEMAPHORE_CACHE_CDN_* variables are defined
+	cdnURL := os.Getenv("SEMAPHORE_CACHE_CDN_URL")
+	cdnKey := os.Getenv("SEMAPHORE_CACHE_CDN_KEY")
+	cdnSecret := os.Getenv("SEMAPHORE_CACHE_CDN_SECRET")
+	if cdnURL == "" || cdnKey == "" || cdnSecret == "" {
+		return storage.Restore(key)
+	}
+
+	log.Infof("Restoring using HTTP URL %s...", cdnURL)
+	return files.DownloadFromHTTP(cdnURL, cdnKey, cdnSecret, key)
+}
+
 func publishMetrics(metricsManager metrics.MetricsManager, fileInfo fs.FileInfo, downloadDuration time.Duration) {
 	metricsToPublish := []metrics.Metric{
 		{Name: metrics.CacheDownloadSize, Value: fmt.Sprintf("%d", fileInfo.Size())},

cache-cli/cmd/restore_test.go

@@ -164,6 +164,67 @@ func Test__Restore(t *testing.T) {
 			os.Remove(tempDir)
 		})
 	})
+
+	runTestForSingleBackend(t, "sftp", func(storage storage.Storage) {
+		t.Run("restoring using HTTP works", func(t *testing.T) {
+			storage.Clear()
+
+			tempDir, _ := ioutil.TempDir(os.TempDir(), "*")
+			tempFile, _ := ioutil.TempFile(tempDir, "*")
+			_ = tempFile.Close()
+
+			archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager())
+			compressAndStore(storage, archiver, "abc", tempDir)
+
+			// set the environment variables to download using HTTP instead before restoring
+			os.Setenv("SEMAPHORE_CACHE_CDN_URL", "http://sftp-server:80")
+			os.Setenv("SEMAPHORE_CACHE_CDN_KEY", "test")
+			os.Setenv("SEMAPHORE_CACHE_CDN_SECRET", "test")
+			defer func() {
+				os.Unsetenv("SEMAPHORE_CACHE_CDN_URL")
+				os.Unsetenv("SEMAPHORE_CACHE_CDN_KEY")
+				os.Unsetenv("SEMAPHORE_CACHE_CDN_SECRET")
+			}()
+
+			RunRestore(restoreCmd, []string{"^abc"})
+			output := readOutputFromFile(t)
+
+			restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir))
+			assert.Contains(t, output, "HIT: '^abc', using key 'abc'.")
+			assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath))
+
+			os.Remove(tempFile.Name())
+			os.Remove(tempDir)
+		})
+
+		t.Run("restoring defaults to use SFTP if not all variables are available", func(t *testing.T) {
+			storage.Clear()
+
+			tempDir, _ := ioutil.TempDir(os.TempDir(), "*")
+			tempFile, _ := ioutil.TempFile(tempDir, "*")
+			_ = tempFile.Close()
+
+			archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager())
+			compressAndStore(storage, archiver, "abc", tempDir)
+
+			// Set just the URL, but not the user/pass
+			// This means SFTP will still be used.
+			os.Setenv("SEMAPHORE_CACHE_CDN_URL", "http://sftp-server:80")
+			defer func() {
+				os.Unsetenv("SEMAPHORE_CACHE_CDN_URL")
+			}()
+
+			RunRestore(restoreCmd, []string{"^abc"})
+			output := readOutputFromFile(t)
+
+			restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir))
+			assert.Contains(t, output, "HIT: '^abc', using key 'abc'.")
+			assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath))
+
+			os.Remove(tempFile.Name())
+			os.Remove(tempDir)
+		})
+	})
 }
 
 func Test__AutomaticRestore(t *testing.T) {

cache-cli/cmd/root_test.go

@@ -38,6 +38,21 @@ var testBackends = map[string]TestBackend{
 	},
 }
 
+func runTestForSingleBackend(t *testing.T, testBackend string, test func(storage.Storage)) {
+	backend := testBackends[testBackend]
+	if runtime.GOOS == "windows" && !backend.runInWindows {
+		return
+	}
+
+	for envVarName, envVarValue := range backend.envVars {
+		os.Setenv(envVarName, envVarValue)
+	}
+
+	storage, err := storage.InitStorage()
+	assert.Nil(t, err)
+	test(storage)
+}
+
 func runTestForAllBackends(t *testing.T, test func(string, storage.Storage)) {
 	for backendType, testBackend := range testBackends {
 		if runtime.GOOS == "windows" && !testBackend.runInWindows {

cache-cli/default.conf

@@ -0,0 +1,10 @@
+server {
+    listen 80 default_server;
+    server_name _;
+    root /home/tester;
+    auth_basic "Auth required";
+    auth_basic_user_file /etc/nginx/.htpasswd;
+    location / {
+        autoindex on;
+    }
+}

cache-cli/docker-compose.yml

@@ -39,6 +39,7 @@ services:
     container_name: sftp-server
     ports:
       - "2222:22"
+      - "8080:8080"
     build:
       context: .
       dockerfile: Dockerfile.sftp_server

cache-cli/nginx.conf

@@ -0,0 +1,18 @@
+user root;
+worker_processes auto;
+pid /run/nginx.pid;
+events {
+    worker_connections 10;
+}
+http {
+    sendfile off;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    server_tokens off;
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    gzip off;
+    include /etc/nginx/sites-enabled/*;
+}

cache-cli/pkg/files/download.go

@@ -0,0 +1,45 @@
+package files
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+)
+
+func DownloadFromHTTP(URL, username, password, key string) (*os.File, error) {
+	client := &http.Client{}
+	downloadURL := fmt.Sprintf("%s/%s", URL, key)
+	req, err := http.NewRequest("GET", downloadURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.SetBasicAuth(username, password)
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to download file: %s", resp.Status)
+	}
+
+	localFile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("%s-*", key))
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = localFile.ReadFrom(resp.Body)
+	if err != nil {
+		_ = localFile.Close()
+		return nil, err
+	}
+
+	err = resp.Body.Close()
+	if err != nil {
+		_ = localFile.Close()
+		return nil, err
+	}
+
+	return localFile, localFile.Close()
+}

cache-cli/pkg/files/download_test.go

@@ -0,0 +1,58 @@
+package files
+
+import (
+	"os"
+	"testing"
+	"runtime"
+	"github.com/semaphoreci/toolbox/cache-cli/pkg/storage"
+	"github.com/stretchr/testify/require"
+)
+
+func Test__DownloadFromHTTP(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+	sftpStorage, err := storage.NewSFTPStorage(storage.SFTPStorageOptions{
+		URL:            "sftp-server:22",
+		Username:       "tester",
+		PrivateKeyPath: "/root/.ssh/semaphore_cache_key",
+		Config: storage.StorageConfig{
+			MaxSpace:   1024,
+			SortKeysBy: storage.SortBySize,
+		},
+	})
+
+	require.NoError(t, err)
+	require.NoError(t, sftpStorage.Clear())
+	require.NoError(t, sftpStorage.Store("abc", "testdata/test.txt"))
+
+	t.Run("download works", func(t *testing.T) {
+		f, err := DownloadFromHTTP("http://sftp-server:80", "test", "test", "abc")
+		require.NoError(t, err)
+		require.FileExists(t, f.Name())
+
+		content, err := os.ReadFile(f.Name())
+		require.NoError(t, err)
+		require.Equal(t, "Test 123", string(content))
+	})
+
+	t.Run("download fails if URL is not reachable", func(t *testing.T) {
+		_, err := DownloadFromHTTP("http://sftp-server:801", "test", "test", "abc")
+		require.ErrorContains(t, err, "connection refused")
+	})
+
+	t.Run("download fails if username is invalid", func(t *testing.T) {
+		_, err := DownloadFromHTTP("http://sftp-server:80", "wrong", "test", "abc")
+		require.ErrorContains(t, err, "failed to download file: 401 Unauthorized")
+	})
+
+	t.Run("download fails if password is wrong", func(t *testing.T) {
+		_, err := DownloadFromHTTP("http://sftp-server:80", "test", "wrong", "abc")
+		require.ErrorContains(t, err, "failed to download file: 401 Unauthorized")
+	})
+
+	t.Run("download fails if file does not exist", func(t *testing.T) {
+		_, err := DownloadFromHTTP("http://sftp-server:80", "test", "test", "does-not-exist")
+		require.ErrorContains(t, err, "failed to download file: 404 Not Found")
+	})
+}