projects refresh background service (#2011)

* projects refresh service

Author: motatoes

.github/workflows/drift_deploy.yml

@@ -1,19 +1,19 @@
 # See https://fly.io/docs/app-guides/continuous-deployment-with-github-actions/
 
-name: Deploy pro-backend
+name: Deploy projects-refresh-service
 on:
   push:
     branches:
       - develop
-      - pro
 jobs:
   deploy:
-    name: Deploy app
+    name: Deploy projects-refresh-service app
     runs-on: ubuntu-latest
-    concurrency: deploy-group    # optional: ensure only one action runs at a time
+    concurrency: projects-refresh    # optional: ensure only one action runs at a time
     steps:
       - uses: actions/checkout@v4
       - uses: superfly/flyctl-actions/setup-flyctl@master
-      - run: flyctl deploy --remote-only --config fly-pro.toml
+      - run: flyctl deploy --remote-only --config fly-projects-refresh-service.toml --image-label latest
+
         env:
-          FLY_API_TOKEN: ${{ secrets.FLYIO_PRO_TOKEN }}
+          FLY_API_TOKEN: ${{ secrets.FLYIO_PROJECTS_REFRESH_SVC_TOKEN }}

.github/workflows/pro-deploy.yml

@@ -0,0 +1,28 @@
+FROM golang:1.24.0 as builder
+ARG COMMIT_SHA
+RUN echo "commit sha: ${COMMIT_SHA}"
+
+# Set the working directory
+WORKDIR $GOPATH/src/github.com/diggerhq/digger
+
+# Copy all required source, blacklist files that are not required through `.dockerignore`
+COPY . .
+
+RUN go build -ldflags="-X 'main.Version=${COMMIT_SHA}'" -o projects_refresh_exe ./background/projects-refresh-service
+
+# Multi-stage build will just copy the binary to an alpine image.
+FROM ubuntu:24.04 as runner
+ARG COMMIT_SHA
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y ca-certificates curl && apt-get install -y git && apt-get clean all
+RUN update-ca-certificates
+
+RUN echo "commit sha: ${COMMIT_SHA}"
+
+# Copy the binary to the corresponding folder
+COPY --from=builder /go/src/github.com/diggerhq/digger/projects_refresh_exe /app/projects_refresh_exe
+RUN chmod +x projects_refresh_exe
+
+# Run the binary
+ENTRYPOINT ["./projects_refresh_exe"]

Dockerfile_bg_projects_refresh

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"fmt"
+	"github.com/diggerhq/digger/libs/git_utils"
 	"log/slog"
 	"net/http"
 	"os"
@@ -67,7 +68,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 
 	// update the cache here, do it async for immediate response
 	go func() {
-		err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
+		err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
 			diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 			diggerYmlStr = string(diggerYmlBytes)
 			config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)

backend/controllers/cache.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/diggerhq/digger/libs/git_utils"
 	"log/slog"
 	"math/rand"
 	"net/http"
@@ -915,7 +916,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 	var diggerYmlStr string
 	var dependencyGraph graph.Graph[string, dg_configuration.Project]
 
-	err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
+	err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
 		slog.Debug("Reading Digger config from cloned repository", "directory", dir)
 
 		diggerYmlStr, err = dg_configuration.ReadDiggerYmlFileContents(dir)

backend/controllers/github.go

@@ -1642,6 +1642,7 @@ func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.D
 
 	err = db.GormDB.Transaction(func(tx *gorm.DB) error {
 		for _, dc := range config.Projects {
+			slog.Debug("refreshing for project", "name", dc.Name, "dir", dc.Dir)
 			projectName := dc.Name
 			projectDirectory := dc.Dir
 			p, err := db.GetProjectByName(orgId, repoFullName, projectName)

backend/models/storage.go

@@ -0,0 +1,107 @@
+package service_clients
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"time"
+)
+
+type MachineConfig struct {
+	Name   string `json:"name"`
+	Config struct {
+		Image string `json:"image"`
+		Env   struct {
+			CloneUrl     string `json:"DIGGER_GITHUB_REPO_CLONE_URL"`
+			Branch       string `json:"DIGGER_GITHUB_REPO_CLONE_BRANCH"`
+			GithubToken  string `json:"DIGGER_GITHUB_TOKEN"`
+			RepoFullName string `json:"DIGGER_REPO_FULL_NAME"`
+			OrgId        string `json:"DIGGER_ORG_ID"`
+		} `json:"env"`
+		Guest struct {
+			CPUs     int    `json:"cpus"`
+			CPUKind  string `json:"cpu_kind"`
+			MemoryMB int    `json:"memory_mb"`
+		} `json:"guest"`
+		AutoDestroy bool `json:"auto_destroy"`
+	} `json:"config"`
+}
+
+type MachineResponse struct {
+	ID string `json:"id"`
+}
+
+type QueuedResponse struct {
+	Queued string `json:"queued"`
+}
+
+func TriggerProjectsRefreshService(cloneUrl string, branch string, githubToken string, repoFullName string, orgId string) (*MachineResponse, error) {
+
+	slog.Debug("awaiting machine fetch")
+
+	// Prepare machine configuration
+	machineConfig := MachineConfig{
+		Name: fmt.Sprintf("hello-%d", time.Now().UnixMilli()),
+	}
+
+	machineConfig.Config.Image = "registry.fly.io/projects-refresh-service:latest"
+	machineConfig.Config.Env.CloneUrl = cloneUrl
+	machineConfig.Config.Env.Branch = branch
+	machineConfig.Config.Env.GithubToken = githubToken
+	machineConfig.Config.Env.RepoFullName = repoFullName
+	machineConfig.Config.Env.OrgId = orgId
+
+	machineConfig.Config.Guest.CPUs = 1
+	machineConfig.Config.Guest.CPUKind = "shared"
+	machineConfig.Config.Guest.MemoryMB = 256
+	machineConfig.Config.AutoDestroy = true
+
+	// Marshal JSON payload
+	payload, err := json.Marshal(machineConfig)
+	if err != nil {
+		slog.Error("Error creating machine config", "error", err)
+		return nil, err
+	}
+
+	// Create HTTP request
+	apiURL := fmt.Sprintf("https://api.machines.dev/v1/apps/%s/machines", os.Getenv("DIGGER_PROJECTS_SVC_APP_NAME"))
+	req2, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(payload))
+	if err != nil {
+		slog.Error("Error creating request", "error", err)
+		return nil, err
+	}
+
+	// Set headers
+	req2.Header.Set("Authorization", "Bearer "+os.Getenv("FLY_PROJECTS_SVC_API_TOKEN"))
+	req2.Header.Set("Content-Type", "application/json")
+
+	// Make HTTP request
+	client := &http.Client{}
+	resp, err := client.Do(req2)
+	if err != nil {
+		slog.Error("Error making request", "error", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, err2 := io.ReadAll(resp.Body)
+		slog.Error("Error triggering projects refresh service", "statusCode", resp.StatusCode, "body", body, "readyErr", err2)
+		return nil, fmt.Errorf("error triggering projects refresh service")
+	}
+
+	// Parse response
+	var machineResp MachineResponse
+	if err := json.NewDecoder(resp.Body).Decode(&machineResp); err != nil {
+		slog.Error("Error parsing response", "error", err)
+		return nil, err
+	}
+
+	slog.Debug("triggered projects refresh service", "machineId", machineResp.ID, "statusCode", resp.StatusCode)
+
+	return &MachineResponse{ID: machineResp.ID}, nil
+}

backend/service_clients/projects_service.go

@@ -3,8 +3,8 @@ package services
 import (
 	"fmt"
 	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/service_clients"
 	utils3 "github.com/diggerhq/digger/backend/utils"
-	dg_configuration "github.com/diggerhq/digger/libs/digger_config"
 	"log/slog"
 	"strconv"
 	"strings"
@@ -41,17 +41,12 @@ func LoadProjectsFromGithubRepo(gh utils3.GithubClientProvider, installationId s
 		return fmt.Errorf("error getting github service")
 	}
 
-	err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
-		config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
-		if err != nil {
-			slog.Error("failed to load digger.yml: %v", "error", err)
-			return fmt.Errorf("error loading digger.yml %v", err)
-		}
-		models.DB.RefreshProjectsFromRepo(strconv.Itoa(int(link.OrganisationId)), *config, repoFullName)
-		return nil
-	})
+	resp, err := service_clients.TriggerProjectsRefreshService(cloneUrl, branch, *token, repoFullName, strconv.Itoa(int(link.OrganisationId)))
 	if err != nil {
-		return fmt.Errorf("error while cloning repo: %v", err)
+		return fmt.Errorf("error triggering projects refresh service: %v", err)
+	}
+	if resp == nil {
+		return fmt.Errorf("error triggering projects refresh service: response nil")
 	}
 
 	return nil

backend/services/repos.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"fmt"
+	"github.com/diggerhq/digger/libs/git_utils"
 	"log/slog"
 	"net/http"
 	"os"
@@ -94,7 +95,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF
 		"changedFilesCount", len(changedFiles),
 	)
 
-	err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error {
+	err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "x-token-auth", func(dir string) error {
 		diggerYmlPath := path.Join(dir, "digger.yml")
 		diggerYmlBytes, err := os.ReadFile(diggerYmlPath)
 		if err != nil {

backend/utils/bitbucket.go

@@ -18,66 +18,6 @@ import (
 	"github.com/google/go-github/v61/github"
 )
 
-func createTempDir() string {
-	tempDir, err := os.MkdirTemp("", "repo")
-	if err != nil {
-		slog.Error("Failed to create temporary directory", "error", err)
-		panic(err)
-	}
-	return tempDir
-}
-
-type action func(string) error
-
-func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, tokenUsername string, action action) error {
-	dir := createTempDir()
-
-	slog.Debug("Cloning git repository",
-		"repoUrl", repoUrl,
-		"branch", branch,
-		"commitHash", commitHash,
-		"directory", dir,
-	)
-
-	git := NewGitShellWithTokenAuth(dir, token, tokenUsername)
-	err := git.Clone(repoUrl, branch)
-	if err != nil {
-		slog.Error("Failed to clone repository",
-			"repoUrl", repoUrl,
-			"branch", branch,
-			"error", err,
-		)
-		return err
-	}
-
-	if commitHash != "" {
-		err := git.Checkout(commitHash)
-		if err != nil {
-			slog.Error("Failed to checkout commit",
-				"commitHash", commitHash,
-				"error", err,
-			)
-			return err
-		}
-	}
-
-	defer func() {
-		slog.Debug("Removing cloned directory", "directory", dir)
-		ferr := os.RemoveAll(dir)
-		if ferr != nil {
-			slog.Warn("Failed to remove directory", "directory", dir, "error", ferr)
-		}
-	}()
-
-	err = action(dir)
-	if err != nil {
-		slog.Error("Error performing action on repository", "directory", dir, "error", err)
-		return err
-	}
-
-	return nil
-}
-
 // just a wrapper around github client to be able to use mocks
 type DiggerGithubRealClientProvider struct {
 }