iterative caching for terragrunt configs (#2051)

* add iterative caching support

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

* Update backend/controllers/github.go

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

* updated docs

* Update backend/controllers/cache.go

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

* fix log msg

* fix io

* Update backend/controllers/cache.go

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

* Update libs/digger_config/digger_config.go

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

---------

Co-authored-by: bismuthdev[bot] <177057995+bismuthdev[bot]@users.noreply.github.com>

Author: motatoes

backend/controllers/cache.go

@@ -1,10 +1,15 @@
 package controllers
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
 	"github.com/diggerhq/digger/libs/git_utils"
+	"io"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strings"
@@ -15,13 +20,14 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+type UpdateCacheRequest struct {
+	RepoFullName   string `json:"repo_full_name"`
+	Branch         string `json:"branch"`
+	OrgId          uint   `json:"org_id"`
+	InstallationId int64  `json:"installation_id"`
+}
+
 func (d DiggerController) UpdateRepoCache(c *gin.Context) {
-	type UpdateCacheRequest struct {
-		RepoFullName   string `json:"repo_full_name"`
-		Branch         string `json:"branch"`
-		OrgId          uint   `json:"org_id"`
-		InstallationId int64  `json:"installation_id"`
-	}
 
 	var request UpdateCacheRequest
 	err := c.BindJSON(&request)
@@ -65,13 +71,14 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 
 	var diggerYmlStr string
 	var config *dg_configuration.DiggerConfig
+	var newAtlantisConfig *tac.AtlantisConfig
 
 	// update the cache here, do it async for immediate response
 	go func() {
 		err = git_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, "", func(dir string) error {
 			diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
 			diggerYmlStr = string(diggerYmlBytes)
-			config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)
+			config, _, _, newAtlantisConfig, err = dg_configuration.LoadDiggerConfig(dir, true, nil, nil)
 			if err != nil {
 				slog.Error("Error loading digger config", "error", err)
 				return err
@@ -83,7 +90,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 			slog.Error("Could not load digger config", "error", err)
 			return
 		}
-		_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config)
+		_, err = models.DB.UpsertRepoCache(orgId, repoFullName, diggerYmlStr, *config, newAtlantisConfig)
 		if err != nil {
 			slog.Error("Could not update repo cache", "error", err)
 			return
@@ -93,3 +100,72 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
 
 	c.String(http.StatusOK, "successfully submitted cache for processing, check backend logs for progress")
 }
+
+func sendProcessCacheRequest(repoFullName string, branch string, installationId int64) error {
+	diggerHostname := os.Getenv("HOSTNAME")
+	webhookSecret := os.Getenv("DIGGER_INTERNAL_SECRET")
+
+	installationLink, err := models.DB.GetGithubInstallationLinkForInstallationId(installationId)
+	if err != nil {
+		slog.Error("Error getting installation link", "installationId", installationId, "error", err)
+		return err
+	}
+
+	orgId := installationLink.OrganisationId
+
+	payload := UpdateCacheRequest{
+		RepoFullName:   repoFullName,
+		Branch:         branch,
+		InstallationId: installationId,
+		OrgId:          orgId,
+	}
+
+	cacheRefreshUrl, err := url.JoinPath(diggerHostname, "_internal/update_repo_cache")
+	if err != nil {
+		slog.Error("Error joining URL paths", "error", err)
+		return err
+	}
+
+	jsonPayload, err := json.Marshal(payload)
+	if err != nil {
+		slog.Error("Process Cache: error marshaling JSON", "error", err)
+		return err
+	}
+
+	req, err := http.NewRequest("POST", cacheRefreshUrl, bytes.NewBuffer(jsonPayload))
+	if err != nil {
+		slog.Error("Process Cache: Error creating request", "error", err)
+		return err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", webhookSecret))
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println("Error sending request:", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	statusCode := resp.StatusCode
+	if statusCode != 200 {
+		// Read response body to get error details
+		responseBody, err := io.ReadAll(resp.Body)
+		if err != nil {
+			slog.Error("Failed to read error response body", "error", err)
+		}
+
+		slog.Error("got unexpected cache status",
+			"statusCode", statusCode,
+			"repoFullName", repoFullName,
+			"orgId", orgId,
+			"branch", branch,
+			"installationId", installationId,
+			"responseBody", string(responseBody))
+
+		return fmt.Errorf("cache update failed with status code %d: %s", statusCode, string(responseBody))
+	}
+	return nil
+}

backend/controllers/github.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
 	"github.com/diggerhq/digger/libs/git_utils"
 	"log/slog"
 	"math/rand"
@@ -416,7 +417,6 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent, a
 	loadProjectsOnPush := os.Getenv("DIGGER_LOAD_PROJECTS_ON_PUSH")
 
 	if loadProjectsOnPush == "true" {
-
 		if strings.HasSuffix(ref, defaultBranch) {
 			slog.Debug("Loading projects from GitHub repo (push event)", "loadProjectsOnPush", loadProjectsOnPush, "ref", ref, "defaultBranch", defaultBranch)
 			err := services.LoadProjectsFromGithubRepo(gh, strconv.FormatInt(installationId, 10), repoFullName, repoOwner, repoName, cloneURL, defaultBranch)
@@ -428,6 +428,15 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent, a
 		slog.Debug("Skipping loading projects from GitHub repo", "loadProjectsOnPush", loadProjectsOnPush)
 	}
 
+	repoCacheEnabled := os.Getenv("DIGGER_CONFIG_REPO_CACHE_ENABLED")
+	if repoCacheEnabled == "1" && strings.HasSuffix(ref, defaultBranch) {
+		go func() {
+			if err := sendProcessCacheRequest(repoFullName, defaultBranch, installationId); err != nil {
+				slog.Error("Failed to process cache request", "error", err, "repoFullName", repoFullName)
+			}
+		}()
+	}
+
 	return nil
 }
 
@@ -914,7 +923,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	return nil
 }
 
-func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
+func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, branch string, changedFiles []string, taConfig *tac.AtlantisConfig) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], error) {
 	slog.Info("Getting Digger config for branch",
 		slog.Group("repository",
 			slog.String("fullName", repoFullName),
@@ -954,7 +963,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 
 		slog.Debug("Successfully read digger.yml file", "configLength", len(diggerYmlStr))
 
-		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
+		config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, taConfig)
 		if err != nil {
 			slog.Error("Error loading and parsing Digger config",
 				"directory", dir,
@@ -1046,6 +1055,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
 
 	// check if items should be loaded from cache
 	useCache := false
+	var taConfig *tac.AtlantisConfig = nil
 	if val, _ := os.LookupEnv("DIGGER_CONFIG_REPO_CACHE_ENABLED"); val == "1" && !slices.Contains(prLabels, "digger:no-cache") {
 		useCache = true
 		slog.Info("Attempting to load config from cache",
@@ -1054,7 +1064,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
 			"prNumber", prNumber,
 		)
 
-		diggerYmlStr, config, dependencyGraph, err := retrieveConfigFromCache(orgId, repoFullName)
+		_, _, _, taConfigTemp, err := retrieveConfigFromCache(orgId, repoFullName)
 		if err != nil {
 			slog.Info("Could not load from cache, falling back to live loading",
 				"orgId", orgId,
@@ -1065,9 +1075,8 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
 			slog.Info("Successfully loaded config from cache",
 				"orgId", orgId,
 				"repoFullName", repoFullName,
-				"projectCount", len(config.Projects),
 			)
-			return diggerYmlStr, ghService, config, *dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil
+			taConfig = taConfigTemp
 		}
 	}
 
@@ -1084,7 +1093,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
 		"prNumber", prNumber,
 	)
 
-	diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles)
+	diggerYmlStr, ghService, config, dependencyGraph, err := GetDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch, changedFiles, taConfig)
 	if err != nil {
 		slog.Error("Error loading Digger config from repository",
 			"prNumber", prNumber,
@@ -1098,7 +1107,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels []
 	return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil
 }
 
-func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], error) {
+func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_configuration.DiggerConfig, *graph.Graph[string, dg_configuration.Project], *tac.AtlantisConfig, error) {
 	slog.Debug("Retrieving config from cache",
 		"orgId", orgId,
 		"repoFullName", repoFullName,
@@ -1111,7 +1120,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
 			"repoFullName", repoFullName,
 			"error", err,
 		)
-		return "", nil, nil, fmt.Errorf("failed to load repo cache: %v", err)
+		return "", nil, nil, nil, fmt.Errorf("failed to load repo cache: %v", err)
 	}
 
 	var config dg_configuration.DiggerConfig
@@ -1122,7 +1131,18 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
 			"repoFullName", repoFullName,
 			"error", err,
 		)
-		return "", nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
+		return "", nil, nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
+	}
+
+	var taConfig tac.AtlantisConfig
+	err = json.Unmarshal(repoCache.TerragruntAtlantisConfig, &taConfig)
+	if err != nil {
+		slog.Error("Failed to unmarshal config from cache",
+			"orgId", orgId,
+			"repoFullName", repoFullName,
+			"error", err,
+		)
+		return "", nil, nil, nil, fmt.Errorf("failed to unmarshal config from cache: %v", err)
 	}
 
 	slog.Debug("Creating project dependency graph from cached config",
@@ -1138,7 +1158,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
 			"repoFullName", repoFullName,
 			"error", err,
 		)
-		return "", nil, nil, fmt.Errorf("error creating dependency graph from cached config: %v", err)
+		return "", nil, nil, nil, fmt.Errorf("error creating dependency graph from cached config: %v", err)
 	}
 
 	slog.Info("Successfully retrieved config from cache",
@@ -1147,7 +1167,7 @@ func retrieveConfigFromCache(orgId uint, repoFullName string) (string, *dg_confi
 		"projectCount", len(config.Projects),
 	)
 
-	return repoCache.DiggerYmlStr, &config, &projectsGraph, nil
+	return repoCache.DiggerYmlStr, &config, &projectsGraph, &taConfig, nil
 }
 
 func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {

backend/migrations/20250725041417.sql

@@ -0,0 +1,2 @@
+-- Modify "repo_caches" table
+ALTER TABLE "public"."repo_caches" ADD COLUMN "terragrunt_atlantis_config" bytea NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:pk1704W4CHtf2K+qFKIkjTh8pN2R8Lor32ecJPtQgoY=
+h1:DO/UG0tYCnZA69yRLMd2ZBKUORlJtZJLkVCiCogKp+k=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -60,3 +60,4 @@ h1:pk1704W4CHtf2K+qFKIkjTh8pN2R8Lor32ecJPtQgoY=
 20250716043004.sql h1:WFz35jhsJ1pswpt5mmewW7L8LP5taRcNMf+pqUi3Yxs=
 20250716222603.sql h1:a66teR/6BQwNwhJ/zgNA/NjUVTz5DcoQl7cDZvzVS8c=
 20250717032021.sql h1:HaIhNsz3C+c87CDmFjgFlc9zGqoE5BU4m0dofDpeDYk=
+20250725041417.sql h1:Dds6fqS415FD1jlsoVEahcEEm1px3EHV5435moL+Vp8=

backend/models/cache.go

@@ -7,8 +7,9 @@ import (
 // storing repo cache such as digger.yml configuration
 type RepoCache struct {
 	gorm.Model
-	OrgId        uint
-	RepoFullName string
-	DiggerYmlStr string
-	DiggerConfig []byte `gorm:"type:bytea"`
+	OrgId                    uint
+	RepoFullName             string
+	DiggerYmlStr             string
+	DiggerConfig             []byte `gorm:"type:bytea"`
+	TerragruntAtlantisConfig []byte `gorm:"type:bytea"`
 }

backend/models/storage.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/diggerhq/digger/libs/digger_config/terragrunt/tac"
 	"log/slog"
 	"math"
 	"net/http"
@@ -1746,7 +1747,7 @@ func (db *Database) GetDiggerLock(resource string) (*DiggerLock, error) {
 	return lock, nil
 }
 
-func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlStr string, diggerConfig configuration.DiggerConfig) (*RepoCache, error) {
+func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlStr string, diggerConfig configuration.DiggerConfig, newAtlantisConfig *tac.AtlantisConfig) (*RepoCache, error) {
 	var repoCache RepoCache
 
 	configMarshalled, err := json.Marshal(diggerConfig)
@@ -1758,6 +1759,15 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
 		return nil, fmt.Errorf("could not marshal config: %v", err)
 	}
 
+	atlantisConfigMarshalled, err := json.Marshal(newAtlantisConfig)
+	if err != nil {
+		slog.Error("could not marshal terragrunt-atlantis-config",
+			"repoFullName", repoFullName,
+			"orgId", orgId,
+			"error", err)
+		return nil, fmt.Errorf("could not marshal config: %v", err)
+	}
+
 	// check if repo exist already, do nothing in this case
 	result := db.GormDB.Where("org_id = ? AND repo_full_name=?", orgId, repoFullName).Find(&repoCache)
 	if result.Error != nil {
@@ -1777,6 +1787,7 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
 
 		repoCache.DiggerConfig = configMarshalled
 		repoCache.DiggerYmlStr = diggerYmlStr
+		repoCache.TerragruntAtlantisConfig = atlantisConfigMarshalled
 		result = db.GormDB.Save(&repoCache)
 	} else {
 		// create record here
@@ -1785,10 +1796,11 @@ func (db *Database) UpsertRepoCache(orgId uint, repoFullName string, diggerYmlSt
 			"orgId", orgId)
 
 		repoCache = RepoCache{
-			OrgId:        orgId,
-			RepoFullName: repoFullName,
-			DiggerYmlStr: diggerYmlStr,
-			DiggerConfig: configMarshalled,
+			OrgId:                    orgId,
+			RepoFullName:             repoFullName,
+			DiggerYmlStr:             diggerYmlStr,
+			DiggerConfig:             configMarshalled,
+			TerragruntAtlantisConfig: atlantisConfigMarshalled,
 		}
 		result = db.GormDB.Save(&repoCache)
 		if result.Error != nil {

backend/utils/bitbucket.go

@@ -107,7 +107,7 @@ func GetDiggerConfigForBitbucketBranch(bb BitbucketProvider, token string, repoF
 		}
 
 		diggerYmlStr = string(diggerYmlBytes)
-		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
+		config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, nil)
 		if err != nil {
 			slog.Error("Error loading Digger config",
 				"repoFullName", repoFullName,

backend/utils/gitlab.go

@@ -131,7 +131,7 @@ func GetDiggerConfigForBranchGitlab(gh GitlabProvider, projectId int, repoFullNa
 			"configLength", len(diggerYmlStr),
 		)
 
-		config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)
+		config, _, dependencyGraph, _, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles, nil)
 		if err != nil {
 			slog.Error("Failed to load Digger config",
 				"projectId", projectId,

background/projects-refresh-service/projects_refesh_main.go

@@ -46,7 +46,7 @@ func main() {
 
 	slog.Info("refreshing projects from repo", "repoFullName", repoFullName)
 	err := utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", token, "", func(dir string) error {
-		config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
+		config, _, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil, nil)
 		if err != nil {
 			slog.Error("failed to load digger.yml: %v", "error", err)
 			return fmt.Errorf("error loading digger.yml %v", err)

cli/pkg/github/github.go

@@ -107,7 +107,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 		usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get current dir. %s", err), 4)
 	}
 
-	diggerConfig, diggerConfigYaml, dependencyGraph, err := digger_config.LoadDiggerConfig("./", true, nil)
+	diggerConfig, diggerConfigYaml, dependencyGraph, _, err := digger_config.LoadDiggerConfig("./", true, nil, nil)
 	if err != nil {
 		usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to read Digger digger_config. %s", err), 4)
 	}