feat: implement changed file detection for stack deployment (#491)

* feat: implement changed file detection for stack deployment

* refactor: rename HasSubdirChangedBetweenCommits to HasChangesInSubdir for clarity

* test: add unit test for changed file detection between commits

* refactor: rename variable for clarity in changed file detection

* fix: resolve issue with empty changedFiles return in changed file detection

* refactor: rename functions for clarity in file change detection

* feat: add configuration and secret files for deployment

* feat: add tests for detecting changes in configs, secrets, and bind mounts

* refactor: streamline error logging in compose file loading and change detection

* chore: update import statement for go-git in compose_test.go

* feat: enhance config change detection tests with parameterized test cases

Author: kimdre

cmd/doco-cd/http_handler.go

@@ -336,16 +336,24 @@ func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter
 				return
 			}
 
+			var changedFiles []git.ChangedFile
 			if deployedCommit != "" {
-				changed, err := git.CompareCommitsInSubdir(repo, plumbing.NewHash(deployedCommit), plumbing.NewHash(latestCommit), deployConfig.WorkingDirectory)
+				changedFiles, err = git.GetChangedFilesBetweenCommits(repo, plumbing.NewHash(deployedCommit), plumbing.NewHash(latestCommit))
+				if err != nil {
+					onError(repoName, w, subJobLog.With(logger.ErrAttr(err)), "failed to get changed files between commits", err.Error(), jobID, http.StatusInternalServerError)
+
+					return
+				}
+
+				hasChanged, err := git.HasChangesInSubdir(changedFiles, deployConfig.WorkingDirectory)
 				if err != nil {
 					onError(repoName, w, subJobLog, fmt.Errorf("failed to compare commits in subdirectory: %w", err).Error(),
 						map[string]string{"stack": deployConfig.Name}, jobID, http.StatusInternalServerError)
 
 					return
 				}
 
-				if !changed {
+				if !hasChanged {
 					jobLog.Debug("no changes detected in subdirectory, skipping deployment",
 						slog.String("directory", deployConfig.WorkingDirectory),
 						slog.String("last_commit", latestCommit),
@@ -360,7 +368,8 @@ func HandleEvent(ctx context.Context, jobLog *slog.Logger, w http.ResponseWriter
 				}
 			}
 
-			err = docker.DeployStack(subJobLog, internalRepoPath, externalRepoPath, &ctx, &dockerCli, &payload, deployConfig, latestCommit, Version, false)
+			err = docker.DeployStack(subJobLog, internalRepoPath, externalRepoPath, &ctx, &dockerCli, &payload,
+				deployConfig, changedFiles, latestCommit, Version, false)
 			if err != nil {
 				onError(repoName, w, subJobLog.With(logger.ErrAttr(err)), "deployment failed", err.Error(), jobID, http.StatusInternalServerError)
 

cmd/doco-cd/poll_handler.go

@@ -382,15 +382,23 @@ func RunPoll(ctx context.Context, pollConfig config.PollConfig, appConfig *confi
 				continue
 			}
 
+			var changedFiles []git.ChangedFile
 			if deployedCommit != "" {
-				changed, err := git.CompareCommitsInSubdir(repo, plumbing.NewHash(deployedCommit), plumbing.NewHash(latestCommit), deployConfig.WorkingDirectory)
+				changedFiles, err = git.GetChangedFilesBetweenCommits(repo, plumbing.NewHash(deployedCommit), plumbing.NewHash(latestCommit))
+				if err != nil {
+					subJobLog.Error("failed to get changed files between commits", log.ErrAttr(err))
+
+					return fmt.Errorf("failed to get changed files between commits: %w", err)
+				}
+
+				hasChanged, err := git.HasChangesInSubdir(changedFiles, deployConfig.WorkingDirectory)
 				if err != nil {
 					subJobLog.Error("failed to compare commits in subdirectory", log.ErrAttr(err))
 
 					return fmt.Errorf("failed to compare commits in subdirectory: %w", err)
 				}
 
-				if !changed {
+				if !hasChanged {
 					jobLog.Debug("no changes detected in subdirectory, skipping deployment",
 						slog.String("directory", deployConfig.WorkingDirectory),
 						slog.String("last_commit", latestCommit),
@@ -415,7 +423,8 @@ func RunPoll(ctx context.Context, pollConfig config.PollConfig, appConfig *confi
 				Private:   pollConfig.Private,
 			}
 
-			err = docker.DeployStack(subJobLog, internalRepoPath, externalRepoPath, &ctx, &dockerCli, &payload, deployConfig, latestCommit, Version, true)
+			err = docker.DeployStack(subJobLog, internalRepoPath, externalRepoPath, &ctx, &dockerCli, &payload,
+				deployConfig, changedFiles, latestCommit, Version, false)
 			if err != nil {
 				subJobLog.Error("failed to deploy stack", log.ErrAttr(err))
 

dev.compose.yaml

@@ -23,7 +23,7 @@ services:
     environment:
       TZ: Europe/Berlin
       HTTP_PORT: 80
-      LOG_LEVEL: info
+      LOG_LEVEL: debug
       # DOCKER_API_VERSION: 1.47
       <<: *poll-config
       #POLL_CONFIG_FILE: /poll.yml
@@ -61,7 +61,7 @@ configs:
   tinyproxy.conf:
     content: |
       # https://tinyproxy.github.io/
-      LogLevel Connect
+      LogLevel Warning
       Port 8888
       Timeout 600
       BasicAuth username password

internal/docker/compose.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 	"time"
 
+	gitInternal "github.com/kimdre/doco-cd/internal/git"
+
 	"github.com/compose-spec/compose-go/v2/cli"
 	"github.com/compose-spec/compose-go/v2/types"
 	"github.com/docker/cli/cli/command"
@@ -210,7 +212,6 @@ func LoadCompose(ctx context.Context, workingDir, projectName string, composeFil
 		cli.WithWorkingDirectory(workingDir),
 		cli.WithInterpolation(true),
 		cli.WithResolvedPaths(true),
-		cli.WithDotEnv,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create project options: %w", err)
@@ -323,7 +324,7 @@ func DeployCompose(ctx context.Context, dockerCli command.Cli, project *types.Pr
 func DeployStack(
 	jobLog *slog.Logger, internalRepoPath, externalRepoPath string, ctx *context.Context,
 	dockerCli *command.Cli, payload *webhook.ParsedPayload, deployConfig *config.DeployConfig,
-	latestCommit, appVersion string, forceDeploy bool,
+	changedFiles []gitInternal.ChangedFile, latestCommit, appVersion string, forceDeploy bool,
 ) error {
 	startTime := time.Now()
 
@@ -437,8 +438,6 @@ func DeployStack(
 			}
 
 			stackLog.Debug("file decrypted successfully", slog.String("file", path))
-		} else {
-			stackLog.Debug("file is not encrypted", slog.String("file", path))
 		}
 
 		return nil
@@ -450,13 +449,25 @@ func DeployStack(
 	project, err := LoadCompose(*ctx, externalWorkingDir, deployConfig.Name, deployConfig.ComposeFiles)
 	if err != nil {
 		errMsg := "failed to load compose config"
-		stackLog.Error(errMsg,
-			logger.ErrAttr(err),
-			slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles)))
+		stackLog.Error(errMsg, logger.ErrAttr(err), slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles)))
+
+		return fmt.Errorf("%s: %w", errMsg, err)
+	}
+
+	hasChanges, err := MountedFilesHaveChanges(changedFiles, project)
+	if err != nil {
+		errMsg := "failed to check for changed project files"
+		stackLog.Error(errMsg, logger.ErrAttr(err), slog.Group("compose_files", slog.Any("files", deployConfig.ComposeFiles)))
 
 		return fmt.Errorf("%s: %w", errMsg, err)
 	}
 
+	if hasChanges {
+		stackLog.Info("mounted files have changed, forcing recreation of the stack")
+
+		deployConfig.ForceRecreate = true
+	}
+
 	stackLog.Info("deploying stack")
 
 	done := make(chan struct{})
@@ -527,3 +538,168 @@ func DestroyStack(
 
 	return nil
 }
+
+// HasChangedConfigs checks if any files used in docker compose `configs:` definitions have changed using the Git status.
+func HasChangedConfigs(changedFiles []gitInternal.ChangedFile, project *types.Project) (bool, error) {
+	for _, c := range project.Configs {
+		configPath := c.File
+		if configPath == "" {
+			continue
+		}
+
+		if !path.IsAbs(configPath) {
+			configPath = filepath.Join(project.WorkingDir, configPath)
+		}
+
+		for _, f := range changedFiles {
+			var paths []string
+
+			if f.From != nil {
+				fromPath := f.From.Path()
+				if !path.IsAbs(fromPath) {
+					fromPath = filepath.Join(project.WorkingDir, fromPath)
+				}
+
+				paths = append(paths, fromPath)
+			}
+
+			if f.To != nil {
+				toPath := f.To.Path()
+				if !path.IsAbs(toPath) {
+					toPath = filepath.Join(project.WorkingDir, toPath)
+				}
+
+				paths = append(paths, toPath)
+			}
+
+			for _, p := range paths {
+				if p == configPath {
+					return true, nil
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+// HasChangedSecrets checks if any files used in docker compose `secrets:` definitions have changed using the Git status.
+func HasChangedSecrets(changedFiles []gitInternal.ChangedFile, project *types.Project) (bool, error) {
+	for _, s := range project.Secrets {
+		secretPath := s.File
+		if secretPath == "" {
+			continue
+		}
+
+		if !path.IsAbs(secretPath) {
+			secretPath = filepath.Join(project.WorkingDir, secretPath)
+		}
+
+		for _, f := range changedFiles {
+			var paths []string
+
+			if f.From != nil {
+				fromPath := f.From.Path()
+				if !path.IsAbs(fromPath) {
+					fromPath = filepath.Join(project.WorkingDir, fromPath)
+				}
+
+				paths = append(paths, fromPath)
+			}
+
+			if f.To != nil {
+				toPath := f.To.Path()
+				if !path.IsAbs(toPath) {
+					toPath = filepath.Join(project.WorkingDir, toPath)
+				}
+
+				paths = append(paths, toPath)
+			}
+
+			for _, p := range paths {
+				if p == secretPath {
+					return true, nil
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+// HasChangedBindMounts checks if any files used in docker compose `volumes:` definitions with type `bind` have changed using the Git status.
+func HasChangedBindMounts(changedFiles []gitInternal.ChangedFile, project *types.Project) (bool, error) {
+	for _, s := range project.Services {
+		for _, v := range s.Volumes {
+			if v.Type == "bind" && v.Source != "" {
+				bindPath := v.Source
+				if !path.IsAbs(bindPath) {
+					bindPath = filepath.Join(project.WorkingDir, bindPath)
+				}
+
+				for _, f := range changedFiles {
+					var paths []string
+
+					if f.From != nil {
+						fromPath := f.From.Path()
+						if !path.IsAbs(fromPath) {
+							fromPath = filepath.Join(project.WorkingDir, fromPath)
+						}
+
+						paths = append(paths, fromPath)
+					}
+
+					if f.To != nil {
+						toPath := f.To.Path()
+						if !path.IsAbs(toPath) {
+							toPath = filepath.Join(project.WorkingDir, toPath)
+						}
+
+						paths = append(paths, toPath)
+					}
+
+					for _, p := range paths {
+						// Check if bindPath is in the changed file path
+						if strings.HasPrefix(p, bindPath) {
+							return true, nil
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return false, nil
+}
+
+// MountedFilesHaveChanges checks if any files from config, secret or bind mounts have changed in the project.
+func MountedFilesHaveChanges(changedFiles []gitInternal.ChangedFile, project *types.Project) (bool, error) {
+	changedConfigs, err := HasChangedConfigs(changedFiles, project)
+	if err != nil {
+		return false, fmt.Errorf("failed to check changed configs: %w", err)
+	}
+
+	if changedConfigs {
+		return true, nil
+	}
+
+	changedSecrets, err := HasChangedSecrets(changedFiles, project)
+	if err != nil {
+		return false, fmt.Errorf("failed to check changed secrets: %w", err)
+	}
+
+	if changedSecrets {
+		return true, nil
+	}
+
+	changedBindMounts, err := HasChangedBindMounts(changedFiles, project)
+	if err != nil {
+		return false, fmt.Errorf("failed to check changed bind mounts: %w", err)
+	}
+
+	if changedBindMounts {
+		return true, nil
+	}
+
+	return false, nil
+}