feat: enable orchestrator gha setup via UI, feat: full docker compose, fix: handle drift job status processing in orchestrator (#2569)

* feat: docker compose for full hosting set up

* feat: use postgres for statesman in docker compose

* fix: sidecar health check endpoint

* chore: move E2B API key to .env file

* fix: update DIGGER_DRIFT_REPORTER_HOSTNAME

* Add all necessary env vars to docker compose file

* Fix: duplicatre drift state machine logic to orchestrator

* handle legacy webhook route

* add postgres for token service, add compose profiles

Author: sidpalas

Dockerfile_backend

@@ -24,6 +24,13 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
     go build -ldflags="-X 'main.Version=${COMMIT_SHA}'" -o backend_exe ./backend/
 
+# Build the projects refresh service binary and ship it in the image.
+# This is required for "project cache local exec" mode, which expects to exec
+# the refresh service locally within the container.
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go build -ldflags="-X 'main.Version=${COMMIT_SHA}'" -o /tmp/projects_refesh_main ./background/projects-refresh-service
+
 # Multi-stage build will just copy the binary to an alpine image.
 FROM ubuntu:24.04 as runner
 ENV ATLAS_VERSION v0.38.0
@@ -48,6 +55,7 @@ EXPOSE 3000
 
 # Copy the binary to the corresponding folder
 COPY --from=builder /go/src/github.com/diggerhq/digger/backend_exe /app/backend
+COPY --from=builder /tmp/projects_refesh_main /app/projects_refesh_main
 COPY --from=builder /go/src/github.com/diggerhq/digger/backend/scripts/entrypoint.sh /app/entrypoint.sh
 COPY --from=builder /go/src/github.com/diggerhq/digger/backend/migrations /app/migrations
 ADD backend/templates ./templates

backend/bootstrap/main.go

@@ -159,6 +159,9 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 		r.LoadHTMLGlob("templates/*.tmpl")
 	}
 
+	// Canonical GitHub App webhook endpoint.
+	r.POST("/github/webhook", diggerController.GithubAppWebHook)
+	// Legacy webhook path kept for backward compatibility.
 	r.POST("/github-app-webhook", diggerController.GithubAppWebHook)
 
 	tenantActionsGroup := r.Group("/api/tenants")
@@ -173,6 +176,20 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController
 	githubGroup.GET("/setup", controllers.GithubAppSetup)
 	githubGroup.GET("/exchange-code", diggerController.GithubSetupExchangeCode)
 
+	publicPrefix := utils.NormalizePublicPathPrefix(os.Getenv("DIGGER_PUBLIC_PATH_PREFIX"))
+	if publicPrefix != "" {
+		prefixed := r.Group(publicPrefix)
+		prefixed.POST("/github/webhook", diggerController.GithubAppWebHook)
+		prefixed.POST("/github-app-webhook", diggerController.GithubAppWebHook)
+
+		prefixedGithubGroup := prefixed.Group("/github")
+		prefixedGithubGroup.Use(middleware.GetWebMiddleware())
+		prefixed.GET("/github/callback", diggerController.GithubAppCallbackPage)
+		prefixedGithubGroup.GET("/repos", diggerController.GithubReposPage)
+		prefixedGithubGroup.GET("/setup", controllers.GithubAppSetup)
+		prefixedGithubGroup.GET("/exchange-code", diggerController.GithubSetupExchangeCode)
+	}
+
 	authorized := r.Group("/")
 	authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(models.CliJobAccessType, models.AccessPolicyType, models.AdminPolicyType))
 

backend/controllers/github_setup.go

@@ -37,17 +37,37 @@ func GithubAppSetup(c *gin.Context) {
 	}
 
 	host := os.Getenv("HOSTNAME")
+	// When the backend is deployed behind a reverse proxy (or behind the UI proxy),
+	// the inbound request Host/TLS reflects the internal hop, not the public origin.
+	// The GitHub App manifest flow requires public callback/webhook URLs, so we
+	// prefer X-Forwarded-Host/Proto when present to construct externally reachable
+	// URLs.
+	forwardedHost := c.Request.Header.Get("X-Forwarded-Host")
+	forwardedProto := c.Request.Header.Get("X-Forwarded-Proto")
+	if forwardedHost != "" {
+		if forwardedProto == "" {
+			forwardedProto = "https"
+		}
+		host = fmt.Sprintf("%s://%s", forwardedProto, forwardedHost)
+	} else if host == "" {
+		scheme := "http"
+		if c.Request.TLS != nil {
+			scheme = "https"
+		}
+		host = fmt.Sprintf("%s://%s", scheme, c.Request.Host)
+	}
+	publicPrefix := utils.NormalizePublicPathPrefix(os.Getenv("DIGGER_PUBLIC_PATH_PREFIX"))
 	manifest := &githubAppRequest{
 		Name:        fmt.Sprintf("Digger app %v", rand.Int31()),
 		Description: fmt.Sprintf("Digger hosted at %s", host),
 		URL:         host,
-		RedirectURL: fmt.Sprintf("%s/github/exchange-code", host),
+		RedirectURL: fmt.Sprintf("%s%s", host, utils.ApplyPublicPathPrefix(publicPrefix, "/github/exchange-code")),
 		Public:      false,
 		Webhook: &githubWebhook{
 			Active: true,
-			URL:    fmt.Sprintf("%s/github-app-webhook", host),
+			URL:    fmt.Sprintf("%s%s", host, utils.ApplyPublicPathPrefix(publicPrefix, "/github/webhook")),
 		},
-		CallbackUrls:          []string{fmt.Sprintf("%s/github/callback", host)},
+		CallbackUrls:          []string{fmt.Sprintf("%s%s", host, utils.ApplyPublicPathPrefix(publicPrefix, "/github/callback"))},
 		SetupOnUpdate:         true,
 		RequestOauthOnInstall: true,
 		Events: []string{

backend/controllers/projects.go

@@ -927,7 +927,40 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 
 		// store digger job summary
 		if request.JobSummary != nil {
-			models.DB.UpdateDiggerJobSummary(job.DiggerJobID, request.JobSummary.ResourcesCreated, request.JobSummary.ResourcesUpdated, request.JobSummary.ResourcesDeleted)
+			job, err = models.DB.UpdateDiggerJobSummary(job.DiggerJobID, request.JobSummary.ResourcesCreated, request.JobSummary.ResourcesUpdated, request.JobSummary.ResourcesDeleted)
+			if err != nil {
+				slog.Warn("Could not update digger job summary for drift handling",
+					"jobId", jobId,
+					"error", err,
+				)
+			}
+
+			isDriftJob, err := IsDriftStatusJob(job)
+			if err != nil {
+				slog.Warn("Could not determine if job is a drift job",
+					"jobId", jobId,
+					"error", err,
+				)
+			} else if isDriftJob {
+				project, err := models.DB.GetProjectByName(orgId, job.Batch.RepoFullName, job.ProjectName)
+				if err != nil {
+					slog.Warn("Could not load project for drift state update",
+						"jobId", jobId,
+						"projectName", job.ProjectName,
+						"repoFullName", job.Batch.RepoFullName,
+						"error", err,
+					)
+				} else {
+					err = ProjectDriftStateMachineApply(*project, job.TerraformOutput, request.JobSummary.ResourcesCreated, request.JobSummary.ResourcesUpdated, request.JobSummary.ResourcesDeleted)
+					if err != nil {
+						slog.Warn("Could not update project drift state",
+							"jobId", jobId,
+							"projectName", job.ProjectName,
+							"error", err,
+						)
+					}
+				}
+			}
 		}
 
 		// Update PR comment with real-time status for succeeded job

backend/controllers/projects_helpers.go

@@ -5,15 +5,66 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/utils"
 	"github.com/diggerhq/digger/libs/ci/github"
 	"github.com/diggerhq/digger/libs/digger_config"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/scheduler"
+	"github.com/diggerhq/digger/libs/spec"
 )
 
+func IsDriftStatusJob(job *models.DiggerJob) (bool, error) {
+	if job == nil || job.Batch == nil {
+		return false, nil
+	}
+
+	if job.Batch.BatchType != orchestrator_scheduler.DiggerCommandPlan || job.Batch.PrNumber != 0 {
+		return false, nil
+	}
+
+	var vcsSpec spec.VcsSpec
+	err := json.Unmarshal(job.SerializedVcsSpec, &vcsSpec)
+	if err != nil {
+		return false, err
+	}
+
+	return strings.EqualFold(vcsSpec.VcsType, "noop"), nil
+}
+
+func ProjectDriftStateMachineApply(project models.Project, tfplan string, resourcesCreated uint, resourcesUpdated uint, resourcesDeleted uint) error {
+	isEmptyPlan := resourcesCreated == 0 && resourcesUpdated == 0 && resourcesDeleted == 0
+	wasEmptyPlan := project.DriftToCreate == 0 && project.DriftToUpdate == 0 && project.DriftToDelete == 0
+	if isEmptyPlan {
+		project.DriftStatus = models.DriftStatusNoDrift
+	}
+	if !isEmptyPlan && wasEmptyPlan {
+		project.DriftStatus = models.DriftStatusNewDrift
+	}
+	if !isEmptyPlan && !wasEmptyPlan {
+		if project.DriftTerraformPlan != tfplan {
+			if project.DriftStatus == models.DriftStatusAcknowledgeDrift {
+				project.DriftStatus = models.DriftStatusNewDrift
+			}
+		}
+	}
+
+	project.DriftTerraformPlan = tfplan
+	project.DriftToCreate = resourcesCreated
+	project.DriftToUpdate = resourcesUpdated
+	project.DriftToDelete = resourcesDeleted
+	project.LatestDriftCheck = time.Now()
+	result := models.DB.GormDB.Save(&project)
+	if result.Error != nil {
+		return result.Error
+	}
+
+	return nil
+}
+
 func GenerateChecksSummaryForBatch(batch *models.DiggerBatch) (string, error) {
 	summaryEndpoint := os.Getenv("DIGGER_AI_SUMMARY_ENDPOINT")
 	if summaryEndpoint == "" {

backend/utils/public_path_prefix.go

@@ -0,0 +1,32 @@
+package utils
+
+import "strings"
+
+// NormalizePublicPathPrefix normalizes a configured URL path prefix (e.g. "/orchestrator")
+// so it can be safely concatenated with other paths.
+//
+// Examples:
+// - "" or "/" -> ""
+// - "orchestrator" -> "/orchestrator"
+// - "/orchestrator/" -> "/orchestrator"
+func NormalizePublicPathPrefix(raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || raw == "/" {
+		return ""
+	}
+	if !strings.HasPrefix(raw, "/") {
+		raw = "/" + raw
+	}
+	return strings.TrimRight(raw, "/")
+}
+
+// ApplyPublicPathPrefix concatenates prefix + p ensuring p starts with "/".
+func ApplyPublicPathPrefix(prefix, p string) string {
+	if p == "" {
+		return prefix
+	}
+	if !strings.HasPrefix(p, "/") {
+		p = "/" + p
+	}
+	return prefix + p
+}