WIP: support single comment summary with orchestrator mode

backend/controllers/github.go

@@ -499,7 +499,15 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		log.Printf("strconv.ParseInt error: %v", err)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not handle commentId: %v", err))
 	}
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggerYmlStr, 0)
+
+	placeholderComment, err := ghService.PublishComment(prNumber, "digger report placehoder")
+	if err != nil {
+		log.Printf("strconv.ParseInt error: %v", err)
+		commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not create placeholder commentId for report: %v", err))
+		return fmt.Errorf("comment reporter error: %v", err)
+	}
+
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGithub, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, &placeholderComment.Id, diggerYmlStr, 0)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -874,7 +882,14 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return fmt.Errorf("comment reporter error: %v", err)
 	}
 
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, diggerYmlStr, 0)
+	placeholderComment, err := ghService.PublishComment(issueNumber, "digger report placehoder")
+	if err != nil {
+		log.Printf("strconv.ParseInt error: %v", err)
+		commentReporterManager.UpdateComment(fmt.Sprintf(":x: could not create placeholder commentId for report: %v", err))
+		return fmt.Errorf("comment reporter error: %v", err)
+	}
+
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, reporterCommentId, &placeholderComment.Id, diggerYmlStr, 0)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

backend/migrations/20241204194016.sql

@@ -0,0 +1,2 @@
+-- Modify "digger_batches" table
+ALTER TABLE "public"."digger_batches" ADD COLUMN "placeholder_comment_id_for_report" text NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:fW4YtFhVfirvwGtHxnGXjFwChYhoj/5pW/XaA6rViHw=
+h1:SB1sRi1zY0IoKioJ+Y0FtuIjHQv6XCvcsAJg0AEJNs4=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -34,3 +34,4 @@ h1:fW4YtFhVfirvwGtHxnGXjFwChYhoj/5pW/XaA6rViHw=
 20241107163722.sql h1:tk28AgXggvpEigTkWMYMxIVDPNdEUFGijaFWBqvlZhA=
 20241107172343.sql h1:wtM1+uJZY6NiiDYabuzj/LAANAV7+xyUCL5U23v3e+c=
 20241114202249.sql h1:LCRf42VUYB/uugR4gwp+fGPUomT0KhFA3eLjBK9JlZo=
+20241204194016.sql h1:RgiJkZlAJSJjNCRD5RHmRr+65O9UDCRwGohQwdIMhGg=

backend/models/scheduler.go

@@ -22,19 +22,20 @@ const DiggerVCSGithub DiggerVCSType = "github"
 const DiggerVCSGitlab DiggerVCSType = "gitlab"
 
 type DiggerBatch struct {
-	ID                   uuid.UUID `gorm:"primary_key"`
-	VCS                  DiggerVCSType
-	PrNumber             int
-	CommentId            *int64
-	Status               orchestrator_scheduler.DiggerBatchStatus
-	BranchName           string
-	DiggerConfig         string
-	GithubInstallationId int64
-	GitlabProjectId      int
-	RepoFullName         string
-	RepoOwner            string
-	RepoName             string
-	BatchType            orchestrator_scheduler.DiggerCommand
+	ID                            uuid.UUID `gorm:"primary_key"`
+	VCS                           DiggerVCSType
+	PrNumber                      int
+	CommentId                     *int64
+	PlaceholderCommentIdForReport *string
+	Status                        orchestrator_scheduler.DiggerBatchStatus
+	BranchName                    string
+	DiggerConfig                  string
+	GithubInstallationId          int64
+	GitlabProjectId               int
+	RepoFullName                  string
+	RepoOwner                     string
+	RepoName                      string
+	BatchType                     orchestrator_scheduler.DiggerCommand
 	// used for module source grouping comments
 	SourceDetails []byte
 }

backend/models/storage.go

@@ -617,22 +617,23 @@ func (db *Database) GetDiggerBatch(batchId *uuid.UUID) (*DiggerBatch, error) {
 	return batch, nil
 }
 
-func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, gitlabProjectId int) (*DiggerBatch, error) {
+func (db *Database) CreateDiggerBatch(vcsType DiggerVCSType, githubInstallationId int64, repoOwner string, repoName string, repoFullname string, PRNumber int, diggerConfig string, branchName string, batchType scheduler.DiggerCommand, commentId *int64, placeholderCommentIdForReport *string, gitlabProjectId int) (*DiggerBatch, error) {
 	uid := uuid.New()
 	batch := &DiggerBatch{
-		ID:                   uid,
-		VCS:                  vcsType,
-		GithubInstallationId: githubInstallationId,
-		RepoOwner:            repoOwner,
-		RepoName:             repoName,
-		RepoFullName:         repoFullname,
-		PrNumber:             PRNumber,
-		CommentId:            commentId,
-		Status:               scheduler.BatchJobCreated,
-		BranchName:           branchName,
-		DiggerConfig:         diggerConfig,
-		BatchType:            batchType,
-		GitlabProjectId:      gitlabProjectId,
+		ID:                            uid,
+		VCS:                           vcsType,
+		GithubInstallationId:          githubInstallationId,
+		RepoOwner:                     repoOwner,
+		RepoName:                      repoName,
+		RepoFullName:                  repoFullname,
+		PrNumber:                      PRNumber,
+		CommentId:                     commentId,
+		PlaceholderCommentIdForReport: placeholderCommentIdForReport,
+		Status:                        scheduler.BatchJobCreated,
+		BranchName:                    branchName,
+		DiggerConfig:                  diggerConfig,
+		BatchType:                     batchType,
+		GitlabProjectId:               gitlabProjectId,
 	}
 	result := db.GormDB.Save(batch)
 	if result.Error != nil {

backend/services/spec.go

@@ -116,8 +116,10 @@ func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
 		CommentId: strconv.FormatInt(*batch.CommentId, 10),
 		Job:       jobSpec,
 		Reporter: spec.ReporterSpec{
-			ReportingStrategy: "comments_per_run",
+			//ReportingStrategy: "comments_per_run",
+			ReportingStrategy: "always_same_comment",
 			ReporterType:      "lazy",
+			ReportCommentId:   job.Batch.PlaceholderCommentIdForReport,
 		},
 		Lock: spec.LockSpec{
 			LockType: "noop",

backend/utils/graphs.go

@@ -14,7 +14,7 @@ import (
 )
 
 // ConvertJobsToDiggerJobs jobs is map with project name as a key and a Job as a value
-func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, diggerConfigStr string, gitlabProjectId int) (*uuid.UUID, map[string]*models.DiggerJob, error) {
+func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.DiggerVCSType, organisationId uint, jobsMap map[string]scheduler.Job, projectMap map[string]configuration.Project, projectsGraph graph.Graph[string, configuration.Project], githubInstallationId int64, branch string, prNumber int, repoOwner string, repoName string, repoFullName string, commitSha string, commentId int64, placeholderCommentIdForReport *string, diggerConfigStr string, gitlabProjectId int) (*uuid.UUID, map[string]*models.DiggerJob, error) {
 	result := make(map[string]*models.DiggerJob)
 	organisation, err := models.DB.GetOrganisationById(organisationId)
 	if err != nil {
@@ -43,7 +43,7 @@ func ConvertJobsToDiggerJobs(jobType scheduler.DiggerCommand, vcsType models.Dig
 
 	log.Printf("marshalledJobsMap: %v\n", marshalledJobsMap)
 
-	batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, gitlabProjectId)
+	batch, err := models.DB.CreateDiggerBatch(vcsType, githubInstallationId, repoOwner, repoName, repoFullName, prNumber, diggerConfigStr, branch, jobType, &commentId, placeholderCommentIdForReport, gitlabProjectId)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create batch: %v", err)
 	}

cli/cmd/digger/root.go

@@ -96,10 +96,6 @@ func PreRun(cmd *cobra.Command, args []string) {
 		ReportStrategy = &reporting.CommentPerRunStrategy{
 			TimeOfRun: time.Now(),
 		}
-	} else if os.Getenv("REPORTING_STRATEGY") == "latest_run_comment" {
-		ReportStrategy = &reporting.LatestRunCommentStrategy{
-			TimeOfRun: time.Now(),
-		}
 	} else {
 		ReportStrategy = &reporting.MultipleCommentsStrategy{}
 	}

cli/pkg/digger/digger.go

@@ -170,12 +170,12 @@ func RunJobs(jobs []orchestrator.Job, prService ci.PullRequestService, orgServic
 func reportPolicyError(projectName string, command string, requestedBy string, reporter reporting.Reporter) string {
 	msg := fmt.Sprintf("User %s is not allowed to perform action: %s. Check your policies :x:", requestedBy, command)
 	if reporter.SupportsMarkdown() {
-		_, _, err := reporter.Report(msg, coreutils.AsCollapsibleComment(fmt.Sprintf("Policy violation for <b>%v - %v</b>", projectName, command), false))
+		err := reporter.Report(msg, coreutils.AsCollapsibleComment(fmt.Sprintf("Policy violation for <b>%v - %v</b>", projectName, command), false))
 		if err != nil {
 			log.Printf("Error publishing comment: %v", err)
 		}
 	} else {
-		_, _, err := reporter.Report(msg, coreutils.AsComment(fmt.Sprintf("Policy violation for %v - %v", projectName, command)))
+		err := reporter.Report(msg, coreutils.AsComment(fmt.Sprintf("Policy violation for %v - %v", projectName, command)))
 		if err != nil {
 			log.Printf("Error publishing comment: %v", err)
 		}
@@ -311,7 +311,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 						preformattedMessaged = append(preformattedMessaged, fmt.Sprintf("    %v", message))
 					}
 					planReportMessage = planReportMessage + strings.Join(preformattedMessaged, "<br>")
-					_, _, err = reporter.Report(planReportMessage, planPolicyFormatter)
+					err = reporter.Report(planReportMessage, planPolicyFormatter)
 
 					if err != nil {
 						log.Printf("Failed to report plan. %v", err)
@@ -320,7 +320,7 @@ func run(command string, job orchestrator.Job, policyChecker policy.Checker, org
 					log.Printf(msg)
 					return nil, msg, fmt.Errorf(msg)
 				} else {
-					_, _, err := reporter.Report("Terraform plan validation checks succeeded :white_check_mark:", planPolicyFormatter)
+					err := reporter.Report("Terraform plan validation checks succeeded :white_check_mark:", planPolicyFormatter)
 					if err != nil {
 						log.Printf("Failed to report plan. %v", err)
 					}
@@ -496,12 +496,12 @@ func reportApplyMergeabilityError(reporter reporting.Reporter) string {
 	log.Println(comment)
 
 	if reporter.SupportsMarkdown() {
-		_, _, err := reporter.Report(comment, coreutils.AsCollapsibleComment("Apply error", false))
+		err := reporter.Report(comment, coreutils.AsCollapsibleComment("Apply error", false))
 		if err != nil {
 			log.Printf("error publishing comment: %v\n", err)
 		}
 	} else {
-		_, _, err := reporter.Report(comment, coreutils.AsComment("Apply error"))
+		err := reporter.Report(comment, coreutils.AsComment("Apply error"))
 		if err != nil {
 			log.Printf("error publishing comment: %v\n", err)
 		}
@@ -518,7 +518,7 @@ func reportTerraformPlanOutput(reporter reporting.Reporter, projectId string, pl
 		formatter = coreutils.GetTerraformOutputAsComment("Plan output")
 	}
 
-	_, _, err := reporter.Report(plan, formatter)
+	err := reporter.Report(plan, formatter)
 	if err != nil {
 		log.Printf("Failed to report plan. %v", err)
 	}
@@ -533,7 +533,7 @@ func reportPlanSummary(reporter reporting.Reporter, summary string) {
 		formatter = coreutils.AsComment("Plan summary")
 	}
 
-	_, _, err := reporter.Report("\n"+summary, formatter)
+	err := reporter.Report("\n"+summary, formatter)
 	if err != nil {
 		log.Printf("Failed to report plan summary. %v", err)
 	}
@@ -543,7 +543,7 @@ func reportEmptyPlanOutput(reporter reporting.Reporter, projectId string) {
 	identityFormatter := func(comment string) string {
 		return comment
 	}
-	_, _, err := reporter.Report("→ No changes in terraform output for "+projectId, identityFormatter)
+	err := reporter.Report("→ No changes in terraform output for "+projectId, identityFormatter)
 	// suppress the comment (if reporter is suppressible)
 	reporter.Suppress()
 	if err != nil {

ee/backend/controllers/gitlab.go

@@ -333,7 +333,7 @@ func handlePullRequestEvent(gitlabProvider utils.GitlabProvider, payload *gitlab
 		log.Printf("strconv.ParseInt error: %v", err)
 		utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: could not handle commentId: %v", err))
 	}
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, diggeryamlStr, projectId)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, 0, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentId, nil, diggeryamlStr, projectId)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(glService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -524,7 +524,7 @@ func handleIssueCommentEvent(gitlabProvider utils.GitlabProvider, payload *gitla
 		log.Printf("ParseInt err: %v", err)
 		return fmt.Errorf("parseint error: %v", err)
 	}
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, diggerYmlStr, projectId)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, models.DiggerVCSGitlab, organisationId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, 0, branch, issueNumber, repoOwner, repoName, repoFullName, commitSha, commentId64, nil, diggerYmlStr, projectId)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(glService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

ee/backend/hooks/github.go

@@ -151,7 +151,7 @@ var DriftReconcilliationHook ce_controllers.IssueCommentHook = func(gh utils.Git
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: could not handle commentId: %v", err))
 	}
 
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, diggerYmlStr, 0)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, "github", orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, defaultBranch, issueNumber, repoOwner, repoName, repoFullName, "", reporterCommentId, nil, diggerYmlStr, 0)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))

ee/cli/cmd/digger/root.go

@@ -90,10 +90,6 @@ func PreRun(cmd *cobra.Command, args []string) {
 		ReportStrategy = &reporting.CommentPerRunStrategy{
 			TimeOfRun: time.Now(),
 		}
-	} else if os.Getenv("REPORTING_STRATEGY") == "latest_run_comment" {
-		ReportStrategy = &reporting.LatestRunCommentStrategy{
-			TimeOfRun: time.Now(),
-		}
 	} else {
 		ReportStrategy = &reporting.MultipleCommentsStrategy{}
 	}

libs/comment_utils/reporting/core.go

@@ -1,7 +1,7 @@
 package reporting
 
 type Reporter interface {
-	Report(report string, reportFormatting func(report string) string) (commentId string, commentUrl string, error error)
+	Report(report string, reportFormatting func(report string) string) (error error)
 	Flush() (string, string, error)
 	Suppress() error
 	SupportsMarkdown() bool

libs/comment_utils/reporting/mock.go

@@ -4,9 +4,9 @@ type MockReporter struct {
 	commands []string
 }
 
-func (mockReporter *MockReporter) Report(report string, reportFormatting func(report string) string) (string, string, error) {
+func (mockReporter *MockReporter) Report(report string, reportFormatting func(report string) string) error {
 	mockReporter.commands = append(mockReporter.commands, "Report")
-	return "", "", nil
+	return nil
 }
 
 func (mockReporter *MockReporter) Flush() (string, string, error) {

libs/comment_utils/reporting/noop.go

@@ -2,8 +2,8 @@ package reporting
 
 type NoopReporter struct{}
 
-func (reporter NoopReporter) Report(report string, reportFormatting func(report string) string) (string, string, error) {
-	return "", "", nil
+func (reporter NoopReporter) Report(report string, reportFormatting func(report string) string) error {
+	return nil
 }
 
 func (reporter NoopReporter) Flush() (string, string, error) {