Merge pull request #1644 from diggerhq/feat/dgctl-artefacts-support

feat/dgctl artefacts support

Author: ZIJ

backend/middleware/basic.go

@@ -59,6 +59,7 @@ func HttpBasicApiAuth() gin.HandlerFunc {
 			} else {
 				setDefaultOrganisationId(c)
 				c.Set(ACCESS_LEVEL_KEY, jobToken.Type)
+				c.Set(JOB_TOKEN_KEY, jobToken.Value)
 			}
 		} else if token == os.Getenv("BEARER_AUTH_TOKEN") {
 			setDefaultOrganisationId(c)

backend/middleware/jwt.go

@@ -309,3 +309,4 @@ func CORSMiddleware() gin.HandlerFunc {
 
 const ORGANISATION_ID_KEY = "organisation_ID"
 const ACCESS_LEVEL_KEY = "access_level"
+const JOB_TOKEN_KEY = "job_token"

backend/migrations/20240729155442.sql

@@ -0,0 +1,13 @@
+-- Create "job_artefacts" table
+CREATE TABLE "public"."job_artefacts" (
+  "id" bigserial NOT NULL,
+  "created_at" timestamptz NULL,
+  "updated_at" timestamptz NULL,
+  "deleted_at" timestamptz NULL,
+  "job_token_id" bigint NULL,
+  "contents" bytea NULL,
+  PRIMARY KEY ("id"),
+  CONSTRAINT "fk_job_artefacts_job_token" FOREIGN KEY ("job_token_id") REFERENCES "public"."job_tokens" ("id") ON UPDATE NO ACTION ON DELETE NO ACTION
+);
+-- Create index "idx_job_artefacts_deleted_at" to table: "job_artefacts"
+CREATE INDEX "idx_job_artefacts_deleted_at" ON "public"."job_artefacts" ("deleted_at");

backend/migrations/20240729155926.sql

@@ -0,0 +1,2 @@
+-- Modify "job_artefacts" table
+ALTER TABLE "public"."job_artefacts" ADD COLUMN "size" bigint NULL, ADD COLUMN "content_type" text NULL;

backend/migrations/20240729160028.sql

@@ -0,0 +1,2 @@
+-- Modify "job_artefacts" table
+ALTER TABLE "public"."job_artefacts" ADD COLUMN "filename" text NULL;

backend/migrations/atlas.sum

@@ -1,4 +1,4 @@
-h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8=
+h1:A7OaxVcBVM26DpHZ8tH+NLsfDZxrkpsFUxFJbK3Am68=
 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw=
 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA=
 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw=
@@ -27,3 +27,6 @@ h1:r0O5Hy6gJdMDgkCW9ocd4ytiCyPghNcXKTAT3tuppZ8=
 20240704192835.sql h1:F84HKuE3qX8hYqDCU2K9NDG4WQaSfmnk2P4OCje54qg=
 20240705144450.sql h1:AAB4GbML2Jea1MSv/+E0DLMMCI9GLqwptpvBScvU19s=
 20240709165155.sql h1:HWDuhwXD+esr9uaKHsqwltPfUEq5uLdcEzpnZ1biKgU=
+20240729155442.sql h1:s7PCALP3SgPz5y9Ya7HkDzYjeN86Q5NniW2YIkOMBrQ=
+20240729155926.sql h1:8vsDrpy/R1UDI+meIp6KoDfhS60t+ngu8aPB+uonFZ4=
+20240729160028.sql h1:snkkxhA2aEQhqBmIhN8l+nPlBhrPOZiPP+dnyhobwD8=

backend/models/artefact.go

@@ -0,0 +1,15 @@
+package models
+
+import (
+	"gorm.io/gorm"
+)
+
+type JobArtefact struct {
+	gorm.Model
+	JobTokenID  uint
+	JobToken    JobToken
+	Filename    string
+	Contents    []byte `gorm:"type:bytea"`
+	Size        int64
+	ContentType string
+}

backend/models/storage.go

@@ -1126,6 +1126,25 @@ func (db *Database) GetJobToken(tenantId any) (*JobToken, error) {
 	return token, nil
 }
 
+func (db *Database) DeleteJobTokenArtefacts(jobTokenId uint) error {
+	artefact := JobArtefact{}
+	result := db.GormDB.Where("job_token_id = ?", jobTokenId).Delete(&artefact)
+	if result.Error != nil {
+		return result.Error
+	}
+	log.Printf("DeleteJobTokenArtefacts %v has been deleted successfully\n", jobTokenId)
+	return nil
+
+}
+
+func (db *Database) GetJobArtefact(jobTokenId uint) (*JobArtefact, error) {
+	var artefact JobArtefact
+	if err := DB.GormDB.Where("job_token_id = ?", jobTokenId).First(&artefact).Error; err != nil {
+		return nil, err
+	}
+	return &artefact, nil
+}
+
 func (db *Database) CreateGithubAppInstallation(installationId int64, githubAppId int64, login string, accountId int, repoFullName string) (*GithubAppInstallation, error) {
 	installation := &GithubAppInstallation{
 		GithubInstallationId: installationId,

cli/pkg/spec/spec.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/cli/pkg/digger"
 	"github.com/diggerhq/digger/cli/pkg/usage"
+	"github.com/diggerhq/digger/cli/pkg/utils"
+	"github.com/diggerhq/digger/libs/backendapi"
 	"github.com/diggerhq/digger/libs/ci"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary"
@@ -14,6 +16,7 @@ import (
 	"github.com/samber/lo"
 	"log"
 	"os"
+	"os/exec"
 	"time"
 )
 
@@ -169,6 +172,45 @@ func RunSpecManualCommand(
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1)
 	}
 
+	// download zip artefact, git init and prepare for job execution
+	tempDir, err := os.MkdirTemp("", "downloaded-zip-")
+	if err != nil {
+		log.Printf("failed to create temp dir: %v", err)
+		os.Exit(1)
+	}
+
+	// downloading artefact zip , extracting, git init and then chdir to that directory for job execution
+	absoluteFileName, err := backendApi.DownloadJobArtefact(tempDir)
+	if err != nil {
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not download job artefact: %v", err), 1)
+	}
+	zipPath := *absoluteFileName
+	err = utils.ExtractZip(zipPath, tempDir)
+	if err != nil {
+		log.Printf("ExtractZip err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("artefact zip extraction err: %v", err), 1)
+
+	}
+
+	// remove the zipPath
+	err = os.Remove(zipPath)
+	if err != nil {
+		log.Printf("os.Remove err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("zip path removal err: %v", err), 1)
+	}
+
+	// Navigating to our diractory
+	err = os.Chdir(tempDir)
+	if err != nil {
+		log.Printf("Chdir err: %v", err)
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Chdir err: %v", err), 1)
+	}
+
+	cmd := exec.Command("git", "init")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Run()
+
 	policyChecker, err := policyProvider.GetPolicyProvider(spec.Policy, spec.Backend.BackendHostname, spec.Backend.BackendOrganisationName, spec.Backend.BackendJobToken)
 	if err != nil {
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get policy provider: %v", err), 1)
@@ -182,11 +224,13 @@ func RunSpecManualCommand(
 
 	jobs := []scheduler.Job{job}
 
-	fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
-	_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "")
-	if err != nil {
-		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4)
-	}
+	//fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)
+	//_, err = backendApi.ReportProjectJobStatus(fullRepoName, spec.Job.ProjectName, spec.JobId, "started", time.Now(), nil, "", "")
+	//if err != nil {
+	//	usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("Failed to report jobSpec status to backend. Exiting. %v", err), 4)
+	//}
+
+	noopBackendApi := backendapi.NoopApi{}
 
 	commentId := spec.CommentId
 	if err != nil {
@@ -199,9 +243,9 @@ func RunSpecManualCommand(
 	}
 
 	commentUpdater := comment_summary.NoopCommentUpdater{}
-	// TODO: do not require conversion to gh service
+	// do not change these placeholders as they are parsed by dgctl to stream logs
 	log.Printf("<========= DIGGER RUNNING IN MANUAL MODE =========>")
-	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, backendApi, spec.JobId, false, false, commentId, currentDir)
+	allAppliesSuccess, _, err := digger.RunJobs(jobs, prService, orgService, lock, reporter, planStorage, policyChecker, commentUpdater, noopBackendApi, spec.JobId, false, false, commentId, currentDir)
 	log.Printf("<========= DIGGER COMPLETED =========>")
 	if err != nil || allAppliesSuccess == false {
 		usage.ReportErrorAndExit(spec.VCS.RepoOwner, "Terraform execution failed", 1)

cli/pkg/utils/io.go

@@ -0,0 +1,52 @@
+package utils
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func ExtractZip(zipFilePath string, outDir string) error {
+
+	// Open the zip file
+	zipReader, err := zip.OpenReader(zipFilePath)
+	if err != nil {
+		return fmt.Errorf("failed to open zip: %w", err)
+	}
+	defer zipReader.Close()
+
+	for _, file := range zipReader.File {
+		path := filepath.Join(outDir, file.Name)
+
+		if file.FileInfo().IsDir() {
+			os.MkdirAll(path, os.ModePerm)
+			continue
+		}
+
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create directory: %w", err)
+		}
+
+		dstFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode())
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+
+		srcFile, err := file.Open()
+		if err != nil {
+			dstFile.Close()
+			return fmt.Errorf("failed to open zip file: %w", err)
+		}
+
+		_, err = io.Copy(dstFile, srcFile)
+		srcFile.Close()
+		dstFile.Close()
+
+		if err != nil {
+			return fmt.Errorf("failed to extract file: %w", err)
+		}
+	}
+	return nil
+}