Wait for files

Author: KillerX

utils/workflows/files.go

@@ -49,10 +49,15 @@ func MoveFile(ctx workflow.Context, source, destination paths.Path, priority rcl
 }
 
 func CopyFile(ctx workflow.Context, source, destination paths.Path) error {
-	return Execute(ctx, activities.Util.CopyFile, activities.MoveFileInput{
-		Source:      source,
-		Destination: destination,
-	}).Wait(ctx)
+	external := source.OnExternalDrive() || destination.OnExternalDrive()
+	if external {
+		return RcloneCopyFile(ctx, source, destination, rclone.PriorityNormal)
+	} else {
+		return Execute(ctx, activities.Util.CopyFile, activities.MoveFileInput{
+			Source:      source,
+			Destination: destination,
+		}).Wait(ctx)
+	}
 }
 
 func CopyToFolder(ctx workflow.Context, file, folder paths.Path, priority rclone.Priority) (paths.Path, error) {
@@ -199,6 +204,27 @@ func RcloneCheckFileExists(ctx workflow.Context, file paths.Path) (bool, error)
 	}).Result(ctx)
 }
 
+func RcloneWaitForFileExists(ctx workflow.Context, file paths.Path, retries int) error {
+	for i := 0; i < retries; i++ {
+		exists, err := RcloneCheckFileExists(ctx, file)
+		if err != nil {
+			return err
+		}
+
+		if exists {
+			return nil
+		}
+
+		err = workflow.Sleep(ctx, time.Second*30)
+		if err != nil {
+			return err
+		}
+	}
+
+	return fmt.Errorf("could not find file %s with retries %d", file, retries)
+
+}
+
 func RcloneWaitForFileGone(ctx workflow.Context, file paths.Path, notificationChannel telegram.Chat, retries int) error {
 	fileExists := true
 

workflows/misc/masv_import.go

@@ -1,12 +1,14 @@
 package miscworkflows
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/bcc-code/bcc-media-flows/paths"
 	"github.com/bcc-code/bcc-media-flows/services/rclone"
 	"github.com/bcc-code/bcc-media-flows/services/telegram"
 	wfutils "github.com/bcc-code/bcc-media-flows/utils/workflows"
 	"go.temporal.io/sdk/workflow"
+	"strings"
 	"time"
 )
 
@@ -20,6 +22,40 @@ type MASVImportParams struct {
 	EventTime  string
 }
 
+type MASVMetadata struct {
+	ID        string    `json:"id"`
+	CreatedAt time.Time `json:"created_at"`
+	UpdatedAt time.Time `json:"updated_at"`
+	Package   Package   `json:"package"`
+}
+type Files struct {
+	ID   string `json:"id"`
+	Kind string `json:"kind"`
+	Name string `json:"name"`
+	Path string `json:"path"`
+	Size int    `json:"size"`
+}
+type Metadata struct {
+	Church             string `json:"church"`
+	PackageDescription string `json:"package_description"`
+	PackageName        string `json:"package_name"`
+	SenderEmail        string `json:"sender_email"`
+}
+type Package struct {
+	ID         string    `json:"id"`
+	Files      []Files   `json:"files"`
+	Name       string    `json:"name"`
+	PortalID   string    `json:"portal_id"`
+	PortalName string    `json:"portal_name"`
+	Sender     string    `json:"sender"`
+	Size       int       `json:"size"`
+	State      string    `json:"state"`
+	TotalFiles int       `json:"total_files"`
+	Metadata   Metadata  `json:"metadata"`
+	CreatedAt  time.Time `json:"created_at"`
+	UpdatedAt  time.Time `json:"updated_at"`
+}
+
 func MASVImport(ctx workflow.Context, params MASVImportParams) error {
 	logger := workflow.GetLogger(ctx)
 	logger.Info("Starting MASVImport workflow", "id", params.ID, "name", params.Name)
@@ -39,36 +75,79 @@ func MASVImport(ctx workflow.Context, params MASVImportParams) error {
 	if err != nil {
 		return err
 	}
+
 	dst := tmpRoot.Append("masv", params.ID)
 	if err := wfutils.CreateFolder(ctx, dst); err != nil {
 		return err
 	}
 
-	fileAvailable := false
+	srcFolder, err := paths.Parse(src)
+	if err != nil {
+		return err
+	}
+
+	var metaFileInfo *rclone.RcloneFile
 	for i := 0; i < 60; i++ {
-		srcFolder, err := paths.Parse(src)
-		if err != nil {
-			return err
-		}
 
 		files, err := wfutils.RcloneListFiles(ctx, srcFolder)
 		if err != nil {
 			return err
 		}
 
-		if len(files) > 0 {
-			fileAvailable = true
+		for _, file := range files {
+
+			if !strings.HasSuffix(file.Name, "metadata.json") {
+				continue
+			}
+
+			metaFileInfo = &file
+			break
+		}
+
+		if metaFileInfo != nil {
 			break
 		}
 
 		err = workflow.Sleep(ctx, 30*time.Second)
+	}
+
+	if metaFileInfo == nil {
+		return fmt.Errorf("could not find metadata file for package %s", params.ID)
+	}
+
+	metaRemotePath, err := paths.Parse("s3prod:" + metaFileInfo.Path)
+	if err != nil {
+		return err
+	}
+
+	metaFilePath := tmpRoot.Append("masv_meta.json")
+	err = wfutils.CopyFile(ctx, metaRemotePath, metaFilePath)
+	if err != nil {
+		return err
+	}
+
+	metaBytes, err := wfutils.ReadFile(ctx, metaFilePath)
+	if err != nil {
+		return err
+	}
+
+	masvMeta := &MASVMetadata{}
+	err = json.Unmarshal(metaBytes, masvMeta)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range masvMeta.Package.Files {
+		fpath := fmt.Sprintf("s3prod:/massiveio-bccm/upload/%s/%s", params.Name, f.Path)
+		parsedPath, err := paths.Parse(fpath)
 		if err != nil {
 			return err
 		}
-	}
 
-	if !fileAvailable {
-		return fmt.Errorf("could not find masv file in %s", src)
+		err = wfutils.RcloneWaitForFileExists(ctx, parsedPath, 30)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Copy the directory contents from s3 to temp using rclone