Merge pull request #388 from bcc-code/feature/bcc-108-ffmpeg-scene-change-detection-for-short-generation

get scene change frames with ffmpeg & send them to yolo

Author: KillerX

activities/crop_shorts.go

@@ -3,11 +3,12 @@ package activities
 import (
 	"context"
 	"fmt"
-	"github.com/bcc-code/bcc-media-flows/paths"
 	"math"
 	"strconv"
 	"strings"
 
+	"github.com/bcc-code/bcc-media-flows/paths"
+
 	"github.com/bcc-code/bcc-media-flows/services/ffmpeg"
 )
 
@@ -125,7 +126,7 @@ func calculatePanDuration(distance float64) float64 {
 	const (
 		minDur = 0.1
 		maxDur = 3.0
-		speed  = 200.0
+		speed  = 100.0
 	)
 	d := distance / speed
 	if d < minDur {

activities/shorts.go

@@ -12,10 +12,11 @@ import (
 var shortServiceURL = os.Getenv("SHORTS_SERVICE_URL")
 
 type SubmitShortJobInput struct {
-	InputPath  string `json:"input_path"`
-	OutputPath string `json:"output_path"`
-	Model      string `json:"model"`
-	Debug      bool   `json:"debug"`
+	InputPath    string    `json:"input_path"`
+	OutputPath   string    `json:"output_path"`
+	Model        string    `json:"model"`
+	Debug        bool      `json:"debug"`
+	SceneChanges []float64 `json:"scene_changes"`
 }
 
 type Square struct {

activities/transcode.go

@@ -3,6 +3,8 @@ package activities
 import (
 	"context"
 	"fmt"
+	"regexp"
+	"strconv"
 
 	"github.com/bcc-code/bcc-media-flows/utils"
 
@@ -323,6 +325,33 @@ func (va VideoActivities) ExecuteFFmpeg(ctx context.Context, input ExecuteFFmpeg
 	return nil, err
 }
 
+func (va VideoActivities) ExecuteFFmpegDump(
+	ctx context.Context,
+	input ExecuteFFmpegInput,
+) ([]float64, error) {
+
+	stopChan, progressCallback := registerProgressCallback(ctx)
+	defer close(stopChan)
+
+	out, err := ffmpeg.Do(input.Arguments, ffmpeg.StreamInfo{}, progressCallback)
+	if err != nil {
+		return nil, err
+	}
+
+	raw := string(out)
+	re := regexp.MustCompile(`(?m)pts_time:([\d.]+)`)
+	matches := re.FindAllStringSubmatch(raw, -1)
+
+	var changes []float64
+	for _, m := range matches {
+		if len(m) >= 2 {
+			t, _ := strconv.ParseFloat(m[1], 64)
+			changes = append(changes, t)
+		}
+	}
+	return changes, nil
+}
+
 type SplitAudioChannelsInput struct {
 	FilePath  paths.Path
 	OutputDir paths.Path

utils/workflows/files.go

@@ -3,13 +3,14 @@ package wfutils
 import (
 	"encoding/xml"
 	"fmt"
+	"path/filepath"
+	"strings"
+	"time"
+
 	"github.com/bcc-code/bcc-media-flows/services/notifications"
 	"github.com/bcc-code/bcc-media-flows/services/rclone"
 	"github.com/bcc-code/bcc-media-flows/services/telegram"
 	"go.temporal.io/sdk/temporal"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"github.com/bcc-code/bcc-media-flows/activities"
 	"github.com/bcc-code/bcc-media-flows/environment"

workflows/export/generate_short.go

@@ -2,10 +2,11 @@ package export
 
 import (
 	"fmt"
-	vsactivity "github.com/bcc-code/bcc-media-flows/activities/vidispine"
 	"strings"
 	"time"
 
+	vsactivity "github.com/bcc-code/bcc-media-flows/activities/vidispine"
+
 	"github.com/bcc-code/bcc-media-flows/activities"
 	"github.com/bcc-code/bcc-media-flows/paths"
 	"github.com/bcc-code/bcc-media-flows/services/vidispine"
@@ -28,6 +29,8 @@ type GenerateShortDataParams struct {
 	OutputDirPath string  `json:"OutputDir"`
 	InSeconds     float64 `json:"InSeconds"`
 	OutSeconds    float64 `json:"OutSeconds"`
+	ModelSize     string  `json:"ModelSize"`
+	DebugMode     bool    `json:"DebugMode"`
 }
 
 func validationError(msg string) error {
@@ -109,11 +112,27 @@ func GenerateShort(ctx workflow.Context, params GenerateShortDataParams) (*Gener
 		return nil, err
 	}
 
+	sceneDetectArgs := []string{
+		"-i", clipResult.VideoFile.Local(),
+		"-filter_complex", "select='gt(scene,0.1)',metadata=print:file=-",
+		"-f", "null", "-",
+	}
+
+	sceneResult, err := wfutils.Execute(ctx,
+		activities.Video.ExecuteFFmpegDump,
+		activities.ExecuteFFmpegInput{Arguments: sceneDetectArgs},
+	).Result(ctx)
+	if err != nil {
+		logger.Error("Scene-detect FFmpeg failed: " + err.Error())
+		return nil, err
+	}
+
 	submitJobParams := activities.SubmitShortJobInput{
-		InputPath:  clipResult.VideoFile.Linux(),
-		OutputPath: tempFolder.Linux(),
-		Model:      "n",
-		Debug:      true,
+		InputPath:    clipResult.VideoFile.Linux(),
+		OutputPath:   tempFolder.Linux(),
+		Model:        params.ModelSize,
+		Debug:        params.DebugMode,
+		SceneChanges: sceneResult,
 	}
 
 	jobResult, err := wfutils.Execute(ctx, activities.Util.SubmitShortJobActivity, submitJobParams).Result(ctx)

workflows/misc/ffmpeg.go

@@ -25,9 +25,22 @@ func ExecuteFFmpeg(
 		Arguments: params.Arguments,
 	}).Wait(ctx)
 
-	if err != nil {
-		return err
-	}
-
 	return err
 }
+
+func ExecuteFFmpegDump(
+	ctx workflow.Context,
+	params ExecuteFFmpegInput,
+) (string, error) {
+	logger := workflow.GetLogger(ctx)
+	logger.Info("Starting ExecuteFFmpegDump")
+
+	ctx = workflow.WithActivityOptions(ctx, wfutils.GetDefaultActivityOptions())
+
+	var out string
+	err := wfutils.Execute(ctx, activities.Video.ExecuteFFmpegDump,
+		activities.ExecuteFFmpegInput{Arguments: params.Arguments},
+	).Get(ctx, &out)
+
+	return out, err
+}