Better support for multi-containers capability by using a lock in the watch folder when possible.

Author: jlesage

README.md

@@ -55,6 +55,7 @@ of modern, widely supported codecs.
    * [Access to Optical Drives](#access-to-optical-drives)
    * [Automatic Video Conversion](#automatic-video-conversion)
       * [Multiple Watch Folders](#multiple-watch-folders)
+      * [Multiple Containers Capability](#multiple-containers-capability)
       * [Video Discs](#video-discs)
       * [Hooks](#hooks)
       * [Temporary Conversion Directory](#temporary-conversion-directory)
@@ -809,13 +810,30 @@ The maximum number of watch folders is defined by the
 `AUTOMATED_CONVERSION_MAX_WATCH_FOLDERS` environment variable.
 
 > [!NOTE]
->Each additional watch folder must be mapped to a host folder via a volume
+> Each additional watch folder must be mapped to a host folder via a volume
 > mapping during container creation.
 
 > [!NOTE]
 > Each output folder defined via `AUTOMATED_CONVERSION_OUTPUT_DIR` must be
 > mapped to a host folder via a volume mapping during container creation.
 
+### Multiple Containers Capability
+
+Multiple container instances can operate on the same watch folder to increase
+throughput and parallelize video conversions. Each container monitors the folder
+independently and picks up available video files for processing.
+
+> [!NOTE]
+> The watch folder must be writable by all containers, since each container
+> creates a lock file in the folder before starting a conversion. This ensures
+> that no two containers process the same video at the same time.
+
+> [!NOTE]
+> To prevent already-converted videos from being picked up again, configure each
+> container to remove the source file once processing is finished. This can be
+> enforced by setting the `AUTOMATED_CONVERSION_KEEP_SOURCE` environment
+> variable to `0`.
+
 ### Video Discs
 
 The automatic video converter supports video discs in the following format:

appdefs.yml

@@ -135,12 +135,29 @@ app:
           `AUTOMATED_CONVERSION_MAX_WATCH_FOLDERS` environment variable.
 
           > [!NOTE]
-          >Each additional watch folder must be mapped to a host folder via a volume
+          > Each additional watch folder must be mapped to a host folder via a volume
           > mapping during container creation.
 
           > [!NOTE]
           > Each output folder defined via `AUTOMATED_CONVERSION_OUTPUT_DIR` must be
           > mapped to a host folder via a volume mapping during container creation.
+      - title: Multiple Containers Capability
+        level: 3
+        content: |-
+          Multiple container instances can operate on the same watch folder to increase
+          throughput and parallelize video conversions. Each container monitors the folder
+          independently and picks up available video files for processing.
+
+          > [!NOTE]
+          > The watch folder must be writable by all containers, since each container
+          > creates a lock file in the folder before starting a conversion. This ensures
+          > that no two containers process the same video at the same time.
+
+          > [!NOTE]
+          > To prevent already-converted videos from being picked up again, configure each
+          > container to remove the source file once processing is finished. This can be
+          > enforced by setting the `AUTOMATED_CONVERSION_KEEP_SOURCE` environment
+          > variable to `0`.
       - title: Video Discs
         level: 3
         content: |-

rootfs/etc/services.d/autovideoconverter/run

@@ -4,9 +4,12 @@ set -u # Treat unset variables as an error.
 
 FAILED_CONVERSIONS="/config/failed_conversions"
 SUCCESSFUL_CONVERSIONS="/config/successful_conversions"
+IGNORED_CONVERSIONS="/config/ignored_conversion"
 
 HANDBRAKE_CLI="/usr/bin/HandBrakeCLI --preset-import-file /config/ghb/presets.json"
 
+CONTAINER_INSTANCE_ID="$(cat /config/machine-id)"
+
 # https://gist.github.com/aaomidi/0a3b5c9bd563c9e012518b495410dc0e
 VIDEO_FILE_EXTENSIONS_INTERNAL_LIST="\
     webm mkv flv vob ogv ogg rrc gifv mng mov avi qt wmv yuv rm asf amv mp4 \
@@ -42,6 +45,13 @@ log() {
     echo "$*"
 }
 
+is_dir_writable() {
+    tmpfile="$(mktemp "$1"/.test_XXXXXX 2>/dev/null)"
+    rc=$?
+    rm -f "$tmpfile"
+    return "$rc"
+}
+
 log_hb_encode_progress() {
     while read OUTPUT; do
         echo "Encoding $video: $OUTPUT"
@@ -121,6 +131,43 @@ get_video_titles() {
     return ${PIPESTATUS[0]}
 }
 
+get_video_lock_dname() {
+    video="$1"
+    wf="$2"
+    lock_dname="$(echo "$video" | sed "s|^$wf||" | tr '/' '.')".lock
+    echo "$lock_dname"
+}
+
+lock_video() {
+    video="$1"
+    wf="$2"
+
+    if ! is_dir_writable "$wf"; then
+        return 0
+    fi
+
+    lock_dname="$(get_video_lock_dname "$video" $"$wf")"
+    err="$(mkdir "$wf"/"$lock_dname" 2>&1)"
+    rc=$?
+    if [ "$rc" -eq 0 ]; then
+        mkdir "$wf"/"$lock_dname"/"$CONTAINER_INSTANCE_ID"
+        return 0
+    else
+        if echo "$err" | grep -iq "File exists"; then
+            return 1
+        else
+            return 2
+        fi
+    fi
+}
+
+unlock_video() {
+    video="$1"
+    wf="$2"
+    lock_dname="$(get_video_lock_dname "$video" $"$wf")"
+    rm  -rf "$wf"/"$lock_dname"
+}
+
 process_video() {
     video="$1"
     wf="$2"
@@ -161,6 +208,24 @@ process_video() {
         return
     fi
 
+    # Skip video if it has been ignored.
+    if [ -f "$IGNORED_CONVERSIONS" ] && grep -q -w "$hash" "$IGNORED_CONVERSIONS"; then
+        log "Skipping video '$video' ($hash): has been processed by another instance."
+        return
+    fi
+
+    # Skip video if it is being processed by another instance.
+    lock_video "$video" "$wf"
+    lock_rc=$?
+    if [ "$lock_rc" -eq 1 ]; then
+        echo "$video $hash" >> "$IGNORED_CONVERSIONS"
+        log "Skipping '$video': currently beging processed by another instance."
+        return
+    elif [ "$lock_rc" -eq 2 ]; then
+        log "ERROR: Unable to acquire lock for video '${video}' (${hash})."
+        return
+    fi
+
     # Set the output directory.
     case "$AC_OUTPUT_SUBDIR" in
         UNSET)
@@ -196,6 +261,7 @@ process_video() {
     # Get video titles.
     if $FILE_EXT_IN_VIDEO_LIST && $FILE_EXT_IN_NON_VIDEO_LIST; then
         log "ERROR: File '${video}' (${hash}) has an extension defined as both a video and non-video file."
+        unlock_video "$video" "$wf"
         return
     elif [ -n "$AC_VIDEO_FILE_EXTENSIONS" ] && ! $FILE_EXT_IN_VIDEO_LIST; then
         log "File '${video}' (${hash}) has an extension not part of the inclusion list."
@@ -417,29 +483,34 @@ process_video() {
         fi
         echo "$video $hash" >> "$FAILED_CONVERSIONS"
     fi
+
+    unlock_video "$video" "$wf"
 }
 
 process_watch_folder() {
     WF="$1"
-
     NUM_PROCESSED_FILES=0
 
+    # Return now if the watch folder doesn't exist.
     [ -d "$WF" ] || return
+
+    # Remove any left-over locks for our own instance.
+    find "$WF" -mindepth 1 -maxdepth 1 -type d -name "*.lock" | while read -r d; do
+        if [ -e "$d"/"$CONTAINER_INSTANCE_ID" ]; then
+            rm -rf "$d"
+        fi
+    done
+
+    # Return now if the watch folder didn't change.
     WATCHDIR_HASH_changed "$WF" || return
 
     # Make sure the output directory is properly setup.
     if [ ! -d "$AC_OUTPUT_DIR" ]; then
         log "ERROR: Cannot process watch folder '$WF', because the associated output directory '$AC_OUTPUT_DIR' doesn't exist."
         return
-    else
-        TMPFILE="$(mktemp "$AC_OUTPUT_DIR"/.test_XXXXXX 2>/dev/null)"
-        RC=$?
-        if [ "$RC" -eq 0 ]; then
-            rm "$TMPFILE"
-        else
-            log "ERROR: Cannot process watch folder '$WF', because the associated output directory '$AC_OUTPUT_DIR' is not writable."
-            return
-        fi
+    elif ! is_dir_writable "$AC_OUTPUT_DIR"; then
+        log "ERROR: Cannot process watch folder '$WF', because the associated output directory '$AC_OUTPUT_DIR' is not writable."
+        return
     fi
 
     if WATCHDIR_HASH_isset "$WF"; then
@@ -507,6 +578,7 @@ process_watch_folder() {
 
 [ -f "$FAILED_CONVERSIONS" ] || touch "$FAILED_CONVERSIONS"
 [ -f "$SUCCESSFUL_CONVERSIONS" ] || touch "$SUCCESSFUL_CONVERSIONS"
+[ -f "$IGNORED_CONVERSIONS" ] || touch "$IGNORED_CONVERSIONS"
 
 while true; do
     for i in $(seq 1 ${AUTOMATED_CONVERSION_MAX_WATCH_FOLDERS:-5}); do