Draft showing changes undergoing testing

.github/workflows/ci.yaml

@@ -12,6 +12,7 @@ on:
   pull_request:
     branches:
       - 'main'
+      - 'test-*'
     types:
       - opened
       - reopened
@@ -41,24 +42,6 @@ jobs:
           set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
           set -eux
           set_sl_var image "${image}" >> "${GITHUB_OUTPUT}"
-          # v0.14.1 = ghcr.io/meeb/tubesync:latest@sha256:63fffee3411bda395c526087d7107f76834141bb0c1443a0bd6bed5533e5a85f
-          tag_tried='v0.14.1'
-          tag_image_digest='latest@sha256:63fffee3411bda395c526087d7107f76834141bb0c1443a0bd6bed5533e5a85f'
-          docker pull "${image}:${tag_image_digest}"
-          docker pull "${image}:${tag_tried}" && tag_found=1 || {
-            tag_found=0
-            missing="${missing}${missing:+|}${tag_tried}=${image}:${tag_image_digest}"
-          }
-          set_sl_var "${tag_tried}" "${tag_found}" >> "${GITHUB_OUTPUT}"
-          # v0.15.1 = ghcr.io/meeb/tubesync:latest@sha256:7303b2d8854aac15f94dbbfdd0ee66ca598ade1af6ac2d9e3d886c93ffa2d596
-          tag_tried='v0.15.1'
-          tag_image_digest='latest@sha256:7303b2d8854aac15f94dbbfdd0ee66ca598ade1af6ac2d9e3d886c93ffa2d596'
-          docker pull "${image}:${tag_image_digest}"
-          docker pull "${image}:${tag_tried}" && tag_found=1 || {
-            tag_found=0
-            missing="${missing}${missing:+|}${tag_tried}=${image}:${tag_image_digest}"
-          }
-          set_sl_var "${tag_tried}" "${tag_found}" >> "${GITHUB_OUTPUT}"
           set_sl_var missing "${missing}" >> "${GITHUB_OUTPUT}"
       - uses: actions/checkout@v4
       - name: Lowercase github username
@@ -132,6 +115,8 @@ jobs:
             pip install --system --strict --requirements requirements.txt
       - name: Set up Django environment
         run: |
+          mkdir -v -p ~/.config/TubeSync/config
+          sudo ln -v -s -f -T ~/.config/TubeSync/config /config
           cp -v -p tubesync/tubesync/local_settings.py.example tubesync/tubesync/local_settings.py
           cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/background_task/ patches/background_task/*
           cp -v -a -t "${Python3_ROOT_DIR}"/lib/python3.*/site-packages/yt_dlp/ patches/yt_dlp/*
@@ -185,7 +170,94 @@ jobs:
     needs: ['info', 'test']
     runs-on: ubuntu-latest
     timeout-minutes: 120
+    services:
+      wormhole-mailbox:
+        image: 'ghcr.io/tcely/docker-magic-wormhole-mailbox-server:service'
+        ports:
+          - '4000:4000'
+      wormhole-transit:
+        image: 'ghcr.io/tcely/docker-magic-wormhole-transit-relay:main'
+        ports:
+          - '4001:4001'
     steps:
+      - name: Set environment variables with jq
+        run: |
+          # jq
+          cat >| .ffmpeg.releases.json <<'EOF'
+          ${{ needs.info.outputs.ffmpeg-releases }}
+          EOF
+
+          FFMPEG_DATE='[foreach .[] as $release ([{}, []]; [ .[0] + { ($release.commit): ([ $release.date ] + (.[0][($release.commit)] // [])) }, [ .[1][0] // $release.commit ] ] ; .[0][(.[1][0])] )][-1][0]' ;
+          FFMPEG_VERSION='.[]|select(.date == $previous)|.versions[]|select(startswith("N-"))' ;
+
+          mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            for var in FFMPEG_DATE FFMPEG_VERSION
+            do
+              # jq_arg="$( eval printf -- "'%s\n'" "$(printf -- '"${%s}"' "${var}")" )" ;
+              jq_arg="$( eval printf -- "'%s\n'" '"${'"${var}"'}"' )" ;
+              delim="$(mk_delim "${var}")" ;
+              open_ml_var "${delim}" "${var}" ;
+              jq -r --arg previous "${previous_value-}" "${jq_arg}" -- .ffmpeg.releases.json ;
+              close_ml_var "${delim}" "${var}" ;
+              previous_value="$( jq -r --arg previous "${previous_value-}" "${jq_arg}" -- .ffmpeg.releases.json )" ;
+            done ;
+
+            unset -v delim jq_arg previous_value var ;
+          } >> "${GITHUB_ENV}"
+          rm -v -f .ffmpeg.releases.json
+      - name: Set environment variables with GitHub CLI
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # gh api
+          GH_UPSTREAM_OWNER='.parent.owner.login' ;
+          GH_UPSTREAM_REPO='.parent.name' ;
+          GH_UPSTREAM_SLUG='.parent.full_name' ;
+
+          mk_delim() { printf -- '"%s_EOF_%d_"' "$1" "${RANDOM}" ; } ;
+          open_ml_var() { local f=''\%'s<<'\%'s\n' ; printf -- "${f}" "$2" "$1" ; } ;
+          close_ml_var() { local f='%s\n' ; printf -- "${f}" "$1" ; } ;
+          {
+            for var in GH_UPSTREAM_OWNER # GH_UPSTREAM_REPO GH_UPSTREAM_SLUG
+            do
+              # jq_arg="$( eval printf -- "'%s\n'" "$(printf -- '"${%s}"' "${var}")" )" ;
+              jq_arg="$( eval printf -- "'%s\n'" '"${'"${var}"'}"' )" ;
+              delim="$(mk_delim "${var}")" ;
+              open_ml_var "${delim}" "${var}" ;
+              gh api "repos/${GITHUB_REPOSITORY}" --cache 1h --jq "${jq_arg}" ;
+              close_ml_var "${delim}" "${var}" ;
+            done ;
+            unset -v delim jq_arg var ;
+          } >> "${GITHUB_ENV}"
+
+          # Delete the oldest unused cache entries
+          printf -- '%s\n' 'Deleting unused cache entries' ;
+          gh cache list --sort last_accessed_at --order asc --ref "${GITHUB_REF}" | \
+            awk '$NF == $(NF-1) {print $1}' | \
+            xargs -r -t -n 1 gh cache delete 2>&1 | \
+            tee /dev/stderr | wc -l | xargs -n 1 printf -- 'Total deleted: %d\n' ;
+      - name: Upstream registry ref
+        id: upstream
+        run: |
+          user_lowercase="$(printf -- '%s\n' "${GH_UPSTREAM_OWNER}" | awk '{print tolower($0);}')" ;
+          repo_image="ghcr.io/${user_lowercase}/${IMAGE_NAME}"
+          set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
+          set_sl_var >> "${GITHUB_OUTPUT}" \
+            ref "${repo_image}:latest" \
+            tag "${repo_image}:latest" ;
+      - name: Registry ref
+        id: origin
+        run: |
+          user_lowercase="$(printf -- '%s\n' "${GITHUB_ACTOR}" | awk '{print tolower($0);}')" ;
+          repo_image="ghcr.io/${user_lowercase}/${IMAGE_NAME}"
+          set_sl_var() { local f='%s=%s\n' ; printf -- "${f}" "$@" ; } ;
+          set_sl_var >> "${GITHUB_OUTPUT}" \
+            ref "${repo_image}:cache" \
+            tag "${repo_image}:latest" ;
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -198,7 +270,7 @@ jobs:
           DOCKER_TOKEN: ${{ 'meeb' == github.repository_owner && secrets.REGISTRY_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
         run: echo "${DOCKER_TOKEN}" | docker login --password-stdin --username "${DOCKER_USERNAME}" "${DOCKER_REGISTRY}"
       - name: Push missing release tags
-        if: ${{ 'meeb' == github.actor && github.actor == github.repository_owner && '' != needs.info.outputs.missing-tags }}
+        if: ${{ false && 'meeb' == github.actor && github.actor == github.repository_owner && '' != needs.info.outputs.missing-tags }}
         run: |
           missing='${{ needs.info.outputs.missing-tags }}'
           image='${{ needs.info.outputs.missing-tags-image }}'
@@ -212,15 +284,97 @@ jobs:
                 docker push "${image}:${tag}" ;
               )
             done
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Create cache directory on the runner
+        run: |
+          mkdir -v -p .cache
+          : >> .cache/reset
+          mkdir -v .cache/saved .cache/removed .cache/runner
+      - name: Cache from tubesync stage
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            .cache/saved
+            .cache/runner
+          key: docker-cache-tubesync-${{ hashFiles('.cache/reset') }}-${{ hashFiles('*file', '.github/workflows/ci.yaml') }}
+          restore-keys: |
+            docker-cache-tubesync-${{ hashFiles('.cache/reset') }}-
+      - name: List cache directory on the runner
+        run: |
+          # limited listing when the cache was restored
+          ls -al .cache &&
+          ls -al .cache/* &&
+          ls -al .cache/*/* &&
+          ls -al .cache/*/*/* ||
+          ls -alR .cache
+      - name: Start magic-wormhole services on the runner
+        if: ${{ 'true' != steps.cache.outputs.cache-hit }}
+        id: wormhole
+        run: |
+          rm -rf .cache/runner/wormhole
+          sudo apt-get install python3-venv
+          venv_dir=".cache/runner/${RUNNER_ARCH}/wormhole" &&
+              python3 -m venv --upgrade-deps "${venv_dir}" &&
+              . "${venv_dir}"/bin/activate || exit
+          pip install 'magic-wormhole'
+          # determine the runner IP address
+          _awk_prog='$0 !~ /scope host/ && "inet" == $1 {split($2, P, "/"); print P[1]; exit;}'
+          runner_ip="$( ip addr sh | awk "${_awk_prog}" )"
+          # set variables
+          relay_arg="ws://${runner_ip}:4000/v1"
+          transit_arg="tcp:[${runner_ip}]:4001"
+          # generate the code and receive the first transfer
+          ( wormhole \
+              --appid TubeSync \
+              --relay-url "${relay_arg}" \
+              --transit-helper "${transit_arg}" \
+              receive -a -c 3 \
+              --accept-file -o .cache/incoming >| .cache/receive.out 2>&1 && \
+              mv --backup=numbered -f .cache/incoming/* .cache/saved/ || : ; \
+              mv --backup=numbered -f .cache/saved/*.~[0-9]~ .cache/removed/ || : ; ) &
+          _pid=$!; sleep 1 && grep -e '^Allocated code:' .cache/receive.out | cut -d ' ' -f 3- >| .cache/.wormhole-code
+          cat -v -n .cache/receive.out
+          rm -v -f .cache/receive.out
+          code="$(< .cache/.wormhole-code)"
+          rm -v -f .cache/.wormhole-code
+          # create output variables
+          printf -- '%s=%s\n' >> "$GITHUB_OUTPUT" \
+              code "${code}" \
+              relay "${relay_arg}" \
+              runner_ip "${runner_ip}" \
+              transit "${transit_arg}" ;
+          # receive the saved directories
+          ( cd .cache &&
+              while test -d /proc/"${_pid}" ; do sleep 5 ; done &&
+              while { \
+                wormhole \
+                  --appid TubeSync \
+                  --relay-url "${relay_arg}" \
+                  --transit-helper "${transit_arg}" \
+                  receive \
+                  --accept-file -o incoming "${code}" || : ; \
+              }
+              do
+                  mv --backup=numbered -f incoming/* saved/ || : ;
+                  mv --backup=numbered -f saved/*.~[0-9]~ removed/ || : ;
+                  rm -rf removed/* || : ;
+              done &)
       - name: Build image for `dive`
         id: build-dive-image
         uses: docker/build-push-action@v6
         with:
           build-args: |
+            CI=${{ env.CI }}
             IMAGE_NAME=${{ env.IMAGE_NAME }}
             FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
             FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
+            WORMHOLE_RELAY=${{ env.WORMHOLE_RELAY }}
+            WORMHOLE_TRANSIT=${{ env.WORMHOLE_TRANSIT }}
+          build-contexts: |
+            cache-tubesync=.cache/saved
           cache-from: type=gha
           load: true
           platforms: linux/amd64
@@ -238,21 +392,34 @@ jobs:
             --highestWastedBytes '50M'
       - name: Build and push
         id: build-push
-        timeout-minutes: 60
+        timeout-minutes: 90
         uses: docker/build-push-action@v6
+        env:
+          WORMHOLE_CODE: ${{ steps.wormhole.outputs.code }}
+          WORMHOLE_RELAY: ${{ steps.wormhole.outputs.relay }}
+          WORMHOLE_TRANSIT: ${{ steps.wormhole.outputs.transit }}
         with:
           platforms: linux/amd64,linux/arm64
-          push: ${{ 'success' == needs.test.result && 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'true' || 'false' }}
-          tags: ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:latest
+          push: ${{ 'success' == needs.test.result && 'pull_request' != github.event_name && 'true' || 'false' }}
+          provenance: false
+          tags: ${{ steps.origin.outputs.tag }}
           cache-from: |
-            type=registry,ref=ghcr.io/${{ needs.info.outputs.lowercase-github-actor }}/${{ env.IMAGE_NAME }}:latest
-            type=registry,ref=ghcr.io/${{ needs.info.outputs.lowercase-github-repository_owner }}/${{ env.IMAGE_NAME }}:latest
             type=gha
+            type=registry,ref=${{ steps.origin.outputs.ref }}
+            type=registry,ref=${{ steps.upstream.outputs.ref }}
           cache-to: |
             type=gha,mode=max
-            ${{ 'meeb' == github.repository_owner && 'pull_request' != github.event_name && 'type=inline' || '' }}
           build-args: |
+            CI=${{ env.CI }}
             IMAGE_NAME=${{ env.IMAGE_NAME }}
             FFMPEG_DATE=${{ needs.info.outputs.ffmpeg-date }}
             FFMPEG_VERSION=${{ needs.info.outputs.ffmpeg-version }}
             YTDLP_DATE=${{ fromJSON(needs.info.outputs.ytdlp-latest-release).tag.name }}
+            WORMHOLE_RELAY=${{ env.WORMHOLE_RELAY }}
+            WORMHOLE_TRANSIT=${{ env.WORMHOLE_TRANSIT }}
+          build-contexts: |
+            cache-tubesync=.cache/saved
+          secret-envs: |
+            WORMHOLE_CODE=WORMHOLE_CODE
+            WORMHOLE_RELAY=WORMHOLE_RELAY
+            WORMHOLE_TRANSIT=WORMHOLE_TRANSIT