Separate task(s) for pre-fetching

This logically separates the part that should be implemented in cachi2
in the future, per:
https://rpm-software-management.github.io/mock/feature-hermetic-builds

Author: praiskup

pipeline/build-rpm-package.yaml

@@ -242,8 +242,8 @@ spec:
           value: $(tasks.get-rpm-sources.results.skip-mpc-tasks.deps-x86_64)
         - name: script-environment-image
           value: $(params.script-environment-image)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
         - name: ociStorage
           value: $(params.ociStorage).calculation-x86_64
         - name: ociArtifactExpiresAfter
@@ -259,7 +259,7 @@ spec:
             value: task/calculate-deps.yaml
     - name: rpmbuild-x86-64
       runAfter:
-        - calculate-deps-x86-64
+        - prefetch-deps-x86-64
       timeout: "72h"
       params:
         - name: package-name
@@ -274,10 +274,12 @@ spec:
           value: $(params.script-environment-image)
         - name: hermetic
           value: $(params.hermetic)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
-        - name: calculation-artifact
-          value: $(tasks.calculate-deps-x86-64.results.calculation-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-x86-64.results.lockfile-artifact)
+        - name: build-deps-artifact
+          value: $(tasks.prefetch-deps-x86-64.results.build-deps-artifact)
         - name: ociStorage
           value: $(params.ociStorage).rpmbuild-x86_64
         - name: ociArtifactExpiresAfter
@@ -305,8 +307,8 @@ spec:
           value: $(tasks.get-rpm-sources.results.skip-mpc-tasks.deps-aarch64)
         - name: script-environment-image
           value: $(params.script-environment-image)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
         - name: ociStorage
           value: $(params.ociStorage).calculation-aarch64
         - name: ociArtifactExpiresAfter
@@ -322,7 +324,7 @@ spec:
             value: task/calculate-deps.yaml
     - name: rpmbuild-aarch64
       runAfter:
-        - calculate-deps-aarch64
+        - prefetch-deps-aarch64
       timeout: "72h"
       params:
         - name: package-name
@@ -337,10 +339,12 @@ spec:
           value: $(params.script-environment-image)
         - name: hermetic
           value: $(params.hermetic)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
-        - name: calculation-artifact
-          value: $(tasks.calculate-deps-aarch64.results.calculation-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-aarch64.results.lockfile-artifact)
+        - name: build-deps-artifact
+          value: $(tasks.prefetch-deps-aarch64.results.build-deps-artifact)
         - name: ociStorage
           value: $(params.ociStorage).rpmbuild-aarch64
         - name: ociArtifactExpiresAfter
@@ -368,8 +372,8 @@ spec:
           value: $(tasks.get-rpm-sources.results.skip-mpc-tasks.deps-s390x)
         - name: script-environment-image
           value: $(params.script-environment-image)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
         - name: ociStorage
           value: $(params.ociStorage).calculation-s390x
         - name: ociArtifactExpiresAfter
@@ -385,7 +389,7 @@ spec:
             value: task/calculate-deps.yaml
     - name: rpmbuild-s390x
       runAfter:
-        - calculate-deps-s390x
+        - prefetch-deps-s390x
       timeout: "72h"
       params:
         - name: package-name
@@ -400,10 +404,12 @@ spec:
           value: $(params.script-environment-image)
         - name: hermetic
           value: $(params.hermetic)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
-        - name: calculation-artifact
-          value: $(tasks.calculate-deps-s390x.results.calculation-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-s390x.results.lockfile-artifact)
+        - name: build-deps-artifact
+          value: $(tasks.prefetch-deps-s390x.results.build-deps-artifact)
         - name: ociStorage
           value: $(params.ociStorage).rpmbuild-s390x
         - name: ociArtifactExpiresAfter
@@ -431,8 +437,8 @@ spec:
           value: $(tasks.get-rpm-sources.results.skip-mpc-tasks.deps-ppc64le)
         - name: script-environment-image
           value: $(params.script-environment-image)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
         - name: ociStorage
           value: $(params.ociStorage).calculation-ppc64le
         - name: ociArtifactExpiresAfter
@@ -446,9 +452,109 @@ spec:
             value: $(params.self-ref-revision)
           - name: pathInRepo
             value: task/calculate-deps.yaml
-    - name: rpmbuild-ppc64le
+    - name: prefetch-deps-x86-64
+      runAfter:
+        - calculate-deps-x86-64
+      params:
+        - name: script-environment-image
+          value: $(params.script-environment-image)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: ociStorage
+          value: $(params.ociStorage).rpmdeps-x86_64
+        - name: ociArtifactExpiresAfter
+          value: 14d
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-x86-64.results.lockfile-artifact)
+        - name: hermetic
+          value: $(params.hermetic)
+      taskRef:
+        resolver: git
+        params:
+          - name: url
+            value: $(params.self-ref-url)
+          - name: revision
+            value: $(params.self-ref-revision)
+          - name: pathInRepo
+            value: task/prefetch-rpmbuild-deps.yaml
+    - name: prefetch-deps-aarch64
+      runAfter:
+        - calculate-deps-aarch64
+      params:
+        - name: script-environment-image
+          value: $(params.script-environment-image)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: ociStorage
+          value: $(params.ociStorage).rpmdeps-aarch64
+        - name: ociArtifactExpiresAfter
+          value: 14d
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-aarch64.results.lockfile-artifact)
+        - name: hermetic
+          value: $(params.hermetic)
+      taskRef:
+        resolver: git
+        params:
+          - name: url
+            value: $(params.self-ref-url)
+          - name: revision
+            value: $(params.self-ref-revision)
+          - name: pathInRepo
+            value: task/prefetch-rpmbuild-deps.yaml
+    - name: prefetch-deps-s390x
+      runAfter:
+        - calculate-deps-s390x
+      params:
+        - name: script-environment-image
+          value: $(params.script-environment-image)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: ociStorage
+          value: $(params.ociStorage).rpmdeps-s390x
+        - name: ociArtifactExpiresAfter
+          value: 14d
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-s390x.results.lockfile-artifact)
+        - name: hermetic
+          value: $(params.hermetic)
+      taskRef:
+        resolver: git
+        params:
+          - name: url
+            value: $(params.self-ref-url)
+          - name: revision
+            value: $(params.self-ref-revision)
+          - name: pathInRepo
+            value: task/prefetch-rpmbuild-deps.yaml
+    - name: prefetch-deps-ppc64le
       runAfter:
         - calculate-deps-ppc64le
+      params:
+        - name: script-environment-image
+          value: $(params.script-environment-image)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: ociStorage
+          value: $(params.ociStorage).rpmdeps-ppc64le
+        - name: ociArtifactExpiresAfter
+          value: 14d
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-ppc64le.results.lockfile-artifact)
+        - name: hermetic
+          value: $(params.hermetic)
+      taskRef:
+        resolver: git
+        params:
+          - name: url
+            value: $(params.self-ref-url)
+          - name: revision
+            value: $(params.self-ref-revision)
+          - name: pathInRepo
+            value: task/prefetch-rpmbuild-deps.yaml
+    - name: rpmbuild-ppc64le
+      runAfter:
+        - prefetch-deps-ppc64le
       timeout: "72h"
       params:
         - name: package-name
@@ -463,10 +569,12 @@ spec:
           value: $(params.script-environment-image)
         - name: hermetic
           value: $(params.hermetic)
-        - name: dependencies-artifact
-          value: $(tasks.get-rpm-sources.results.dependencies-artifact)
-        - name: calculation-artifact
-          value: $(tasks.calculate-deps-ppc64le.results.calculation-artifact)
+        - name: sources-with-lookaside-artifact
+          value: $(tasks.get-rpm-sources.results.sources-with-lookaside-artifact)
+        - name: lockfile-artifact
+          value: $(tasks.calculate-deps-ppc64le.results.lockfile-artifact)
+        - name: build-deps-artifact
+          value: $(tasks.prefetch-deps-ppc64le.results.build-deps-artifact)
         - name: ociStorage
           value: $(params.ociStorage).rpmbuild-ppc64le
         - name: ociArtifactExpiresAfter

task/calculate-deps.yaml

@@ -29,13 +29,13 @@ spec:
       name: script-environment-image
       type: string
     - description: The Trusted Artifact URI pointing to the artifact with the rpm deps and source.
-      name: dependencies-artifact
+      name: sources-with-lookaside-artifact
       type: string
     - name: ociStorage
       description: The OCI repository where the Trusted Artifacts are stored.
       type: string
   results:
-    - name: calculation-artifact
+    - name: lockfile-artifact
       description: The Trusted Artifact URI pointing to the artifact with the result of the deps calculation.
       type: string
   stepTemplate:
@@ -47,7 +47,7 @@ spec:
       image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:9b180776a41d9a22a1c51539f1647c60defbbd55b44bbebdd4130e33512d8b0d
       args:
         - use
-        - $(params.dependencies-artifact)=/var/workdir/source
+        - $(params.sources-with-lookaside-artifact)=/var/workdir/source
     - name: mock-build
       image: "quay.io/redhat-appstudio/multi-platform-runner:01c7670e81d5120347cf0ad13372742489985e5f@sha256:246adeaaba600e207131d63a7f706cffdcdc37d8f600c56187123ec62823ff44"
       script: |
@@ -122,17 +122,10 @@ spec:
         remote_cmd tar xfO $HOMEDIR/results/chroot_scan.tar.gz
         $success
 
-        remote_cmd podman run -v "$HOMEDIR/results:/results" \
-                              --privileged --rm -ti "$mock_img" \
-            mock-hermetic-repo \
-            --lockfile /results/buildroot_lock.json \
-            --output-repo /results/buildroot_repo
-
-        resultdir=$workdir/results/$arch/results
+        resultdir=$workdir/lock-file
         mkdir -p "$resultdir"
         # Send only repo and lockfile, ignore other artifacts
         receive "$HOMEDIR/results/buildroot_lock.json" "$resultdir/buildroot_lock.json"
-        receive "$HOMEDIR/results/buildroot_repo" "$resultdir"
         cat "$resultdir/buildroot_lock.json"
       volumeMounts:
         - mountPath: /ssh
@@ -147,7 +140,7 @@ spec:
         - create
         - --store
         - $(params.ociStorage)
-        - $(results.calculation-artifact.path)=/var/workdir/results
+        - $(results.lockfile-artifact.path)=/var/workdir/lock-file
   volumes:
     - name: ssh
       secret:

task/get-rpm-sources.yaml

@@ -31,8 +31,10 @@ spec:
       name: build-architectures
       type: array
   results:
-    - name: dependencies-artifact
-      description: The Trusted Artifact URI pointing to the artifact with the rpm deps and source.
+    - name: sources-with-lookaside-artifact
+      description: |
+        The Trusted Artifact URI pointing to the artifact with git checkout and
+        files from the lookaside cache.
       type: string
     - name: skip-mpc-tasks
       description: |
@@ -110,7 +112,7 @@ spec:
         - create
         - --store
         - $(params.ociStorage)
-        - $(results.dependencies-artifact.path)=/var/workdir/source
+        - $(results.sources-with-lookaside-artifact.path)=/var/workdir/source
       env:
         - name: IMAGE_EXPIRES_AFTER
           value: $(params.ociArtifactExpiresAfter)

task/prefetch-rpmbuild-deps.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  annotations:
+    tekton.dev/tags: rpm-build
+  name: download-builddeps
+spec:
+  description: |-
+    Download all build-time dependencies for an RPM build, per given lockfile.
+  params:
+    - description: The Trusted Artifact URI pointing to the lockfile location.
+      name: lockfile-artifact
+      type: string
+    - name: ociStorage
+      description: The OCI repository where the Trusted Artifacts are stored.
+      type: string
+    - name: ociArtifactExpiresAfter
+      description: How long Trusted Artifacts should be retained
+      type: string
+    - description: RPM Build environment OCI image to run scripts in
+      name: script-environment-image
+      type: string
+    - description: Is the build hermetic?
+      name: hermetic
+      type: string
+  results:
+    - name: build-deps-artifact
+      description: |
+        The Trusted Artifact URI pointing to the artifact with the result of the
+        build-time deps downloads.
+      type: string
+  stepTemplate:
+    volumeMounts:
+      - mountPath: /var/workdir
+        name: workdir
+  steps:
+    - name: use-trusted-artifact
+      image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:9b180776a41d9a22a1c51539f1647c60defbbd55b44bbebdd4130e33512d8b0d
+      args:
+        - use
+        - $(params.lockfile-artifact)=/var/workdir/lockfile
+    - name: download
+      image: $(params.script-environment-image)
+      script: |
+        set -ex
+        outputdir=/var/workdir/build-deps
+        mkdir -p "$outputdir"
+        lockfile=/var/workdir/lockfile/buildroot_lock.json
+        # non-hermetic mode = we don't prefetch
+        test $(params.hermetic) = true || exit 0
+        # no lockfile = calculate-deps skipped = architecture skipped
+        test -f "$lockfile" || exit 0
+        mock-hermetic-repo \
+          --lockfile /var/workdir/lockfile/buildroot_lock.json \
+          --output-repo "$outputdir"
+    - name: create-trusted-artifact
+      image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:9b180776a41d9a22a1c51539f1647c60defbbd55b44bbebdd4130e33512d8b0d
+      args:
+        - create
+        - --store
+        - $(params.ociStorage)
+        - $(results.build-deps-artifact.path)=/var/workdir/build-deps
+      env:
+        - name: IMAGE_EXPIRES_AFTER
+          value: $(params.ociArtifactExpiresAfter)
+  volumes:
+    - name: workdir
+      emptyDir: {}