RHAIENG-2860: Move custom packages from artifact to .json files

ysok · ysok · commit 94f89bc9dbfb · 2026-03-02T13:41:38.000-05:00
Add hermetic Konflux support: Tekton prefetch + custom-packages (codeload URLs),
drop generic artifacts for css-parser/watcher, update rewrite-npm-urls for
codeload and integrity; document Hermeto layout in custom-packages README.
diff --git a/.tekton/odh-workbench-codeserver-datascience-cpu-py312-ubi9-pull-request.yaml b/.tekton/odh-workbench-codeserver-datascience-cpu-py312-ubi9-pull-request.yaml
@@ -172,6 +172,9 @@ spec:
       type: npm
     - path: codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/lib/vscode/extensions/microsoft-authentication
       type: npm
+    # GitHub tarball deps (git+ssh not available in Konflux); same URLs as artifacts.in.yaml
+    - path: codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages
+      type: npm
   pipelineRef:
     name: multiarch-combined-pipeline
   taskRunTemplate:
diff --git a/.tekton/odh-workbench-codeserver-datascience-cpu-py312-ubi9-push.yaml b/.tekton/odh-workbench-codeserver-datascience-cpu-py312-ubi9-push.yaml
@@ -169,6 +169,9 @@ spec:
       type: npm
     - path: codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/lib/vscode/extensions/microsoft-authentication
       type: npm
+    # GitHub tarball deps (git+ssh not available in Konflux); same URLs as artifacts.in.yaml
+    - path: codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages
+      type: npm
 
   pipelineRef:
     name: multiarch-combined-pipeline
diff --git a/codeserver/ubi9-python-3.12/Dockerfile.cpu b/codeserver/ubi9-python-3.12/Dockerfile.cpu
@@ -81,10 +81,6 @@ RUN dnf install -y \
         /cachi2/output/deps/generic/nfpm-2.44.1-1.$(uname -m).rpm && \
     dnf clean all
 
-# There was limitation on Hermeto, it can't fetch npm packages using git/ssh protocol.
-# To work around this, need to fetch some npm packages as generic artifacts and copy to npm directory.
-RUN cp /cachi2/output/deps/generic/npm/* /cachi2/output/deps/npm/
-
 # [HERMETIC] Git metadata needed by code-server's build scripts (version detection, submodules).
 COPY .git /root/.git
 # [HERMETIC] Rewrite script: used by setup-offline-binaries.sh to rewrite npm
diff --git a/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu b/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu
@@ -81,10 +81,6 @@ RUN dnf install -y \
         /cachi2/output/deps/generic/nfpm-2.44.1-1.$(uname -m).rpm && \
     dnf clean all
 
-# There was limitation on Hermeto, it can't fetch npm packages using git/ssh protocol.
-# To work around this, need to fetch some npm packages as generic artifacts and copy to npm directory.
-RUN cp /cachi2/output/deps/generic/npm/* /cachi2/output/deps/npm/
-
 # [HERMETIC] Git metadata needed by code-server's build scripts (version detection, submodules).
 COPY .git /root/.git
 # [HERMETIC] Rewrite script: used by setup-offline-binaries.sh to rewrite npm
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/odh/artifacts.in.yaml b/codeserver/ubi9-python-3.12/prefetch-input/odh/artifacts.in.yaml
@@ -59,16 +59,6 @@ input:
     - url: https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/4.18.33/openshift-client-linux.tar.gz
       filename: openshift-client-linux-s390x.tar.gz
 
-    # GitHub tarball dependency for @emmetio/css-parser (used by VSCode emmet extension)
-    # Filename must use .tgz to match rewrite-npm-urls.sh and download-npm.sh conventions.
-    - url: https://codeload.github.com/ramya-rao-a/css-parser/tar.gz/370c480ac103bd17c7bcfb34bf5d577dc40d3660
-      filename: npm/ramya-rao-a-css-parser-370c480ac103bd17c7bcfb34bf5d577dc40d3660.tgz
-
-    # GitHub tarball dependency for @parcel/watcher (v4.106.3 uses git ref instead of npm registry)
-    # Filename must use .tgz to match rewrite-npm-urls.sh and download-npm.sh conventions.
-    - url: https://codeload.github.com/parcel-bundler/watcher/tar.gz/1ca032aa8339260a8a3bcf825c3a1a71e3e43542
-      filename: npm/parcel-bundler-watcher-1ca032aa8339260a8a3bcf825c3a1a71e3e43542.tgz
-
     # VSCode marketplace extensions (built-in extensions bundled with code-server)
     - url: https://github.com/microsoft/vscode-js-debug-companion/releases/download/v1.1.3/ms-vscode.js-debug-companion.1.1.3.vsix
       filename: ms-vscode.js-debug-companion.1.1.3.vsix
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/odh/artifacts.lock.yaml b/codeserver/ubi9-python-3.12/prefetch-input/odh/artifacts.lock.yaml
@@ -68,12 +68,6 @@ artifacts:
   - download_url: https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/4.18.33/openshift-client-linux.tar.gz
     checksum: sha256:2162b0a596f5f29fbd290382b52ccf60bf6e9739346b52698a55ac77396989d9
     filename: openshift-client-linux-s390x.tar.gz
-  - download_url: https://codeload.github.com/ramya-rao-a/css-parser/tar.gz/370c480ac103bd17c7bcfb34bf5d577dc40d3660
-    checksum: sha256:6004ae05ad17931a62a72b27ebf1dab54fca6e72c0a9f0b7925a8b315bbe776d
-    filename: npm/ramya-rao-a-css-parser-370c480ac103bd17c7bcfb34bf5d577dc40d3660.tgz
-  - download_url: https://codeload.github.com/parcel-bundler/watcher/tar.gz/1ca032aa8339260a8a3bcf825c3a1a71e3e43542
-    checksum: sha256:2d22d1ade4bc4d7bc96c1b61888a2b9b7e50d01740d75c92a865834ce7c4359e
-    filename: npm/parcel-bundler-watcher-1ca032aa8339260a8a3bcf825c3a1a71e3e43542.tgz
   - download_url: https://github.com/microsoft/vscode-js-debug-companion/releases/download/v1.1.3/ms-vscode.js-debug-companion.1.1.3.vsix
     checksum: sha256:7380a890787452f14b2db7835dfa94de538caf358ebc263f9d46dd68ac52de93
     filename: ms-vscode.js-debug-companion.1.1.3.vsix
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/README.md b/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/README.md
@@ -0,0 +1,10 @@
+# custom-packages
+
+Declares npm dependencies that **Konflux cannot fetch via git+ssh** (hermetic build limitation). These are provided as **HTTPS GitHub tarball URLs** so the pipeline prefetches them as `type: npm` into `deps/npm/` directly.
+
+- **@emmetio/css-parser** — used by VSCode emmet extension (from `ramya-rao-a/css-parser`).
+- **@parcel/watcher** — used by VSCode (from `parcel-bundler/watcher`).
+
+The main code-server/VSCode lockfiles reference these via git+ssh; `rewrite-npm-urls.sh` rewrites them to `file:///cachi2/output/deps/npm/owner-repo-<commit>.tgz`, which are supplied by this prefetch.
+
+**Note (Hermeto):** If Konflux uses [Hermeto](https://hermetoproject.github.io/) for npm prefetch, codeload.github.com tarballs are stored under `deps/npm/external-<pkg>/<pkg>-external-sha512-<contenthash>.tgz`, not as flat `owner-repo-<ref>.tgz`. In that case the pipeline should use Hermeto’s inject-files output (lockfile already pointing to those paths) and must not run `rewrite-npm-urls.sh` for these deps, or the build will look for the wrong filenames.
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/package-lock.json b/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/package-lock.json
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/package.json b/codeserver/ubi9-python-3.12/prefetch-input/patches/code-server-v4.106.3/custom-packages/package.json
@@ -0,0 +1,10 @@
+{
+  "name": "code-server-hermetic-git-deps",
+  "version": "1.0.0",
+  "description": "Pins npm deps that Konflux cannot fetch via git+ssh (GitHub tarball URLs only). Prefetched into deps/npm/ and consumed by rewrite-npm-urls.sh when building code-server/VSCode.",
+  "private": true,
+  "dependencies": {
+    "@emmetio/css-parser": "https://codeload.github.com/ramya-rao-a/css-parser/tar.gz/370c480ac103bd17c7bcfb34bf5d577dc40d3660",
+    "@parcel/watcher": "https://codeload.github.com/parcel-bundler/watcher/tar.gz/1ca032aa8339260a8a3bcf825c3a1a71e3e43542"
+  }
+}
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/rhds/artifacts.in.yaml b/codeserver/ubi9-python-3.12/prefetch-input/rhds/artifacts.in.yaml
@@ -64,16 +64,6 @@ input:
     - url: https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/4.18.33/openshift-client-linux.tar.gz
       filename: openshift-client-linux-s390x.tar.gz
 
-    # GitHub tarball dependency for @emmetio/css-parser (used by VSCode emmet extension)
-    # Filename must use .tgz to match rewrite-npm-urls.sh and download-npm.sh conventions.
-    - url: https://codeload.github.com/ramya-rao-a/css-parser/tar.gz/370c480ac103bd17c7bcfb34bf5d577dc40d3660
-      filename: npm/ramya-rao-a-css-parser-370c480ac103bd17c7bcfb34bf5d577dc40d3660.tgz
-
-    # GitHub tarball dependency for @parcel/watcher (v4.106.3 uses git ref instead of npm registry)
-    # Filename must use .tgz to match rewrite-npm-urls.sh and download-npm.sh conventions.
-    - url: https://codeload.github.com/parcel-bundler/watcher/tar.gz/1ca032aa8339260a8a3bcf825c3a1a71e3e43542
-      filename: npm/parcel-bundler-watcher-1ca032aa8339260a8a3bcf825c3a1a71e3e43542.tgz
-
     # VSCode marketplace extensions (built-in extensions bundled with code-server)
     - url: https://github.com/microsoft/vscode-js-debug-companion/releases/download/v1.1.3/ms-vscode.js-debug-companion.1.1.3.vsix
       filename: ms-vscode.js-debug-companion.1.1.3.vsix
diff --git a/codeserver/ubi9-python-3.12/prefetch-input/rhds/artifacts.lock.yaml b/codeserver/ubi9-python-3.12/prefetch-input/rhds/artifacts.lock.yaml
@@ -83,12 +83,6 @@ artifacts:
   - download_url: https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/4.18.33/openshift-client-linux.tar.gz
     checksum: sha256:2162b0a596f5f29fbd290382b52ccf60bf6e9739346b52698a55ac77396989d9
     filename: openshift-client-linux-s390x.tar.gz
-  - download_url: https://codeload.github.com/ramya-rao-a/css-parser/tar.gz/370c480ac103bd17c7bcfb34bf5d577dc40d3660
-    checksum: sha256:6004ae05ad17931a62a72b27ebf1dab54fca6e72c0a9f0b7925a8b315bbe776d
-    filename: npm/ramya-rao-a-css-parser-370c480ac103bd17c7bcfb34bf5d577dc40d3660.tgz
-  - download_url: https://codeload.github.com/parcel-bundler/watcher/tar.gz/1ca032aa8339260a8a3bcf825c3a1a71e3e43542
-    checksum: sha256:2d22d1ade4bc4d7bc96c1b61888a2b9b7e50d01740d75c92a865834ce7c4359e
-    filename: npm/parcel-bundler-watcher-1ca032aa8339260a8a3bcf825c3a1a71e3e43542.tgz
   - download_url: https://github.com/microsoft/vscode-js-debug-companion/releases/download/v1.1.3/ms-vscode.js-debug-companion.1.1.3.vsix
     checksum: sha256:7380a890787452f14b2db7835dfa94de538caf358ebc263f9d46dd68ac52de93
     filename: ms-vscode.js-debug-companion.1.1.3.vsix
diff --git a/scripts/lockfile-generators/rewrite-npm-urls.sh b/scripts/lockfile-generators/rewrite-npm-urls.sh
@@ -7,10 +7,11 @@
 #   file:///cachi2/output/deps/npm/<filename>
 #
 # Handles:
-#   1. HTTPS registry URLs  (https://registry.npmjs.org/.../-/name-ver.tgz)
-#   2. git+ssh:// URLs       (git+ssh://git@github.com/owner/repo.git#hash)
-#   3. git+https:// URLs     (git+https://github.com/owner/repo.git#hash)
-#   4. GitHub shortname refs  (owner/repo#hash-or-branch  in dependency values)
+#   1. HTTPS registry URLs   (https://registry.npmjs.org/.../-/name-ver.tgz)
+#   2. codeload.github.com  (https://codeload.github.com/owner/repo/tar.gz/ref)
+#   3. git+ssh:// URLs      (git+ssh://git@github.com/owner/repo.git#hash)
+#   4. git+https:// URLs   (git+https://github.com/owner/repo.git#hash)
+#   5. GitHub shortname refs (owner/repo#hash-or-branch  in dependency values)
 #
 # Usage:
 #   ./rewrite-npm-urls.sh .                          # run on the current directory
@@ -85,30 +86,36 @@ find_target_files() {
 # Rewrite types:
 #   1. Remove integrity hashes for git-resolved deps
 #   2. Registry URLs   https://registry.npmjs.org/...  → file:///cachi2/...
-#   3. git+ssh/https   git+ssh://...#hash              → file:///cachi2/...-hash.tgz
-#   4. Shortname refs  "owner/repo#ref"                → file:///cachi2/...-ref.tgz
+#   3. codeload.github.com (custom-packages / hermetic git workaround) → owner-repo-ref.tgz
+#   4. git+ssh/https   git+ssh://...#hash              → file:///cachi2/...-hash.tgz
+#   5. Shortname refs  "owner/repo#ref"                → file:///cachi2/...-ref.tgz
 # =============================================================================
 process_file() {
     local file="$1"
     log_step "Processing: ${file}"
 
-    # 1) Strip integrity hashes for git deps (npm pack is not reproducible,
-    #    so offline tarballs will have different hashes — safe to remove
-    #    because content is pinned by git commit hash).
+    # 1) Strip integrity hashes for git and codeload deps (tarballs can be
+    #    non-reproducible or fetched by cachi2 with different bytes — safe to
+    #    remove because content is pinned by git ref / URL).
     perl -i -ne '
         if ($s) { $s=0; next if /^\s*"integrity":\s*"/ }
-        $s=1 if /"resolved":\s*"git\+/; print
+        $s=1 if /"resolved":\s*"(git\+|https:\/\/codeload\.github\.com)/; print
     ' "${file}"
 
     # 2) Registry URLs (scoped @scope/pkg and unscoped pkg in one pass).
     #    Scoped: @types/cookie-parser/-/cookie-parser-1.4.7.tgz → types-cookie-parser-1.4.7.tgz
     #    Unscoped: express/-/express-4.19.2.tgz                  → express-4.19.2.tgz
     perl -i -pe 's!"resolved": "https?://[^"]*?/(?:\@([^/]+)/)?[^/]+/-/([^"]+)"! "\"resolved\": \"'"${CACHI2_BASE}"'/" . ($1 ? "$1-$2" : "$2") . "\"" !ge' "${file}"
 
-    # 3) git+ssh and git+https URLs → owner-repo-hash.tgz
+    # 3) codeload.github.com URLs (e.g. custom-packages) → owner-repo-ref.tgz
+    #    Same filename convention as git+ssh so deps/npm/ has one naming scheme.
+    #    Match both "resolved": "https://..." (lockfile) and "pkg": "https://..." (package.json deps).
+    perl -i -pe 's!: "https://codeload\.github\.com/([^/]+)/([^/]+)/tar\.gz/([^"]+)"!: "'"${CACHI2_BASE}"'/$1-$2-$3.tgz"!g' "${file}"
+
+    # 4) git+ssh and git+https URLs → owner-repo-hash.tgz
     perl -i -pe 's!"resolved": "git\+(ssh://git\@|https://)github\.com/([^/]+)/([^.#"]+)(?:\.git)?#([^"]+)"!"resolved": "'"${CACHI2_BASE}"'/$2-$3-$4.tgz"!g' "${file}"
 
-    # 4) GitHub shortname refs: "owner/repo#ref" → cachi2 path
+    # 5) GitHub shortname refs: "owner/repo#ref" → cachi2 path
     perl -i -pe 's!": "([a-zA-Z0-9_.-]+)/([a-zA-Z0-9_.-]+)#([^"]+)"!": "'"${CACHI2_BASE}"'/$1-$2-$3.tgz"!g' "${file}"
 
     TOTAL_FILES_PROCESSED=$((TOTAL_FILES_PROCESSED + 1))
