feat: implement cache:key:prefix and some enhancement to make gcl cache work more similar to gitlab.com (#1556)

Author: ANGkeith

src/job.ts

@@ -16,6 +16,7 @@ import {handler} from "./handler.js";
 import * as yaml from "js-yaml";
 import {Parser} from "./parser.js";
 import {resolveIncludeLocal, validateIncludeLocal} from "./parser-includes.js";
+import globby from "globby";
 import terminalLink from "terminal-link";
 
 const GCL_SHELL_PROMPT_PLACEHOLDER = "<gclShellPromptPlaceholder>";
@@ -470,7 +471,22 @@ export class Job {
         return this.jobData["cache"] || [];
     }
 
-    public async getUniqueCacheName (cwd: string, expanded: {[key: string]: string}, key: any) {
+    public async getUniqueCacheName (cwd: string, expanded: {[key: string]: string}, cacheIndex: number) {
+        const getCachePrefix = (index: number) => {
+            const prefix = this.jobData["cache"][index]["key"]["prefix"];
+            if (prefix) {
+                return `${index}_${Utils.expandText(prefix, expanded)}-`;
+            };
+
+            const filenames = this.jobData["cache"][index]["key"]["files"].map((p: string) => {
+                const expandP = Utils.expandText(p, expanded);
+                return expandP.split(".")[0];
+            }).join("_");
+            return `${index}_${filenames}-`;
+        };
+
+        const key = this.jobData["cache"][cacheIndex].key;
+
         if (typeof key === "string" || key == null) {
             return Utils.expandText(key ?? "default", expanded);
         }
@@ -482,7 +498,7 @@ export class Job {
             }
             return `${cwd}/${path}`;
         });
-        return "md-" + await Utils.checksumFiles(cwd, files);
+        return getCachePrefix(cacheIndex) + await Utils.checksumFiles(cwd, files);
     }
 
     get beforeScripts (): string[] {
@@ -747,8 +763,8 @@ export class Job {
         if (this.imageName(expanded) && !this.argv.mountCache) return [];
 
         const cmd: string[] = [];
-        for (const c of this.cache) {
-            const uniqueCacheName = await this.getUniqueCacheName(this.argv.cwd, expanded, c.key);
+        for (const [index, c] of this.cache.entries()) {
+            const uniqueCacheName = await this.getUniqueCacheName(this.argv.cwd, expanded, index);
             c.paths.forEach((p) => {
                 const path = Utils.expandText(p, expanded);
                 writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright mounting cache} for path ${path}\n`);
@@ -1116,11 +1132,11 @@ export class Job {
         const cwd = this.argv.cwd;
         const stateDir = this.argv.stateDir;
 
-        for (const c of this.cache) {
+        for (const [index, c] of this.cache.entries()) {
             if (!["pull", "pull-push"].includes(c.policy)) return;
 
             const time = process.hrtime();
-            const cacheName = await this.getUniqueCacheName(cwd, expanded, c.key);
+            const cacheName = await this.getUniqueCacheName(cwd, expanded, index);
             const cacheFolder = `${cwd}/${stateDir}/cache/${cacheName}`;
             if (!await fs.pathExists(cacheFolder)) {
                 continue;
@@ -1177,30 +1193,56 @@ export class Job {
         const cachePath = this.imageName(expanded) ? "/cache" : "../../cache";
 
         let time, endTime;
-        for (const c of this.cache) {
+        for (const [index, c] of this.cache.entries()) {
             if (!["push", "pull-push"].includes(c.policy)) return;
             if ("on_success" === c.when && this.jobStatus !== "success") return;
             if ("on_failure" === c.when && this.jobStatus === "success") return;
-            const cacheName = await this.getUniqueCacheName(cwd, expanded, c.key);
+            const cacheName = await this.getUniqueCacheName(cwd, expanded, index);
+
+            let paths = "";
             for (const path of c.paths) {
-                time = process.hrtime();
-                const expandedPath = Utils.expandText(path, expanded).replace(`${expanded.CI_PROJECT_DIR}/`, "");
-                let cmd = "shopt -s globstar nullglob dotglob\n";
-                cmd += `mkdir -p ${cachePath}/${cacheName}\n`;
-                cmd += `rsync -Ra ${expandedPath} ${cachePath}/${cacheName}/. || true\n`;
-
-                await Mutex.exclusive(cacheName, async () => {
-                    await this.copyOut(cmd, stateDir, "cache", []);
-                });
-                endTime = process.hrtime(time);
+                if (!Utils.isSubpath(path, this.argv.cwd, this.argv.cwd)) continue;
 
-                const readdir = await fs.readdir(`${this.argv.cwd}/${stateDir}/cache/${cacheName}`);
-                if (readdir.length === 0) {
-                    writeStreams.stdout(chalk`${this.formattedJobName} {yellow !! no cache was copied for ${path} !!}\n`);
-                } else {
-                    writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright exported cache ${expandedPath} '${cacheName}'} in {magenta ${prettyHrtime(endTime)}}\n`);
+                paths += " " + Utils.expandText(path, expanded).replace(`${expanded.CI_PROJECT_DIR}/`, "");
+            }
+
+            time = process.hrtime();
+            let cmd = "shopt -s globstar nullglob dotglob\n";
+            cmd += `mkdir -p ${cachePath}/${cacheName}\n`;
+            cmd += `rsync -Ra ${paths} ${cachePath}/${cacheName}/. || true\n`;
+
+            await Mutex.exclusive(cacheName, async () => {
+                await this.copyOut(cmd, stateDir, "cache", []);
+            });
+            endTime = process.hrtime(time);
+
+            for (const _path of c.paths) {
+                if (!Utils.isSubpath(_path, this.argv.cwd, this.argv.cwd)) {
+                    writeStreams.stdout(chalk`{yellow WARNING: processPath: artifact path is not a subpath of project directory: ${_path}}\n`);
+                    continue;
+                }
+
+                let path = _path;
+                if (globby.hasMagic(path) && !path.endsWith("*")) {
+                    path = `${path}/**`;
                 }
+
+                let numOfFiles = globby.sync(path, {
+                    dot: true,
+                    onlyFiles: false,
+                    cwd: `${this.argv.cwd}/${stateDir}/cache/${cacheName}`,
+                }).length;
+
+                if (numOfFiles == 0) {
+                    writeStreams.stdout(chalk`{yellow WARNING: ${path}: no matching files. Ensure that the artifact path is relative to the working directory}\n`);
+                    continue;
+                }
+
+                if (!globby.hasMagic(path)) numOfFiles++; // add one because the pattern itself is a folder
+
+                writeStreams.stdout(`${_path}: found ${numOfFiles} artifact files and directories\n`);
             }
+            writeStreams.stdout(chalk`${this.formattedJobName} {magentaBright cache created in '${stateDir}/cache/${cacheName}'} in {magenta ${prettyHrtime(endTime)}}\n`);
         }
     }
 

src/utils.ts

@@ -328,6 +328,25 @@ export class Utils {
         });
     }
 
+    static isSubpath (lhs: string, rhs: string, cwd: string = process.cwd()) {
+        let absLhs = "";
+        if (path.isAbsolute(lhs)) {
+            absLhs = lhs;
+        } else {
+            absLhs = path.resolve(cwd, lhs);
+        }
+
+        let absRhs = "";
+        if (path.isAbsolute(rhs)) {
+            absRhs = rhs;
+        } else {
+            absRhs = path.resolve(cwd, rhs);
+        }
+
+        const relative = path.relative(absRhs, absLhs);
+        return !relative.startsWith("..");
+    }
+
     static async rsyncTrackedFiles (cwd: string, stateDir: string, target: string): Promise<{hrdeltatime: [number, number]}> {
         const time = process.hrtime();
         await fs.mkdirp(`${cwd}/${stateDir}/builds/${target}`);

tests/test-cases/cache-docker/.gitlab-ci.yml

@@ -4,19 +4,58 @@ variables:
 
 produce-cache:
   stage: build
-  image: docker.io/library/alpine
+  image: busybox
   cache:
     key: $VALUE
-    paths: [.cache]
-  script: mkdir -p .cache && touch .cache/file1
+    paths:
+      - .cache
+      - .cache/
+      - .cache/*
+      - .cache/**
+      - .cache2/*/bar
+      - .cache2/**/bar
+      - .cache3
+      - /tmp
+  script:
+    - |
+      # creating 4 files/folder that should match .cache
+      mkdir -p .cache
+      touch .cache/file1
+      touch .cache/file2
+      touch .cache/.hiddenfile
+
+    - |
+      # creating 4 files/folder that should match .cache2/*/bar
+      mkdir -p .cache2/foo/bar
+      touch .cache2/foo/bar/file1
+      touch .cache2/foo/bar/file2
+      touch .cache2/foo/bar/.hiddenfile
+
+    - |
+      # creating files in .cache2 which should not match .cache2/*.bar
+      touch .cache2/a
+      touch .cache2/b
+      mkdir -p .cache2/foo/bazz
+      touch .cache2/foo/bazz/a
 
 consume-cache:
   stage: test
-  image: docker.io/library/alpine
+  image: busybox
   needs: [produce-cache]
   dependencies: [produce-cache]  # testing the absence of artifacts
   cache:
     key: maven
     paths: [.cache]
     policy: pull
-  script: if [ ! -f .cache/file1 ]; then exit 1; fi
+  script:
+    - test -f .cache/file1
+    - test -f .cache/file2
+    - test -f .cache/.hiddenfile
+
+    - test -f .cache2/foo/bar/file1
+    - test -f .cache2/foo/bar/file2
+    - test -f .cache2/foo/bar/.hiddenfile
+
+    - "! test -f .cache2/a"
+    - "! test -f .cache2/b"
+    - "! test -f .cache2/foo/bazz/a"

tests/test-cases/cache-docker/integration.test.ts

@@ -3,18 +3,51 @@ import {handler} from "../../../src/handler.js";
 import fs from "fs-extra";
 import {initSpawnSpy} from "../../mocks/utils.mock.js";
 import {WhenStatics} from "../../mocks/when-statics.js";
+import {basename} from "node:path/posix";
+import {dirname} from "node:path";
 
 beforeAll(() => {
     initSpawnSpy(WhenStatics.all);
 });
 
-test.concurrent("cache-docker <consume-cache> --needs", async () => {
-    await fs.rm("tests/test-cases/cache-docker/.gitlab-ci-local/cache/", {recursive: true, force: true});
+const name = basename(dirname(import.meta.url));
+const cwd = `tests/test-cases/${name}`;
+
+describe(name, () => {
     const writeStreams = new WriteStreamsMock();
-    await handler({
-        cwd: "tests/test-cases/cache-docker",
-        job: ["consume-cache"],
-        needs: true,
-    }, writeStreams);
-    expect(writeStreams.stdoutLines.join("\n")).toEqual(expect.stringMatching(/exported cache .cache 'maven'/));
+    beforeAll(async () => {
+        await fs.rm(`${cwd}/.gitlab-ci-local/cache/`, {recursive: true, force: true}); // to ensure that the cache from previous runs gets deleted
+
+        await handler({
+            cwd,
+            noColor: true,
+            file: ".gitlab-ci.yml",
+        }, writeStreams);
+    });
+
+    it("should show export cache message", () => {
+        expect(writeStreams.stdoutLines.join("\n")).toContain("produce-cache cache created in '.gitlab-ci-local/cache/maven'");
+    });
+
+    it("should show the correct number of files that's exported", () => {
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache: found 4 artifact files and directories");
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache/: found 4 artifact files and directories");
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache/*: found 3 artifact files and directories");
+        // NOTE: gitlab.com shows .cache/**: found 7 matching artifact files and directories
+        //       i can't make any sense of it, i think it's probably a bug?
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache/**: found 3 artifact files and directories");
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache2/*/bar: found 4 artifact files and directories");
+        expect(writeStreams.stdoutLines.join("\n")).toContain(".cache2/**/bar: found 4 artifact files and directories");
+        expect(writeStreams.stdoutLines.join("\n")).toContain("WARNING: .cache3: no matching files. Ensure that the artifact path is relative to the working directory");
+        expect(writeStreams.stdoutLines.join("\n")).toContain("WARNING: processPath: artifact path is not a subpath of project directory: /tmp");
+    });
+
+    it("should export cache to local fs", () => {
+        expect(fs.existsSync(`${cwd}/.gitlab-ci-local/cache/maven`)).toBe(true);
+    });
+
+    it("should be pull the cache with the expected content", () => {
+        // The assertions are done via the consume-cache job's script
+        expect(writeStreams.stdoutLines.join("\n")).toContain("PASS  consume-cache");
+    });
 });

tests/test-cases/cache-key-files/integration.test.ts

@@ -3,13 +3,12 @@ import {handler} from "../../../src/handler.js";
 import fs from "fs-extra";
 import {initSpawnSpy} from "../../mocks/utils.mock.js";
 import {WhenStatics} from "../../mocks/when-statics.js";
-import chalk from "chalk";
 
 beforeAll(() => {
     initSpawnSpy(WhenStatics.all);
 });
 
-test.concurrent("cache-key-files <consume-cache> --shell-isolation --needs", async () => {
+test("cache-key-files <consume-cache> --shell-isolation --needs", async () => {
     await fs.rm("tests/test-cases/cache-key-files/.gitlab-ci-local/cache/", {recursive: true, force: true});
     const writeStreams = new WriteStreamsMock();
     await handler({
@@ -27,24 +26,22 @@ test("cache-key-files <cache-key-file referencing $CI_PROJECT_DIR>", async () =>
     await handler({
         cwd: "tests/test-cases/cache-key-files",
         job: ["cache-key-file referencing $CI_PROJECT_DIR"],
+        noColor: true,
     }, writeStreams);
 
-    const expected = [
-        chalk`{blueBright cache-key-file referencing $CI_PROJECT_DIR} {magentaBright exported cache fakepackage.json 'md-8aaa60c7b3009df8ce6973111af131bbcde5636e'}`,
-    ];
+    const expected = "cache-key-file referencing $CI_PROJECT_DIR cache created in '.gitlab-ci-local/cache/0_/builds/gcl/test-project/fakepackage-8aaa60c7b3009df8ce6973111af131bbcde5636e'";
 
-    expect(writeStreams.stdoutLines.join("\n")).toContain(expected.join("\n"));
+    expect(writeStreams.stdoutLines.join("\n")).toContain(expected);
 });
 
 test("cache-key-files <cache-key-file file dont exist>", async () => {
     const writeStreams = new WriteStreamsMock();
     await handler({
         cwd: "tests/test-cases/cache-key-files",
         job: ["cache-key-file file dont exist"],
+        noColor: true,
     }, writeStreams);
 
-    const expected = [
-        chalk`{blueBright cache-key-file file dont exist} {magentaBright exported cache /tmp 'md-18bbe9d7603e540e28418cf4a072938ac477a2c1'}`,
-    ];
-    expect(writeStreams.stdoutLines.join("\n")).toContain(expected.join("\n"));
+    const expected = "cache-key-file file dont exist cache created in '.gitlab-ci-local/cache/0_no-such-file-18bbe9d7603e540e28418cf4a072938ac477a2c1'";
+    expect(writeStreams.stdoutLines.join("\n")).toContain(expected);
 });

tests/test-cases/cache-key-prefix/.gitlab-ci-1.yml

@@ -0,0 +1,26 @@
+---
+development-cache:
+  image: busybox
+  script:
+    - mkdir -p cached
+    - echo "development" > cached/value.txt
+  cache:
+    paths:
+      - cached/
+    key:
+      prefix: development
+      files:
+        - Dockerfile
+
+production-cache:
+  image: busybox
+  script:
+    - mkdir -p cached
+    - echo "production" > cached/value.txt
+  cache:
+    paths:
+      - cached/
+    key:
+      prefix: production
+      files:
+        - Dockerfile

tests/test-cases/cache-key-prefix/.gitlab-ci-2.yml

@@ -0,0 +1,12 @@
+---
+rspec:
+  image: busybox
+  script:
+    - echo "This rspec job uses a cache."
+  cache:
+    key:
+      files:
+        - Gemfile.lock
+      prefix: $CI_JOB_NAME
+    paths:
+      - vendor/ruby