fix: use regex instead of globby for include:local wildcard file paths implementation (#1548)

* fix: use regex instead of globby for include:local wildcard file paths implementation

* refactor: fix sonarqube Prefer using nullish coalescing operator (`??`) instead of a logical or (`||`), as it is a safer operator.

Author: ANGkeith

src/job.ts

@@ -15,8 +15,7 @@ import {GitlabRunnerCPUsPresetValue, GitlabRunnerMemoryPresetValue, GitlabRunner
 import {handler} from "./handler.js";
 import * as yaml from "js-yaml";
 import {Parser} from "./parser.js";
-import globby from "globby";
-import {validateIncludeLocal} from "./parser-includes.js";
+import {resolveIncludeLocal, validateIncludeLocal} from "./parser-includes.js";
 import terminalLink from "terminal-link";
 
 const GCL_SHELL_PROMPT_PLACEHOLDER = "<gclShellPromptPlaceholder>";
@@ -1499,13 +1498,13 @@ export class Job {
             if (include["local"]) {
                 const expandedInclude = Utils.expandText(include["local"], this._variables);
                 validateIncludeLocal(expandedInclude);
-                const files = await globby(expandedInclude.replace(/^\//, ""), {dot: true, cwd});
+                const files = resolveIncludeLocal(expandedInclude, cwd);
                 if (files.length == 0) {
                     throw new AssertionError({message: `Local include file \`${include["local"]}\` specified in \`.${this.name}\` cannot be found!`});
                 }
 
                 for (const file of files) {
-                    const content = await Parser.loadYaml(`${cwd}/${file}`, {});
+                    const content = await Parser.loadYaml(file, {});
                     contents = {
                         ...contents,
                         ...content,

src/parser-includes.ts

@@ -10,6 +10,7 @@ import axios from "axios";
 import globby from "globby";
 import path from "path";
 import semver from "semver";
+import RE2 from "re2";
 
 type ParserIncludesInitOptions = {
     argv: Argv;
@@ -66,13 +67,7 @@ export class ParserIncludes {
                     continue;
                 }
             }
-            if (value["local"]) {
-                validateIncludeLocal(value["local"]);
-                const files = await globby(value["local"].replace(/^\//, ""), {dot: true, cwd});
-                if (files.length == 0) {
-                    throw new AssertionError({message: `Local include file cannot be found ${value["local"]}`});
-                }
-            } else if (value["file"]) {
+            if (value["file"]) {
                 for (const fileValue of Array.isArray(value["file"]) ? value["file"] : [value["file"]]) {
                     promises.push(this.downloadIncludeProjectFile(cwd, stateDir, value["project"], value["ref"] || "HEAD", fileValue, gitData, fetchIncludes));
                 }
@@ -97,9 +92,13 @@ export class ParserIncludes {
                 }
             }
             if (value["local"]) {
-                const files = await globby([value["local"].replace(/^\//, "")], {dot: true, cwd});
+                validateIncludeLocal(value["local"]);
+                const files = resolveIncludeLocal(value["local"], cwd);
+                if (files.length == 0) {
+                    throw new AssertionError({message: `Local include file cannot be found ${value["local"]}`});
+                }
                 for (const localFile of files) {
-                    const content = await Parser.loadYaml(`${cwd}/${localFile}`, {inputs: value.inputs || {}}, expandVariables);
+                    const content = await Parser.loadYaml(localFile, {inputs: value.inputs ?? {}}, expandVariables);
                     includeDatas = includeDatas.concat(await this.init(content, opts));
                 }
             } else if (value["project"]) {
@@ -321,6 +320,18 @@ export class ParserIncludes {
             throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${normalizedFile} }\n${e}`});
         }
     }
+
+    static readonly memoLocalRepoFiles = (() => {
+        const cache = new Map<string, string[]>();
+        return (path: string) => {
+            let result = cache.get(path);
+            if (typeof result !== "undefined") return result;
+
+            result = globby.sync(path, {dot: true, gitignore: true});
+            cache.set(path, result);
+            return result;
+        };
+    })();
 }
 
 export function validateIncludeLocal (filePath: string) {
@@ -345,3 +356,24 @@ export function resolveSemanticVersionRange (range: string, gitTags: string[]) {
     });
     return found;
 }
+
+export function resolveIncludeLocal (pattern: string, cwd: string) {
+    const repoFiles = ParserIncludes.memoLocalRepoFiles(cwd);
+
+    if (!pattern.startsWith("/")) pattern = `/${pattern}`; // Ensure pattern starts with `/`
+    pattern = `${cwd}${pattern}`;
+
+    // escape all special regex metacharacters
+    pattern = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+    // `**` matches anything
+    const anything = ".*?";
+    pattern = pattern.replace(/\\\*\\\*/g, anything);
+
+    // `*` matches anything except for `/`
+    const anything_but_not_slash = "([^/])*?";
+    pattern = pattern.replace(/\\\*/g, anything_but_not_slash);
+
+    const re2 = new RE2(`^${pattern}`);
+    return repoFiles.filter((f: any) => re2.test(f));
+}

tests/test-cases/include-local-wildcard/.gitlab-ci-1.yml

@@ -0,0 +1,3 @@
+---
+# This matches all `.yml` files in `configs` and any subfolder in it.
+include: 'configs/**.yml'

tests/test-cases/include-local-wildcard/.gitlab-ci-2.yml

@@ -0,0 +1,3 @@
+---
+# This matches all `.yml` files only in subfolders of `configs`.
+include: 'configs/**/*.yml'

tests/test-cases/include-local-wildcard/.gitlab-ci-3.yml

@@ -0,0 +1,3 @@
+---
+# This matches all `.yml` files only in `configs`.
+include: 'configs/*.yml'

tests/test-cases/include-local-wildcard/configs/.gitlab-ci.yml

@@ -0,0 +1,4 @@
+---
+configs/.gitlab-ci.yml:
+  script:
+    - echo hello world

tests/test-cases/include-local-wildcard/configs/subfolder/.gitlab-ci.yml

@@ -0,0 +1,4 @@
+---
+configs/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world

tests/test-cases/include-local-wildcard/configs/subfolder/subfolder/.gitlab-ci.yml

@@ -0,0 +1,4 @@
+---
+configs/subfolder/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world

tests/test-cases/include-local-wildcard/integration.test.ts

@@ -18,3 +18,78 @@ test("include-local-wildcard <build-job>", async () => {
     expect(output).toContain("cache-repo executed!");
     expect(output).toContain("docs executed!");
 });
+
+test("expect `configs/**.yml` to match all `.yml` files in `configs` and any subfolder in it.", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/include-local-wildcard",
+        file: ".gitlab-ci-1.yml",
+        preview: true,
+        noColor: true,
+    }, writeStreams);
+    expect(writeStreams.stdoutLines.join()).toEqual(`
+---
+stages:
+  - .pre
+  - build
+  - test
+  - deploy
+  - .post
+configs/.gitlab-ci.yml:
+  script:
+    - echo hello world
+configs/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world
+configs/subfolder/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world
+`.trim());
+});
+
+test("expect `configs/**/*.yml` to match files only in subfolders of `configs`", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/include-local-wildcard",
+        file: ".gitlab-ci-2.yml",
+        preview: true,
+        noColor: true,
+    }, writeStreams);
+    expect(writeStreams.stdoutLines.join()).toEqual(`
+---
+stages:
+  - .pre
+  - build
+  - test
+  - deploy
+  - .post
+configs/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world
+configs/subfolder/subfolder/.gitlab-ci.yml:
+  script:
+    - echo hello world
+`.trim());
+});
+
+test("expect `configs/*.yml` to match only `.yml` files in `configs`.", async () => {
+    const writeStreams = new WriteStreamsMock();
+    await handler({
+        cwd: "tests/test-cases/include-local-wildcard",
+        file: ".gitlab-ci-3.yml",
+        preview: true,
+        noColor: true,
+    }, writeStreams);
+    expect(writeStreams.stdoutLines.join()).toEqual(`
+---
+stages:
+  - .pre
+  - build
+  - test
+  - deploy
+  - .post
+configs/.gitlab-ci.yml:
+  script:
+    - echo hello world
+`.trim());
+});