Merge branch 'main' into 948-neDB-implementation-issues

kriswest · web-flow · commit 12f20c9f93d4 · 2025-05-19T16:14:53.000+01:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -60,8 +60,8 @@ jobs:
     #   if: ${{ steps.test.outputs.exit_code }} != 0
     #   run: exit ${{ steps.test.outputs.exit_code }}
     
-    - name: Build application
-      run: npm run build
+    - name: Build frontend
+      run: npm run build-ui
 
     - name: Save build folder
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml
@@ -22,6 +22,8 @@ jobs:
           registry-url: 'https://registry.npmjs.org'
       - run: npm ci
       - run: npm run build
+        env:
+          IS_PUBLISHING: 'YES'
       - run: npm publish --access=public
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
diff --git a/.npmignore b/.npmignore
@@ -1,3 +1,5 @@
 # This file required to override .gitignore when publishing to npm
 website/
 plugins/
+experimental/
+cypress/
diff --git a/README.md b/README.md
@@ -84,6 +84,7 @@ $ git push proxy $(git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remo
 Using the default configuration, GitProxy intercepts the push and _blocks_ it. To enable code pushing to your fork via GitProxy, add your repository URL into the GitProxy config file (`proxy.config.json`). For more information, refer to [our documentation](https://git-proxy.finos.org).
 
 ## Documentation
+
 For detailed step-by-step instructions for how to install, deploy & configure GitProxy and
 customize for your environment, see the [project's documentation](https://git-proxy.finos.org/docs/):
 
@@ -101,11 +102,11 @@ If you identify a security vulnerability in the codebase, please follow the step
 
 ## Code of Conduct
 
-We are committed to making open source an enjoyable and respectful experience for our community. See <a href="https://github.com/finos/git-proxy/blob/main/CODE_OF_CONDUCT.md"><code>CODE_OF_CONDUCT</code></a> for more information.
+We are committed to making open source an enjoyable and respectful experience for our community. See [`CODE_OF_CONDUCT`](CODE_OF_CONDUCT.md) for more information.
 
 ## License
 
-This project is distributed under the Apache-2.0 license. See <a href="./LICENSE"><code>LICENSE</code></a> for more information.
+This project is distributed under the Apache-2.0 license. See [`LICENSE`](LICENSE) for more information.
 
 ## Contact
 
@@ -115,4 +116,4 @@ If you can't access Slack, you can also [subscribe to our mailing list](mailto:g
 
 Join our [fortnightly Zoom meeting](https://zoom.us/j/97235277537?pwd=aDJsaE8zcDJpYW1vZHJmSTJ0RXNZUT09) on Monday, 11AM EST (odd week numbers). Send an e-mail to [help@finos.org](mailto:help@finos.org) to get a calendar invitation.
 
-Otherwise, if you have a deeper query or require more support, please [raise an issue](https://github.com/finos/git-proxy/issues). 
+Otherwise, if you have a deeper query or require more support, please [raise an issue](https://github.com/finos/git-proxy/issues).
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,15 +1,18 @@
 {
   "name": "@finos/git-proxy",
-  "version": "1.11.0",
+  "version": "1.13.0",
   "description": "Deploy custom push protections and policies on top of Git.",
   "scripts": {
     "cli": "node ./packages/git-proxy-cli/index.js",
     "client": "vite --config vite.config.js",
     "clientinstall": "npm install --prefix client",
     "server": "tsx index.ts",
     "start": "concurrently \"npm run server\" \"npm run client\"",
-    "build": "vite build",
-    "build-ts": "tsc",
+    "build": "npm run build-ui && npm run build-lib",
+    "build-ui": "vite build",
+    "build-lib": "./scripts/build-for-publish.sh",
+    "restore-lib": "./scripts/undo-build.sh",
+    "check-types": "tsc",
     "test": "NODE_ENV=test ts-mocha './test/*.js' --exit",
     "test-coverage": "nyc npm run test",
     "test-coverage-ci": "nyc --reporter=lcovonly --reporter=text npm run test",
diff --git a/scripts/build-for-publish.sh b/scripts/build-for-publish.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# This script allows for emitting js and definitions from the typescript into
+# the same import locations as the original files.
+# When we adjust how we import the library we can move to a "dist" folder and
+# explicit "exports".
+
+if [ "${IS_PUBLISHING:-}" != "YES" ]; then
+  echo "This script is intended to prepare the directory for publishing"
+  echo "and replaces files. If you only want to build the UI run \`npm run build-ui\`."
+  echo "Otherwise set IS_PUBLISHING to \"YES\""
+  exit 1
+fi
+
+set -x
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "$REPO_ROOT"
+
+rm -rf dist || true
+tsc --project tsconfig.publish.json
+# replace tsx with node for the new index.js
+sed -ie '1s/tsx/node/' dist/index.js
+# ensure it's executable
+chmod +x dist/index.js
+# move the ts source
+mv src src-old
+# move the built source
+mv dist/src dist/index.js dist/index.d.ts .
+# copy back unchanged ui code
+# could probably drop this as the ui code shouldn't really be imported from
+# the main package but keep for compat until split out.
+mv src-old/ui src/ui
+rm -rf src-old index.ts dist
diff --git a/scripts/undo-build.sh b/scripts/undo-build.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# Undo what was done by build-for-publish.sh in the event this was ran locally
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "$REPO_ROOT"
+
+rm -rf dist index.js index.d.ts || true
+git checkout src index.ts
+git clean -f src
diff --git a/src/proxy/chain.ts b/src/proxy/chain.ts
@@ -14,6 +14,8 @@ const pushActionChain: ((req: any, action: Action) => Promise<Action>)[] = [
   proc.push.writePack,
   proc.push.preReceive,
   proc.push.getDiff,
+  // run before clear remote
+  proc.push.gitleaks,
   proc.push.clearBareClone,
   proc.push.scanDiff,
   proc.push.blockForAuth,
diff --git a/src/proxy/index.ts b/src/proxy/index.ts
@@ -27,7 +27,7 @@ const options = {
   cert: getTLSEnabled() ? fs.readFileSync(getTLSCertPemPath()) : undefined,
 };
 
-const proxyPreparations = async () => {
+export const proxyPreparations = async () => {
   const plugins = getPlugins();
   const pluginLoader = new PluginLoader(plugins);
   await pluginLoader.load();
@@ -47,15 +47,15 @@ const proxyPreparations = async () => {
 };
 
 // just keep this async incase it needs async stuff in the future
-const createApp = async () => {
+export const createApp = async () => {
   const app = express();
   // Setup the proxy middleware
   app.use(bodyParser.raw(options));
   app.use('/', router);
   return app;
 };
 
-const start = async () => {
+export const start = async () => {
   const app = await createApp();
   await proxyPreparations();
   http.createServer(options as any, app).listen(proxyHttpPort, () => {
diff --git a/src/proxy/processors/push-action/checkCommitMessages.ts b/src/proxy/processors/push-action/checkCommitMessages.ts
@@ -70,7 +70,7 @@ const exec = async (req: any, action: Action): Promise<Action> => {
     step.error = true;
     step.log(`The following commit messages are illegal: ${illegalMessages}`);
     step.setError(
-      '\n\n\n\nYour push has been blocked.\nPlease ensure your commit message(s) does not contain sensitive information or URLs.\n\n\n',
+      `\n\n\nYour push has been blocked.\nPlease ensure your commit message(s) does not contain sensitive information or URLs.\n\nThe following commit messages are illegal: ${JSON.stringify(illegalMessages)}\n\n`,
     );
 
     action.addStep(step);
diff --git a/src/proxy/processors/push-action/gitleaks.ts b/src/proxy/processors/push-action/gitleaks.ts
@@ -0,0 +1,184 @@
+import { Action, Step } from '../../actions';
+import { getAPIs } from '../../../config';
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import { PathLike } from 'node:fs';
+
+const EXIT_CODE = 99;
+
+function runCommand(
+  cwd: string,
+  command: string,
+  args: readonly string[] = [],
+): Promise<{
+  exitCode: number | null;
+  stdout: string;
+  stderr: string;
+}> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { cwd, shell: true });
+
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout.on('data', (data) => {
+      stdout += data?.toString() ?? '';
+    });
+
+    child.stderr.on('data', (data) => {
+      stderr += data?.toString() ?? '';
+    });
+
+    child.on('close', (exitCode) => {
+      resolve({ exitCode, stdout, stderr });
+    });
+
+    child.on('error', (err) => {
+      reject(err);
+    });
+  });
+}
+
+type ConfigOptions = {
+  enabled: boolean;
+  ignoreGitleaksAllow: boolean;
+  noColor: boolean;
+  configPath: string | undefined;
+};
+
+const DEFAULT_CONFIG: ConfigOptions = {
+  // adding gitleaks into main git-proxy for now as default off
+  // in the future will likely be moved to a plugin where it'll be default on
+  enabled: false,
+  ignoreGitleaksAllow: true,
+  noColor: false,
+  configPath: undefined,
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}
+
+async function fileIsReadable(path: PathLike): Promise<boolean> {
+  try {
+    if (!(await fs.stat(path)).isFile()) {
+      return false;
+    }
+    await fs.access(path, fs.constants.R_OK);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+const getPluginConfig = async (): Promise<ConfigOptions> => {
+  const userConfig = getAPIs();
+  if (typeof userConfig !== 'object') {
+    return DEFAULT_CONFIG;
+  }
+  if (!Object.hasOwn(userConfig, 'gitleaks')) {
+    return DEFAULT_CONFIG;
+  }
+  const gitleaksConfig = userConfig.gitleaks;
+  if (!isRecord(gitleaksConfig)) {
+    return DEFAULT_CONFIG;
+  }
+
+  let configPath: string | undefined = undefined;
+  if (typeof gitleaksConfig.configPath === 'string') {
+    const userConfigPath = gitleaksConfig.configPath.trim();
+    if (userConfigPath.length > 0 && (await fileIsReadable(userConfigPath))) {
+      configPath = userConfigPath;
+    } else {
+      console.error('could not read file at the config path provided, will not be fed to gitleaks');
+      throw new Error("could not check user's config path");
+    }
+  }
+
+  // TODO: integrate zod
+  return {
+    enabled:
+      typeof gitleaksConfig.enabled === 'boolean' ? gitleaksConfig.enabled : DEFAULT_CONFIG.enabled,
+    ignoreGitleaksAllow:
+      typeof gitleaksConfig.ignoreGitleaksAllow === 'boolean'
+        ? gitleaksConfig.ignoreGitleaksAllow
+        : DEFAULT_CONFIG.ignoreGitleaksAllow,
+    noColor:
+      typeof gitleaksConfig.noColor === 'boolean' ? gitleaksConfig.noColor : DEFAULT_CONFIG.noColor,
+    configPath,
+  };
+};
+
+const exec = async (req: any, action: Action): Promise<Action> => {
+  const step = new Step('gitleaks');
+
+  let config: ConfigOptions | undefined = undefined;
+  try {
+    config = await getPluginConfig();
+  } catch (e) {
+    console.error('failed to get gitleaks config, please fix the error:', e);
+    action.error = true;
+    step.setError('failed setup gitleaks, please contact an administrator\n');
+    action.addStep(step);
+    return action;
+  }
+
+  const { commitFrom, commitTo } = action;
+  const workingDir = `${action.proxyGitPath}/${action.repoName}`;
+  console.log(`Scanning range with gitleaks: ${commitFrom}:${commitTo}`, workingDir);
+
+  try {
+    const gitRootCommit = await runCommand(workingDir, 'git', [
+      'rev-list',
+      '--max-parents=0',
+      'HEAD',
+    ]);
+    if (gitRootCommit.exitCode !== 0) {
+      throw new Error('failed to run git');
+    }
+    const rootCommit = gitRootCommit.stdout.trim();
+
+    const gitleaksArgs = [
+      `--exit-code=${EXIT_CODE}`,
+      '--platform=none',
+      config.configPath ? `--config=${config.configPath}` : undefined, // allow for custom config
+      config.ignoreGitleaksAllow ? '--ignore-gitleaks-allow' : undefined, // force scanning for security
+      '--no-banner', // reduce git-proxy error output
+      config.noColor ? '--no-color' : undefined, // colour output should appear properly in the console
+      '--redact', // avoid printing the contents
+      '--verbose',
+      'git',
+      // not using --no-merges to be sure we're scanning the diff
+      // only add ^ if the commitFrom isn't the repo's rootCommit
+      `--log-opts='--first-parent ${rootCommit === commitFrom ? rootCommit : `${commitFrom}^`}..${commitTo}'`,
+    ].filter((v) => typeof v === 'string');
+    const gitleaks = await runCommand(workingDir, 'gitleaks', gitleaksArgs);
+
+    if (gitleaks.exitCode !== 0) {
+      // any failure
+      step.error = true;
+      if (gitleaks.exitCode !== EXIT_CODE) {
+        step.setError('failed to run gitleaks, please contact an administrator\n');
+      } else {
+        // exit code matched our gitleaks findings exit code
+        // newline prefix to avoid tab indent at the start
+        step.setError('\n' + gitleaks.stdout + gitleaks.stderr);
+      }
+    } else {
+      console.log('succeded');
+      console.log(gitleaks.stderr);
+    }
+  } catch (e) {
+    action.error = true;
+    step.setError('failed to spawn gitleaks, please contact an administrator\n');
+    action.addStep(step);
+    return action;
+  }
+
+  action.addStep(step);
+  return action;
+};
+
+exec.displayName = 'gitleaks.exec';
+
+export { exec };
diff --git a/src/proxy/processors/push-action/index.ts b/src/proxy/processors/push-action/index.ts
@@ -5,6 +5,7 @@ import { exec as audit } from './audit';
 import { exec as pullRemote } from './pullRemote';
 import { exec as writePack } from './writePack';
 import { exec as getDiff } from './getDiff';
+import { exec as gitleaks } from './gitleaks';
 import { exec as scanDiff } from './scanDiff';
 import { exec as blockForAuth } from './blockForAuth';
 import { exec as checkIfWaitingAuth } from './checkIfWaitingAuth';
@@ -21,6 +22,7 @@ export {
   pullRemote,
   writePack,
   getDiff,
+  gitleaks,
   scanDiff,
   blockForAuth,
   checkIfWaitingAuth,
diff --git a/test/chain.test.js b/test/chain.test.js
diff --git a/tsconfig.json b/tsconfig.json
diff --git a/tsconfig.publish.json b/tsconfig.publish.json
