Support overlay database creation

Author: cklin

src/analyze.test.ts

@@ -101,6 +101,7 @@ test("status report fields", async (t) => {
         memoryFlag,
         addSnippetsFlag,
         threadsFlag,
+        "brutal",
         undefined,
         undefined,
         config,

src/codeql.test.ts

@@ -20,6 +20,7 @@ import { DocUrl } from "./doc-url";
 import { FeatureEnablement } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
+import { OverlayDatabaseMode } from "./overlay-database-utils";
 import { ToolsSource } from "./setup-codeql";
 import {
   setupTests,
@@ -510,6 +511,7 @@ const injectedConfigMacro = test.macro({
         "",
         undefined,
         undefined,
+        OverlayDatabaseMode.None,
         getRunnerLogger(true),
       );
 
@@ -723,6 +725,7 @@ test("passes a code scanning config AND qlconfig to the CLI", async (t: Executio
       "",
       undefined,
       "/path/to/qlconfig.yml",
+      OverlayDatabaseMode.None,
       getRunnerLogger(true),
     );
 
@@ -752,6 +755,7 @@ test("does not pass a qlconfig to the CLI when it is undefined", async (t: Execu
       "",
       undefined,
       undefined, // undefined qlconfigFile
+      OverlayDatabaseMode.None,
       getRunnerLogger(true),
     );
 
@@ -1005,6 +1009,7 @@ test("Avoids duplicating --overwrite flag if specified in CODEQL_ACTION_EXTRA_OP
     "sourceRoot",
     undefined,
     undefined,
+    OverlayDatabaseMode.None,
     getRunnerLogger(false),
   );
 

src/codeql.ts

@@ -24,6 +24,11 @@ import {
 import { isAnalyzingDefaultBranch } from "./git-utils";
 import { Language } from "./languages";
 import { Logger } from "./logging";
+import {
+  OverlayDatabaseMode,
+  writeBaseDatabaseOidsFile,
+  writeOverlayChangedFilesFile,
+} from "./overlay-database-utils";
 import * as setupCodeql from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";
@@ -82,6 +87,7 @@ export interface CodeQL {
     sourceRoot: string,
     processName: string | undefined,
     qlconfigFile: string | undefined,
+    overlayDatabaseMode: OverlayDatabaseMode,
     logger: Logger,
   ): Promise<void>;
   /**
@@ -552,6 +558,7 @@ export async function getCodeQLForCmd(
       sourceRoot: string,
       processName: string | undefined,
       qlconfigFile: string | undefined,
+      overlayDatabaseMode: OverlayDatabaseMode,
       logger: Logger,
     ) {
       const extraArgs = config.languages.map(
@@ -606,12 +613,21 @@ export async function getCodeQLForCmd(
         ? "--force-overwrite"
         : "--overwrite";
 
+      if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
+        await writeOverlayChangedFilesFile(config, sourceRoot, logger);
+        extraArgs.push("--overlay");
+      } else if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
+        extraArgs.push("--overlay-base");
+      }
+
       await runCli(
         cmd,
         [
           "database",
           "init",
-          overwriteFlag,
+          ...(overlayDatabaseMode === OverlayDatabaseMode.Overlay
+            ? []
+            : [overwriteFlag]),
           "--db-cluster",
           config.dbLocation,
           `--source-root=${sourceRoot}`,
@@ -625,6 +641,10 @@ export async function getCodeQLForCmd(
         ],
         { stdin: externalRepositoryToken },
       );
+
+      if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
+        await writeBaseDatabaseOidsFile(config, sourceRoot);
+      }
     },
     async runAutobuild(config: Config, language: Language) {
       applyAutobuildAzurePipelinesTimeoutFix();

src/git-utils.ts

@@ -300,6 +300,61 @@ export const decodeGitFilePath = function (filePath: string): string {
   return filePath;
 };
 
+/**
+ * Get the oids of all files in HEAD.
+ *
+ * @param checkoutPath A path into the Git repository.
+ * @returns a map from file paths to the corresponding oid.
+ * @throws {Error} if "git ls-tree" produces unexpected output.
+ */
+export const getAllFileOids = async function (
+  checkoutPath: string,
+): Promise<{ [key: string]: string }> {
+  const stdout = await runGitCommand(
+    checkoutPath,
+    ["ls-tree", "--format=%(objectname)_%(path)", "-r", "HEAD"],
+    "Cannot list file OIDs in HEAD.",
+  );
+
+  const fileOidMap: { [key: string]: string } = {};
+  const regex = /^([0-9a-f]{40})_(.+)$/;
+  for (const line of stdout.split("\n")) {
+    if (line) {
+      const match = line.match(regex);
+      if (match) {
+        const oid = match[1];
+        const path = decodeGitFilePath(match[2]);
+        fileOidMap[path] = oid;
+      } else {
+        throw new Error(`Unexpected "git ls-tree" output: ${line}`);
+      }
+    }
+  }
+  return fileOidMap;
+};
+
+/**
+ * Get the root of the Git repository.
+ *
+ * @param sourceRoot The source root of the code being analyzed.
+ * @returns The root of the Git repository.
+ */
+export const getGitRoot = async function (
+  sourceRoot: string,
+): Promise<string | undefined> {
+  try {
+    const stdout = await runGitCommand(
+      sourceRoot,
+      ["rev-parse", "--show-toplevel"],
+      `Cannot find Git repository root from the source root ${sourceRoot}.`,
+    );
+    return stdout.trim();
+  } catch {
+    // Errors are already logged by runGitCommand()
+    return undefined;
+  }
+};
+
 function getRefFromEnv(): string {
   // To workaround a limitation of Actions dynamic workflows not setting
   // the GITHUB_REF in some cases, we accept also the ref within the

src/init-action.ts

@@ -36,12 +36,14 @@ import { Feature, featureConfig, Features } from "./feature-flags";
 import {
   checkInstallPython311,
   cleanupDatabaseClusterDirectory,
+  getOverlayDatabaseMode,
   initCodeQL,
   initConfig,
   runInit,
 } from "./init";
 import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
+import { OverlayDatabaseMode } from "./overlay-database-utils";
 import { parseRepositoryNwo } from "./repository";
 import { ToolsSource } from "./setup-codeql";
 import {
@@ -395,7 +397,22 @@ async function run() {
   }
 
   try {
-    cleanupDatabaseClusterDirectory(config, logger);
+    const sourceRoot = path.resolve(
+      getRequiredEnvParam("GITHUB_WORKSPACE"),
+      getOptionalInput("source-root") || "",
+    );
+
+    const overlayDatabaseMode = await getOverlayDatabaseMode(
+      (await codeql.getVersion()).version,
+      config,
+      sourceRoot,
+      logger,
+    );
+    logger.info(`Using overlay database mode: ${overlayDatabaseMode}`);
+
+    if (overlayDatabaseMode !== OverlayDatabaseMode.Overlay) {
+      cleanupDatabaseClusterDirectory(config, logger);
+    }
 
     if (zstdAvailability) {
       await recordZstdAvailability(config, zstdAvailability);
@@ -675,18 +692,14 @@ async function run() {
       }
     }
 
-    const sourceRoot = path.resolve(
-      getRequiredEnvParam("GITHUB_WORKSPACE"),
-      getOptionalInput("source-root") || "",
-    );
-
     const tracerConfig = await runInit(
       codeql,
       config,
       sourceRoot,
       "Runner.Worker.exe",
       getOptionalInput("registries"),
       apiDetails,
+      overlayDatabaseMode,
       logger,
     );
     if (tracerConfig !== undefined) {

src/init.ts

@@ -3,14 +3,20 @@ import * as path from "path";
 
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as io from "@actions/io";
+import * as semver from "semver";
 
 import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { CodeQLDefaultVersionInfo, FeatureEnablement } from "./feature-flags";
+import { getGitRoot } from "./git-utils";
 import { Language, isScannedLanguage } from "./languages";
 import { Logger } from "./logging";
+import {
+  CODEQL_OVERLAY_MINIMUM_VERSION,
+  OverlayDatabaseMode,
+} from "./overlay-database-utils";
 import { ToolsSource } from "./setup-codeql";
 import { ZstdAvailability } from "./tar";
 import { ToolsDownloadStatusReport } from "./tools-download";
@@ -79,13 +85,55 @@ export async function initConfig(
   return config;
 }
 
+export async function getOverlayDatabaseMode(
+  codeqlVersion: string,
+  config: configUtils.Config,
+  sourceRoot: string,
+  logger: Logger,
+): Promise<OverlayDatabaseMode> {
+  const overlayDatabaseMode = process.env.CODEQL_OVERLAY_DATABASE_MODE;
+
+  if (
+    overlayDatabaseMode === OverlayDatabaseMode.Overlay ||
+    overlayDatabaseMode === OverlayDatabaseMode.OverlayBase
+  ) {
+    if (config.buildMode !== util.BuildMode.None) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `build-mode is set to "${config.buildMode}" instead of "none". ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if (semver.lt(codeqlVersion, CODEQL_OVERLAY_MINIMUM_VERSION)) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the CodeQL CLI is older than ${CODEQL_OVERLAY_MINIMUM_VERSION}. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    if ((await getGitRoot(sourceRoot)) === undefined) {
+      logger.warning(
+        `Cannot build an ${overlayDatabaseMode} database because ` +
+          `the source root "${sourceRoot}" is not inside a git repository. ` +
+          "Falling back to creating a normal full database instead.",
+      );
+      return OverlayDatabaseMode.None;
+    }
+    return overlayDatabaseMode as OverlayDatabaseMode;
+  }
+  return OverlayDatabaseMode.None;
+}
+
 export async function runInit(
   codeql: CodeQL,
   config: configUtils.Config,
   sourceRoot: string,
   processName: string | undefined,
   registriesInput: string | undefined,
   apiDetails: GitHubApiCombinedDetails,
+  overlayDatabaseMode: OverlayDatabaseMode,
   logger: Logger,
 ): Promise<TracerConfig | undefined> {
   fs.mkdirSync(config.dbLocation, { recursive: true });
@@ -109,6 +157,7 @@ export async function runInit(
         sourceRoot,
         processName,
         qlconfigFile,
+        overlayDatabaseMode,
         logger,
       ),
   );