|
| 1 | +import * as fs from "fs"; |
| 2 | +import * as path from "path"; |
| 3 | + |
| 4 | +import { getTemporaryDirectory } from "./actions-util"; |
| 5 | +import { type Config } from "./config-utils"; |
| 6 | +import { getAllFileOids, getGitRoot } from "./git-utils"; |
| 7 | +import { Logger } from "./logging"; |
| 8 | +import { pathStartsWith } from "./util"; |
| 9 | + |
1 | 10 | export enum OverlayDatabaseMode { |
2 | 11 | Overlay = "overlay", |
3 | 12 | OverlayBase = "overlay-base", |
4 | 13 | None = "none", |
5 | 14 | } |
6 | 15 |
|
7 | 16 | export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5"; |
| 17 | + |
| 18 | +/** |
| 19 | + * Writes a JSON file containing Git OIDs for all tracked files (represented |
| 20 | + * by path relative to the source root) under the source root. The file is |
| 21 | + * written into the database location specified in the config. |
| 22 | + * |
| 23 | + * @param config The configuration object containing the database location |
| 24 | + * @param sourceRoot The root directory containing the source files to process |
| 25 | + * @throws {Error} If the Git repository root cannot be determined |
| 26 | + */ |
| 27 | +export async function writeBaseDatabaseOidsFile( |
| 28 | + config: Config, |
| 29 | + sourceRoot: string, |
| 30 | +): Promise<void> { |
| 31 | + const gitFileOids = await getFileOidsUnderSourceRoot(sourceRoot); |
| 32 | + const gitFileOidsJson = JSON.stringify(gitFileOids); |
| 33 | + const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); |
| 34 | + await fs.promises.writeFile(baseDatabaseOidsFilePath, gitFileOidsJson); |
| 35 | +} |
| 36 | + |
| 37 | +/** |
| 38 | + * Reads and parses the JSON file containing the base database Git OIDs. |
| 39 | + * This file contains the mapping of file paths to their corresponding Git OIDs |
| 40 | + * that was previously written by writeBaseDatabaseOidsFile(). |
| 41 | + * |
| 42 | + * @param config The configuration object containing the database location |
| 43 | + * @param logger The logger instance to use for error reporting |
| 44 | + * @returns An object mapping file paths (relative to source root) to their Git OIDs |
| 45 | + * @throws {Error} If the file cannot be read or parsed |
| 46 | + */ |
| 47 | +async function readBaseDatabaseOidsFile( |
| 48 | + config: Config, |
| 49 | + logger: Logger, |
| 50 | +): Promise<{ [key: string]: string }> { |
| 51 | + const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config); |
| 52 | + try { |
| 53 | + const contents = await fs.promises.readFile( |
| 54 | + baseDatabaseOidsFilePath, |
| 55 | + "utf-8", |
| 56 | + ); |
| 57 | + return JSON.parse(contents) as { [key: string]: string }; |
| 58 | + } catch (e) { |
| 59 | + logger.error( |
| 60 | + "Failed to read overlay-base file OIDs from " + |
| 61 | + `${baseDatabaseOidsFilePath}: ${(e as any).message || e}`, |
| 62 | + ); |
| 63 | + throw e; |
| 64 | + } |
| 65 | +} |
| 66 | + |
| 67 | +/** |
| 68 | + * Writes a JSON file containing the absolute paths of files under `sourceRoot` |
| 69 | + * that have changed (added, removed, or modified) relative to the overlay base |
| 70 | + * database. |
| 71 | + * |
| 72 | + * This function uses the Git index to determine which files have changed, so it |
| 73 | + * has a few limitations: |
| 74 | + * |
| 75 | + * - It requires that `sourceRoot` is inside a Git repository. |
| 76 | + * - It only works for files tracked by the Git repository that `sourceRoot` is |
| 77 | + * in. If the Git repository has submodules, this function will not detect |
| 78 | + * changes in those submodules. |
| 79 | + * - It assumes that the Git repository is in a clean state, i.e. there are no |
| 80 | + * uncommitted changes in the repository. |
| 81 | + * - It assumes that all files of interest are tracked by Git, e.g. not covered |
| 82 | + * by `.gitignore`. |
| 83 | + */ |
| 84 | +export async function writeOverlayChangedFilesFile( |
| 85 | + config: Config, |
| 86 | + sourceRoot: string, |
| 87 | + logger: Logger, |
| 88 | +): Promise<string> { |
| 89 | + const baseFileOids = await readBaseDatabaseOidsFile(config, logger); |
| 90 | + const overlayFileOids = await getFileOidsUnderSourceRoot(sourceRoot); |
| 91 | + const gitChangedFiles = computeChangedFiles(baseFileOids, overlayFileOids); |
| 92 | + |
| 93 | + const overlayChangedFiles: string[] = []; |
| 94 | + for (const pathInSourceRoot of gitChangedFiles) { |
| 95 | + overlayChangedFiles.push(path.join(sourceRoot, pathInSourceRoot)); |
| 96 | + } |
| 97 | + |
| 98 | + logger.info( |
| 99 | + `Found ${overlayChangedFiles.length} changed file(s) ` + |
| 100 | + `under ${sourceRoot}.`, |
| 101 | + ); |
| 102 | + |
| 103 | + const changedFilesJson = JSON.stringify(overlayChangedFiles); |
| 104 | + const overlayChangedFilesFilePath = path.join( |
| 105 | + getTemporaryDirectory(), |
| 106 | + "overlay-changed-files.json", |
| 107 | + ); |
| 108 | + logger.debug( |
| 109 | + "Writing overlay changed files to " + |
| 110 | + `${overlayChangedFilesFilePath}: ${changedFilesJson}`, |
| 111 | + ); |
| 112 | + await fs.promises.writeFile(overlayChangedFilesFilePath, changedFilesJson); |
| 113 | + return overlayChangedFilesFilePath; |
| 114 | +} |
| 115 | + |
| 116 | +/** |
| 117 | + * Gets the Git oids of all files under the source root. |
| 118 | + * |
| 119 | + * @param sourceRoot The source root, which must be inside a Git repository. |
| 120 | + * @returns A map from file paths (relative to source root) to the corresponding Git OIDs. |
| 121 | + * @throws {Error} If the Git repository root cannot be determined. |
| 122 | + */ |
| 123 | +async function getFileOidsUnderSourceRoot( |
| 124 | + sourceRoot: string, |
| 125 | +): Promise<{ [key: string]: string }> { |
| 126 | + const gitRoot = await getGitRoot(sourceRoot); |
| 127 | + if (!gitRoot) { |
| 128 | + throw new Error("Failed to determine Git repository root"); |
| 129 | + } |
| 130 | + |
| 131 | + const allFileOids = await getAllFileOids(sourceRoot); |
| 132 | + const filteredFileOids: { [key: string]: string } = {}; |
| 133 | + |
| 134 | + for (const [pathInGitRepo, oid] of Object.entries(allFileOids)) { |
| 135 | + const absolutePath = path.join(gitRoot, pathInGitRepo); |
| 136 | + if (pathStartsWith(absolutePath, sourceRoot)) { |
| 137 | + // Convert absolutePath to be relative to sourceRoot |
| 138 | + const pathInSourceRoot = path.relative(sourceRoot, absolutePath); |
| 139 | + filteredFileOids[pathInSourceRoot] = oid; |
| 140 | + } |
| 141 | + } |
| 142 | + |
| 143 | + return filteredFileOids; |
| 144 | +} |
| 145 | + |
| 146 | +function getBaseDatabaseOidsFilePath(config: Config): string { |
| 147 | + return path.join(config.dbLocation, "base-database-oids.json"); |
| 148 | +} |
| 149 | + |
| 150 | +function computeChangedFiles( |
| 151 | + baseFileOids: { [key: string]: string }, |
| 152 | + overlayFileOids: { [key: string]: string }, |
| 153 | +): string[] { |
| 154 | + const changes: string[] = []; |
| 155 | + for (const [file, oid] of Object.entries(overlayFileOids)) { |
| 156 | + if (!(file in baseFileOids) || baseFileOids[file] !== oid) { |
| 157 | + changes.push(file); |
| 158 | + } |
| 159 | + } |
| 160 | + for (const file of Object.keys(baseFileOids)) { |
| 161 | + if (!(file in overlayFileOids)) { |
| 162 | + changes.push(file); |
| 163 | + } |
| 164 | + } |
| 165 | + return changes; |
| 166 | +} |
0 commit comments