diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 028b5b5846..36455b2358 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -117769,6 +117769,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 927bbd8f79..223ddadf40 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -91003,6 +91003,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -93201,81 +93206,130 @@ var os3 = __toESM(require("os")); var import_path = require("path"); var actionsCache3 = __toESM(require_cache3()); var glob = __toESM(require_glob2()); +var NoMatchingFilesError = class extends Error { + constructor(msg) { + super(msg); + this.name = "NoMatchingFilesError"; + } +}; var CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; var CODEQL_DEPENDENCY_CACHE_VERSION = 1; function getJavaTempDependencyDir() { return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository"); } -function getDefaultCacheConfig() { - return { - java: { - paths: [ - // Maven - (0, import_path.join)(os3.homedir(), ".m2", "repository"), - // Gradle - (0, import_path.join)(os3.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir() - ], - hash: [ - // Maven - "**/pom.xml", - // Gradle - "**/*.gradle*", - "**/gradle-wrapper.properties", - "buildSrc/**/Versions.kt", - "buildSrc/**/Dependencies.kt", - "gradle/*.versions.toml", - "**/versions.properties" - ] - }, - csharp: { - paths: [(0, import_path.join)(os3.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock" - ] - }, - go: { - paths: [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"] - } - }; +function getJavaDependencyDirs() { + return [ + // Maven + (0, import_path.join)(os3.homedir(), ".m2", "repository"), + // Gradle + (0, import_path.join)(os3.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir() + ]; +} +async function makePatternCheck(patterns) { + const globber = await makeGlobber(patterns); + if ((await globber.glob()).length === 0) { + throw new NoMatchingFilesError(); + } + return patterns; } +async function getCsharpHashPatterns(codeql, features) { + const basePatterns = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock" + ]; + const globber = await makeGlobber(basePatterns); + if ((await globber.glob()).length > 0) { + return basePatterns; + } + if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) { + return makePatternCheck([ + "**/*.csproj", + "**/packages.config", + "**/nuget.config" + ]); + } + throw new NoMatchingFilesError(); +} +var defaultCacheConfigs = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => makePatternCheck([ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties" + ]) + }, + csharp: { + getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns + }, + go: { + getDependencyPaths: () => [(0, import_path.join)(os3.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => makePatternCheck(["**/go.sum"]) + } +}; async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function uploadDependencyCaches(config, logger, minimizeJavaJars) { +async function checkHashPatterns(codeql, features, language, cacheConfig, logger) { + try { + return cacheConfig.getHashPatterns(codeql, features); + } catch (err) { + if (err instanceof NoMatchingFilesError) { + logger.info( + `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.` + ); + return void 0; + } + throw err; + } +} +async function uploadDependencyCaches(codeql, features, config, logger) { for (const language of config.languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === void 0) { logger.info( `Skipping upload of dependency cache for ${language} as we have no caching configuration for it.` ); continue; } - const globber = await makeGlobber(cacheConfig.hash); - if ((await globber.glob()).length === 0) { - logger.info( - `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.` - ); + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + logger + ); + if (patterns === void 0) { continue; } - const size = await getTotalCacheSize(cacheConfig.paths, logger, true); + const size = await getTotalCacheSize( + cacheConfig.getDependencyPaths(), + logger, + true + ); if (size === 0) { logger.info( `Skipping upload of dependency cache for ${language} since it is empty.` ); continue; } - const key = await cacheKey2(language, cacheConfig, minimizeJavaJars); + const key = await cacheKey2(codeql, features, language, patterns); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...` ); try { - await actionsCache3.saveCache(cacheConfig.paths, key); + await actionsCache3.saveCache(cacheConfig.getDependencyPaths(), key); } catch (error2) { if (error2 instanceof actionsCache3.ReserveCacheError) { logger.info( @@ -93288,17 +93342,21 @@ async function uploadDependencyCaches(config, logger, minimizeJavaJars) { } } } -async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { - const hash2 = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language, minimizeJavaJars)}${hash2}`; +async function cacheKey2(codeql, features, language, patterns) { + const hash2 = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix2(codeql, features, language)}${hash2}`; } -async function cachePrefix2(language, minimizeJavaJars) { +async function cachePrefix2(codeql, features, language) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); if (language === "java" /* java */ && minimizeJavaJars) { prefix = `minify-${prefix}`; } @@ -96083,11 +96141,7 @@ async function run() { logger ); if (shouldStoreCache(config.dependencyCachingEnabled)) { - const minimizeJavaJars = await features.getValue( - "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, - codeql - ); - await uploadDependencyCaches(config, logger, minimizeJavaJars); + await uploadDependencyCaches(codeql, features, config, logger); } if (isInTestMode()) { logger.debug("In test mode. Waiting for processing is disabled."); diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index 526f1b97e2..fab4c86a98 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -78507,6 +78507,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/init-action-post.js b/lib/init-action-post.js index bc86cec133..6adc558f97 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -129102,6 +129102,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/init-action.js b/lib/init-action.js index 51b9c5febd..fd9d04a427 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -86602,6 +86602,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", @@ -87928,72 +87933,117 @@ var os2 = __toESM(require("os")); var import_path = require("path"); var actionsCache3 = __toESM(require_cache3()); var glob = __toESM(require_glob2()); +var NoMatchingFilesError = class extends Error { + constructor(msg) { + super(msg); + this.name = "NoMatchingFilesError"; + } +}; var CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; var CODEQL_DEPENDENCY_CACHE_VERSION = 1; function getJavaTempDependencyDir() { return (0, import_path.join)(getTemporaryDirectory(), "codeql_java", "repository"); } -function getDefaultCacheConfig() { - return { - java: { - paths: [ - // Maven - (0, import_path.join)(os2.homedir(), ".m2", "repository"), - // Gradle - (0, import_path.join)(os2.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir() - ], - hash: [ - // Maven - "**/pom.xml", - // Gradle - "**/*.gradle*", - "**/gradle-wrapper.properties", - "buildSrc/**/Versions.kt", - "buildSrc/**/Dependencies.kt", - "gradle/*.versions.toml", - "**/versions.properties" - ] - }, - csharp: { - paths: [(0, import_path.join)(os2.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock" - ] - }, - go: { - paths: [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"] - } - }; +function getJavaDependencyDirs() { + return [ + // Maven + (0, import_path.join)(os2.homedir(), ".m2", "repository"), + // Gradle + (0, import_path.join)(os2.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir() + ]; +} +async function makePatternCheck(patterns) { + const globber = await makeGlobber(patterns); + if ((await globber.glob()).length === 0) { + throw new NoMatchingFilesError(); + } + return patterns; +} +async function getCsharpHashPatterns(codeql, features) { + const basePatterns = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock" + ]; + const globber = await makeGlobber(basePatterns); + if ((await globber.glob()).length > 0) { + return basePatterns; + } + if (await features.getValue("csharp_new_cache_key" /* CsharpNewCacheKey */, codeql)) { + return makePatternCheck([ + "**/*.csproj", + "**/packages.config", + "**/nuget.config" + ]); + } + throw new NoMatchingFilesError(); } +var defaultCacheConfigs = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => makePatternCheck([ + // Maven + "**/pom.xml", + // Gradle + "**/*.gradle*", + "**/gradle-wrapper.properties", + "buildSrc/**/Versions.kt", + "buildSrc/**/Dependencies.kt", + "gradle/*.versions.toml", + "**/versions.properties" + ]) + }, + csharp: { + getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns + }, + go: { + getDependencyPaths: () => [(0, import_path.join)(os2.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => makePatternCheck(["**/go.sum"]) + } +}; async function makeGlobber(patterns) { return glob.create(patterns.join("\n")); } -async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { +async function checkHashPatterns(codeql, features, language, cacheConfig, logger) { + try { + return cacheConfig.getHashPatterns(codeql, features); + } catch (err) { + if (err instanceof NoMatchingFilesError) { + logger.info( + `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.` + ); + return void 0; + } + throw err; + } +} +async function downloadDependencyCaches(codeql, features, languages, logger) { const restoredCaches = []; for (const language of languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === void 0) { logger.info( `Skipping download of dependency cache for ${language} as we have no caching configuration for it.` ); continue; } - const globber = await makeGlobber(cacheConfig.hash); - if ((await globber.glob()).length === 0) { - logger.info( - `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.` - ); + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + logger + ); + if (patterns === void 0) { continue; } - const primaryKey = await cacheKey2(language, cacheConfig, minimizeJavaJars); + const primaryKey = await cacheKey2(codeql, features, language, patterns); const restoreKeys = [ - await cachePrefix2(language, minimizeJavaJars) + await cachePrefix2(codeql, features, language) ]; logger.info( `Downloading cache for ${language} with key ${primaryKey} and restore keys ${restoreKeys.join( @@ -88001,7 +88051,7 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { )}` ); const hitKey = await actionsCache3.restoreCache( - cacheConfig.paths, + cacheConfig.getDependencyPaths(), primaryKey, restoreKeys ); @@ -88014,17 +88064,21 @@ async function downloadDependencyCaches(languages, logger, minimizeJavaJars) { } return restoredCaches; } -async function cacheKey2(language, cacheConfig, minimizeJavaJars = false) { - const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix2(language, minimizeJavaJars)}${hash}`; +async function cacheKey2(codeql, features, language, patterns) { + const hash = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix2(codeql, features, language)}${hash}`; } -async function cachePrefix2(language, minimizeJavaJars) { +async function cachePrefix2(codeql, features, language) { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env["CODEQL_ACTION_DEPENDENCY_CACHE_PREFIX" /* DEPENDENCY_CACHING_PREFIX */]; let prefix = CODEQL_DEPENDENCY_CACHE_PREFIX; if (customPrefix !== void 0 && customPrefix.length > 0) { prefix = `${prefix}-${customPrefix}`; } + const minimizeJavaJars = await features.getValue( + "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, + codeql + ); if (language === "java" /* java */ && minimizeJavaJars) { prefix = `minify-${prefix}`; } @@ -90633,15 +90687,12 @@ exec ${goBinaryPath} "$@"` core13.exportVariable(envVar, "false"); } } - const minimizeJavaJars = await features.getValue( - "java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, - codeql - ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { await downloadDependencyCaches( + codeql, + features, config.languages, - logger, - minimizeJavaJars + logger ); } if (await codeQlVersionAtLeast(codeql, "2.17.1")) { @@ -90679,7 +90730,7 @@ exec ${goBinaryPath} "$@"` logger.debug( `${"CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */} is already set to '${process.env["CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */]}', so the Action will not override it.` ); - } else if (minimizeJavaJars && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) { + } else if (await features.getValue("java_minimize_dependency_jars" /* JavaMinimizeDependencyJars */, codeql) && config.dependencyCachingEnabled && config.buildMode === "none" /* None */ && config.languages.includes("java" /* java */)) { core13.exportVariable( "CODEQL_EXTRACTOR_JAVA_OPTION_MINIMIZE_DEPENDENCY_JARS" /* JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS */, "true" diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index 784b37f3cf..842dfda136 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -78498,6 +78498,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index c26090ba03..b5d0648159 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -117178,6 +117178,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 88dc2d5890..8f27190ebc 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -89194,6 +89194,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index 0d03682546..4fc2c33ba6 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -117343,6 +117343,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index f603d0aa17..c7d9dbfb2c 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -89190,6 +89190,11 @@ var featureConfig = { legacyApi: true, minimumVersion: "2.15.0" }, + ["csharp_new_cache_key" /* CsharpNewCacheKey */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: void 0 + }, ["diff_informed_queries" /* DiffInformedQueries */]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/src/analyze-action.ts b/src/analyze-action.ts index 31251be488..45e61867bd 100644 --- a/src/analyze-action.ts +++ b/src/analyze-action.ts @@ -29,7 +29,7 @@ import { uploadDatabases } from "./database-upload"; import { uploadDependencyCaches } from "./dependency-caching"; import { getDiffInformedAnalysisBranches } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; -import { Feature, Features } from "./feature-flags"; +import { Features } from "./feature-flags"; import { KnownLanguage } from "./languages"; import { getActionsLogger, Logger } from "./logging"; import { uploadOverlayBaseDatabaseToCache } from "./overlay-database-utils"; @@ -384,11 +384,7 @@ async function run() { // Store dependency cache(s) if dependency caching is enabled. if (shouldStoreCache(config.dependencyCachingEnabled)) { - const minimizeJavaJars = await features.getValue( - Feature.JavaMinimizeDependencyJars, - codeql, - ); - await uploadDependencyCaches(config, logger, minimizeJavaJars); + await uploadDependencyCaches(codeql, features, config, logger); } // We don't upload results in test mode, so don't wait for processing diff --git a/src/dependency-caching.ts b/src/dependency-caching.ts index 6289ca2f68..895ea55bfa 100644 --- a/src/dependency-caching.ts +++ b/src/dependency-caching.ts @@ -6,24 +6,37 @@ import * as glob from "@actions/glob"; import { getTemporaryDirectory } from "./actions-util"; import { getTotalCacheSize } from "./caching-utils"; +import { CodeQL } from "./codeql"; import { Config } from "./config-utils"; import { EnvVar } from "./environment"; +import { Feature, Features } from "./feature-flags"; import { KnownLanguage, Language } from "./languages"; import { Logger } from "./logging"; import { getRequiredEnvParam } from "./util"; +class NoMatchingFilesError extends Error { + constructor(msg?: string) { + super(msg); + + this.name = "NoMatchingFilesError"; + } +} + /** * Caching configuration for a particular language. */ interface CacheConfig { - /** The paths of directories on the runner that should be included in the cache. */ - paths: string[]; + /** Gets the paths of directories on the runner that should be included in the cache. */ + getDependencyPaths: () => string[]; /** - * Patterns for the paths of files whose contents affect which dependencies are used - * by a project. We find all files which match these patterns, calculate a hash for - * their contents, and use that hash as part of the cache key. + * Gets an array of glob patterns for the paths of files whose contents affect which dependencies are used + * by a project. This function also checks whether there are any matching files and throws + * a `NoMatchingFilesError` error if no files match. + * + * The glob patterns are intended to be used for cache keys, where we find all files which match these + * patterns, calculate a hash for their contents, and use that hash as part of the cache key. */ - hash: string[]; + getHashPatterns: (codeql: CodeQL, features: Features) => Promise; } const CODEQL_DEPENDENCY_CACHE_PREFIX = "codeql-dependencies"; @@ -38,21 +51,98 @@ export function getJavaTempDependencyDir(): string { return join(getTemporaryDirectory(), "codeql_java", "repository"); } +/** + * Returns an array of paths of directories on the runner that should be included in a dependency cache + * for a Java analysis. It is important that this is a function, because we call `getTemporaryDirectory` + * which would otherwise fail in tests if we haven't had a chance to initialise `RUNNER_TEMP`. + * + * @returns The paths of directories on the runner that should be included in a dependency cache + * for a Java analysis. + */ +export function getJavaDependencyDirs(): string[] { + return [ + // Maven + join(os.homedir(), ".m2", "repository"), + // Gradle + join(os.homedir(), ".gradle", "caches"), + // CodeQL Java build-mode: none + getJavaTempDependencyDir(), + ]; +} + +/** + * Checks that there are files which match `patterns`. If there are matching files for any of the patterns, + * this function returns all `patterns`. Otherwise, a `NoMatchingFilesError` is thrown. + * + * @param patterns The glob patterns to find matching files for. + * @returns The array of glob patterns if there are matching files. + */ +async function makePatternCheck(patterns: string[]): Promise { + const globber = await makeGlobber(patterns); + + if ((await globber.glob()).length === 0) { + throw new NoMatchingFilesError(); + } + + return patterns; +} + +/** + * Returns the list of glob patterns that should be used to calculate the cache key hash + * for a C# dependency cache. + * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. + * @returns A list of glob patterns to use for hashing. + */ +async function getCsharpHashPatterns( + codeql: CodeQL, + features: Features, +): Promise { + // These files contain accurate information about dependencies, including the exact versions + // that the relevant package manager has determined for the project. Using these gives us + // stable hashes unless the dependencies change. + const basePatterns = [ + // NuGet + "**/packages.lock.json", + // Paket + "**/paket.lock", + ]; + const globber = await makeGlobber(basePatterns); + + if ((await globber.glob()).length > 0) { + return basePatterns; + } + + if (await features.getValue(Feature.CsharpNewCacheKey, codeql)) { + // These are less accurate for use in cache key calculations, because they: + // + // - Don't contain the exact versions used. They may only contain version ranges or none at all. + // - They contain information unrelated to dependencies, which we don't care about. + // + // As a result, the hash we compute from these files may change, even if + // the dependencies haven't changed. + return makePatternCheck([ + "**/*.csproj", + "**/packages.config", + "**/nuget.config", + ]); + } + + // If we get to this point, the `basePatterns` didn't find any files, + // and `Feature.CsharpNewCacheKey` is either not enabled or we didn't + // find any files using those patterns either. + throw new NoMatchingFilesError(); +} + /** * Default caching configurations per language. */ -function getDefaultCacheConfig(): { [language: string]: CacheConfig } { - return { - java: { - paths: [ - // Maven - join(os.homedir(), ".m2", "repository"), - // Gradle - join(os.homedir(), ".gradle", "caches"), - // CodeQL Java build-mode: none - getJavaTempDependencyDir(), - ], - hash: [ +const defaultCacheConfigs: { [language: string]: CacheConfig } = { + java: { + getDependencyPaths: getJavaDependencyDirs, + getHashPatterns: async () => + makePatternCheck([ // Maven "**/pom.xml", // Gradle @@ -62,45 +152,72 @@ function getDefaultCacheConfig(): { [language: string]: CacheConfig } { "buildSrc/**/Dependencies.kt", "gradle/*.versions.toml", "**/versions.properties", - ], - }, - csharp: { - paths: [join(os.homedir(), ".nuget", "packages")], - hash: [ - // NuGet - "**/packages.lock.json", - // Paket - "**/paket.lock", - ], - }, - go: { - paths: [join(os.homedir(), "go", "pkg", "mod")], - hash: ["**/go.sum"], - }, - }; -} + ]), + }, + csharp: { + getDependencyPaths: () => [join(os.homedir(), ".nuget", "packages")], + getHashPatterns: getCsharpHashPatterns, + }, + go: { + getDependencyPaths: () => [join(os.homedir(), "go", "pkg", "mod")], + getHashPatterns: async () => makePatternCheck(["**/go.sum"]), + }, +}; async function makeGlobber(patterns: string[]): Promise { return glob.create(patterns.join("\n")); } +/** + * A wrapper around `cacheConfig.getHashPatterns` which catches `NoMatchingFilesError` errors, + * and logs that there are no files to calculate a hash for the cache key from. + * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. + * @param language The language the `CacheConfig` is for. For use in the log message. + * @param cacheConfig The caching configuration to call `getHashPatterns` on. + * @param logger The logger to write the log message to if there is an error. + * @returns An array of glob patterns to use for hashing files, or `undefined` if there are no matching files. + */ +async function checkHashPatterns( + codeql: CodeQL, + features: Features, + language: Language, + cacheConfig: CacheConfig, + logger: Logger, +): Promise { + try { + return cacheConfig.getHashPatterns(codeql, features); + } catch (err) { + if (err instanceof NoMatchingFilesError) { + logger.info( + `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, + ); + return undefined; + } + throw err; + } +} + /** * Attempts to restore dependency caches for the languages being analyzed. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param languages The languages being analyzed. * @param logger A logger to record some informational messages to. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns A list of languages for which dependency caches were restored. */ export async function downloadDependencyCaches( + codeql: CodeQL, + features: Features, languages: Language[], logger: Logger, - minimizeJavaJars: boolean, ): Promise { const restoredCaches: Language[] = []; for (const language of languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === undefined) { logger.info( @@ -111,18 +228,20 @@ export async function downloadDependencyCaches( // Check that we can find files to calculate the hash for the cache key from, so we don't end up // with an empty string. - const globber = await makeGlobber(cacheConfig.hash); - - if ((await globber.glob()).length === 0) { - logger.info( - `Skipping download of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, - ); + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + logger, + ); + if (patterns === undefined) { continue; } - const primaryKey = await cacheKey(language, cacheConfig, minimizeJavaJars); + const primaryKey = await cacheKey(codeql, features, language, patterns); const restoreKeys: string[] = [ - await cachePrefix(language, minimizeJavaJars), + await cachePrefix(codeql, features, language), ]; logger.info( @@ -132,7 +251,7 @@ export async function downloadDependencyCaches( ); const hitKey = await actionsCache.restoreCache( - cacheConfig.paths, + cacheConfig.getDependencyPaths(), primaryKey, restoreKeys, ); @@ -151,17 +270,19 @@ export async function downloadDependencyCaches( /** * Attempts to store caches for the languages that were analyzed. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param config The configuration for this workflow. * @param logger A logger to record some informational messages to. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. */ export async function uploadDependencyCaches( + codeql: CodeQL, + features: Features, config: Config, logger: Logger, - minimizeJavaJars: boolean, ): Promise { for (const language of config.languages) { - const cacheConfig = getDefaultCacheConfig()[language]; + const cacheConfig = defaultCacheConfigs[language]; if (cacheConfig === undefined) { logger.info( @@ -172,12 +293,14 @@ export async function uploadDependencyCaches( // Check that we can find files to calculate the hash for the cache key from, so we don't end up // with an empty string. - const globber = await makeGlobber(cacheConfig.hash); - - if ((await globber.glob()).length === 0) { - logger.info( - `Skipping upload of dependency cache for ${language} as we cannot calculate a hash for the cache key.`, - ); + const patterns = await checkHashPatterns( + codeql, + features, + language, + cacheConfig, + logger, + ); + if (patterns === undefined) { continue; } @@ -191,7 +314,11 @@ export async function uploadDependencyCaches( // use the cache quota that we compete with. In that case, we do not wish to use up all of the quota // with the dependency caches. For this, we could use the Cache API to check whether other workflows // are using the quota and how full it is. - const size = await getTotalCacheSize(cacheConfig.paths, logger, true); + const size = await getTotalCacheSize( + cacheConfig.getDependencyPaths(), + logger, + true, + ); // Skip uploading an empty cache. if (size === 0) { @@ -201,14 +328,14 @@ export async function uploadDependencyCaches( continue; } - const key = await cacheKey(language, cacheConfig, minimizeJavaJars); + const key = await cacheKey(codeql, features, language, patterns); logger.info( `Uploading cache of size ${size} for ${language} with key ${key}...`, ); try { - await actionsCache.saveCache(cacheConfig.paths, key); + await actionsCache.saveCache(cacheConfig.getDependencyPaths(), key); } catch (error) { // `ReserveCacheError` indicates that the cache key is already in use, which means that a // cache with that key already exists or is in the process of being uploaded by another @@ -229,31 +356,35 @@ export async function uploadDependencyCaches( /** * Computes a cache key for the specified language. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param language The language being analyzed. * @param cacheConfig The cache configuration for the language. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns A cache key capturing information about the project(s) being analyzed in the specified language. */ async function cacheKey( + codeql: CodeQL, + features: Features, language: Language, - cacheConfig: CacheConfig, - minimizeJavaJars: boolean = false, + patterns: string[], ): Promise { - const hash = await glob.hashFiles(cacheConfig.hash.join("\n")); - return `${await cachePrefix(language, minimizeJavaJars)}${hash}`; + const hash = await glob.hashFiles(patterns.join("\n")); + return `${await cachePrefix(codeql, features, language)}${hash}`; } /** * Constructs a prefix for the cache key, comprised of a CodeQL-specific prefix, a version number that * can be changed to invalidate old caches, the runner's operating system, and the specified language name. * + * @param codeql The CodeQL instance to use. + * @param features Information about which FFs are enabled. * @param language The language being analyzed. - * @param minimizeJavaJars Whether the Java extractor should rewrite downloaded JARs to minimize their size. * @returns The prefix that identifies what a cache is for. */ async function cachePrefix( + codeql: CodeQL, + features: Features, language: Language, - minimizeJavaJars: boolean, ): Promise { const runnerOs = getRequiredEnvParam("RUNNER_OS"); const customPrefix = process.env[EnvVar.DEPENDENCY_CACHING_PREFIX]; @@ -264,6 +395,10 @@ async function cachePrefix( } // To ensure a safe rollout of JAR minimization, we change the key when the feature is enabled. + const minimizeJavaJars = await features.getValue( + Feature.JavaMinimizeDependencyJars, + codeql, + ); if (language === KnownLanguage.java && minimizeJavaJars) { prefix = `minify-${prefix}`; } diff --git a/src/feature-flags.ts b/src/feature-flags.ts index b7946d62f4..89cea7de8d 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -45,6 +45,7 @@ export interface FeatureEnablement { export enum Feature { CleanupTrapCaches = "cleanup_trap_caches", CppDependencyInstallation = "cpp_dependency_installation_enabled", + CsharpNewCacheKey = "csharp_new_cache_key", DiffInformedQueries = "diff_informed_queries", DisableCsharpBuildless = "disable_csharp_buildless", DisableJavaBuildlessEnabled = "disable_java_buildless_enabled", @@ -119,6 +120,11 @@ export const featureConfig: Record< legacyApi: true, minimumVersion: "2.15.0", }, + [Feature.CsharpNewCacheKey]: { + defaultValue: false, + envVar: "CODEQL_ACTION_CSHARP_NEW_CACHE_KEY", + minimumVersion: undefined, + }, [Feature.DiffInformedQueries]: { defaultValue: true, envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES", diff --git a/src/init-action.ts b/src/init-action.ts index 508d17333b..3f3dbb955d 100644 --- a/src/init-action.ts +++ b/src/init-action.ts @@ -547,15 +547,12 @@ async function run() { } // Restore dependency cache(s), if they exist. - const minimizeJavaJars = await features.getValue( - Feature.JavaMinimizeDependencyJars, - codeql, - ); if (shouldRestoreCache(config.dependencyCachingEnabled)) { await downloadDependencyCaches( + codeql, + features, config.languages, logger, - minimizeJavaJars, ); } @@ -617,7 +614,7 @@ async function run() { `${EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS} is already set to '${process.env[EnvVar.JAVA_EXTRACTOR_MINIMIZE_DEPENDENCY_JARS]}', so the Action will not override it.`, ); } else if ( - minimizeJavaJars && + (await features.getValue(Feature.JavaMinimizeDependencyJars, codeql)) && config.dependencyCachingEnabled && config.buildMode === BuildMode.None && config.languages.includes(KnownLanguage.java)