feat: introduce enhanced secret scanning (#6230)

Author: aitchiss

packages/build/docs/flow.md

@@ -95,7 +95,8 @@ Current core steps are:
   [build command](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/build_command.js#L11).
 - [Functions bundling](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/functions/index.js#L142),
   which uses [`zip-it-and-ship-it`](https://github.com/netlify/zip-it-and-ship-it).
-- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys.
+- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys, or when the account has
+  enhancedSecretScan enabled (only applies to certain plan types).
 - [Site deploy](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/deploy/index.js#L66):
   - This sends a network request to the buildbot to initiate site deploy.
   - When the buildbot

packages/build/src/core/build.ts

@@ -85,6 +85,7 @@ const tExecBuild = async function ({
   quiet,
   framework,
   explicitSecretKeys,
+  enhancedSecretScan,
   edgeFunctionsBootstrapURL,
   eventHandlers,
 }) {
@@ -223,6 +224,7 @@ const tExecBuild = async function ({
     quiet,
     integrations,
     explicitSecretKeys,
+    enhancedSecretScan,
     edgeFunctionsBootstrapURL,
     eventHandlers,
   })
@@ -284,6 +286,7 @@ export const runAndReportBuild = async function ({
   quiet,
   integrations,
   explicitSecretKeys,
+  enhancedSecretScan,
   edgeFunctionsBootstrapURL,
   eventHandlers,
 }) {
@@ -340,6 +343,7 @@ export const runAndReportBuild = async function ({
       quiet,
       integrations,
       explicitSecretKeys,
+      enhancedSecretScan,
       edgeFunctionsBootstrapURL,
       eventHandlers,
     })
@@ -446,6 +450,7 @@ const initAndRunBuild = async function ({
   quiet,
   integrations,
   explicitSecretKeys,
+  enhancedSecretScan,
   edgeFunctionsBootstrapURL,
   eventHandlers,
 }) {
@@ -551,6 +556,7 @@ const initAndRunBuild = async function ({
       devCommand,
       quiet,
       explicitSecretKeys,
+      enhancedSecretScan,
       edgeFunctionsBootstrapURL,
       eventHandlers,
     })
@@ -627,6 +633,7 @@ const runBuild = async function ({
   devCommand,
   quiet,
   explicitSecretKeys,
+  enhancedSecretScan,
   edgeFunctionsBootstrapURL,
   eventHandlers,
 }) {
@@ -694,6 +701,7 @@ const runBuild = async function ({
     quiet,
     userNodeVersion,
     explicitSecretKeys,
+    enhancedSecretScan,
     edgeFunctionsBootstrapURL,
   })
 

packages/build/src/core/flags.js

@@ -218,4 +218,9 @@ Default: false`,
     describe: 'Env var keys that are marked as secret explicitly.',
     hidden: true,
   },
+  enhancedSecretScan: {
+    boolean: true,
+    hidden: true,
+    describe: 'Scan for potential secrets in all env vars',
+  },
 }

packages/build/src/log/messages/config.js

@@ -56,6 +56,7 @@ const INTERNAL_FLAGS = [
   'systemLogFile',
   'timeline',
   'explicitSecretKeys',
+  'enhancedSecretScan',
   'edgeFunctionsBootstrapURL',
   'eventHandlers',
 ]

packages/build/src/log/messages/core_steps.js

@@ -129,15 +129,31 @@ export const logSecretsScanSuccessMessage = function (logs, msg) {
 }
 
 export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
+  const { secretMatches, enhancedSecretMatches } = groupedResults
+
   logErrorSubHeader(
     logs,
     `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`,
   )
 
-  Object.keys(groupedResults).forEach((key) => {
+  // Explicit secret matches
+  Object.keys(secretMatches).forEach((key) => {
     logError(logs, `Secret env var "${key}"'s value detected:`)
 
-    groupedResults[key]
+    secretMatches[key]
+      .sort((a, b) => {
+        return a.file > b.file ? 0 : 1
+      })
+      .forEach(({ lineNumber, file }) => {
+        logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true })
+      })
+  })
+
+  // Likely secret matches from enhanced scan
+  Object.keys(enhancedSecretMatches).forEach((key) => {
+    logError(logs, `Env var "${key}"'s value detected as a likely secret value:`)
+
+    enhancedSecretMatches[key]
       .sort((a, b) => {
         return a.file > b.file ? 0 : 1
       })

packages/build/src/plugins_core/secrets_scanning/index.ts

@@ -14,8 +14,9 @@ import {
   ScanResults,
   SecretScanResult,
   getFilePathsToScan,
+  getNonSecretKeysToScanFor,
   getSecretKeysToScanFor,
-  groupScanResultsByKey,
+  groupScanResultsByKeyAndScanType,
   isSecretsScanningEnabled,
   scanFilesForKeyValues,
 } from './utils.js'
@@ -27,6 +28,7 @@ const coreStep: CoreStepFunction = async function ({
   logs,
   netlifyConfig,
   explicitSecretKeys,
+  enhancedSecretScan,
   systemLog,
   deployId,
   api,
@@ -51,13 +53,15 @@ const coreStep: CoreStepFunction = async function ({
     log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`)
   }
 
-  const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys)
+  const explicitSecretKeysToScanFor = getSecretKeysToScanFor(envVars, passedSecretKeys)
+  const potentialSecretKeysToScanFor = enhancedSecretScan ? getNonSecretKeysToScanFor(envVars, passedSecretKeys) : []
+  const keysToSearchFor = explicitSecretKeysToScanFor.concat(potentialSecretKeysToScanFor)
 
   if (keysToSearchFor.length === 0) {
-    logSecretsScanSkipMessage(
-      logs,
-      'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.',
-    )
+    const msg = enhancedSecretScan
+      ? 'Secrets scanning skipped because no env vars are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.'
+      : 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.'
+    logSecretsScanSkipMessage(logs, msg)
     return stepResults
   }
 
@@ -75,6 +79,8 @@ const coreStep: CoreStepFunction = async function ({
   }
 
   let scanResults: ScanResults | undefined
+  let secretMatches: SecretScanResult['secretsScanMatches'] | undefined
+  let enhancedSecretMatches: SecretScanResult['enhancedSecretsScanMatches'] | undefined
 
   await tracer.startActiveSpan(
     'scanning-files',
@@ -87,9 +93,14 @@ const coreStep: CoreStepFunction = async function ({
         filePaths,
       })
 
+      secretMatches = scanResults.matches.filter((match) => explicitSecretKeysToScanFor.includes(match.key))
+      enhancedSecretMatches = scanResults.matches.filter((match) => potentialSecretKeysToScanFor.includes(match.key))
+
       const attributesForLogsAndSpan = {
-        secretsScanFoundSecrets: scanResults.matches.length > 0,
-        secretsScanMatchesCount: scanResults.matches.length,
+        secretsScanFoundSecrets: secretMatches.length > 0,
+        enhancedSecretsScanFoundSecrets: enhancedSecretMatches.length > 0,
+        secretsScanMatchesCount: secretMatches.length,
+        enhancedSecretsScanMatchesCount: enhancedSecretMatches.length,
         secretsFilesCount: scanResults.scannedFilesCount,
         keysToSearchFor,
       }
@@ -103,7 +114,8 @@ const coreStep: CoreStepFunction = async function ({
   if (deployId !== '0') {
     const secretScanResult: SecretScanResult = {
       scannedFilesCount: scanResults?.scannedFilesCount ?? 0,
-      secretsScanMatches: scanResults?.matches ?? [],
+      secretsScanMatches: secretMatches ?? [],
+      enhancedSecretsScanMatches: enhancedSecretMatches ?? [],
     }
     reportValidations({ api, secretScanResult, deployId, systemLog })
   }
@@ -118,18 +130,27 @@ const coreStep: CoreStepFunction = async function ({
 
   // at this point we have found matching secrets
   // Output the results and fail the build
-
-  logSecretsScanFailBuildMessage({ logs, scanResults, groupedResults: groupScanResultsByKey(scanResults) })
+  logSecretsScanFailBuildMessage({
+    logs,
+    scanResults,
+    groupedResults: groupScanResultsByKeyAndScanType(scanResults, potentialSecretKeysToScanFor),
+  })
 
   const error = new Error(`Secrets scanning found secrets in build.`)
   addErrorInfo(error, { type: 'secretScanningFoundSecrets' })
   throw error
 }
 
-// We run this core step if the build was run with explicit secret keys. This
-// is passed from BB to build so only accounts that are allowed to have explicit
-// secrets and actually have them will have them.
-const hasExplicitSecretsKeys: CoreStepCondition = function ({ explicitSecretKeys }): boolean {
+// We run this core step if the build was run with explicit secret keys or if enhanced secret scanning is enabled.
+// This is passed from BB to build so only accounts that are allowed to have explicit
+// secrets and actually have them / have enhanced secret scanning enabled will have them.
+const hasExplicitSecretsKeysOrEnhancedScanningEnabled: CoreStepCondition = function ({
+  explicitSecretKeys,
+  enhancedSecretScan,
+}): boolean {
+  if (enhancedSecretScan) {
+    return true
+  }
   if (typeof explicitSecretKeys !== 'string') {
     return false
   }
@@ -143,5 +164,5 @@ export const scanForSecrets: CoreStep = {
   coreStepId: 'secrets_scanning',
   coreStepName: 'Secrets scanning',
   coreStepDescription: () => 'Scanning for secrets in code and build output.',
-  condition: hasExplicitSecretsKeys,
+  condition: hasExplicitSecretsKeysOrEnhancedScanningEnabled,
 }