Skip to content

Commit 9f063f8

Browse files
aitchisspieh
aitchiss
and
pieh
authored
feat: allow enhanced scan to run without impacting builds (#6376)
--------- Co-authored-by: Michal Piechowiak <[email protected]>
1 parent 0a3cdc2 commit 9f063f8

File tree

3 files changed

+176
-44
lines changed

3 files changed

+176
-44
lines changed

‎packages/build/src/log/messages/core_steps.js‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -128,14 +128,19 @@ export const logSecretsScanSuccessMessage = function (logs, msg) {
128128
log(logs, msg, { color: THEME.highlightWords })
129129
}
130130

131-
export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
131+
export const logSecretsScanFailBuildMessage = function ({
132+
logs,
133+
scanResults,
134+
groupedResults,
135+
enhancedScanShouldRunInActiveMode,
136+
}) {
132137
const { secretMatches, enhancedSecretMatches } = groupedResults
133138
const secretMatchesKeys = Object.keys(secretMatches)
134139
const enhancedSecretMatchesKeys = Object.keys(enhancedSecretMatches)
135140

136141
logErrorSubHeader(
137142
logs,
138-
`Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`,
143+
`Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 && enhancedScanShouldRunInActiveMode ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`,
139144
)
140145

141146
// Explicit secret matches
@@ -162,28 +167,30 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro
162167
)
163168
}
164169

165-
// Likely secret matches from enhanced scan
166-
enhancedSecretMatchesKeys.forEach((key, index) => {
167-
logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`)
168-
169-
enhancedSecretMatches[key]
170-
.sort((a, b) => {
171-
return a.file > b.file ? 0 : 1
172-
})
173-
.forEach(({ lineNumber, file }) => {
174-
logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true })
175-
})
176-
})
170+
if (enhancedScanShouldRunInActiveMode) {
171+
// Likely secret matches from enhanced scan
172+
enhancedSecretMatchesKeys.forEach((key, index) => {
173+
logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`)
174+
175+
enhancedSecretMatches[key]
176+
.sort((a, b) => {
177+
return a.file > b.file ? 0 : 1
178+
})
179+
.forEach(({ lineNumber, file }) => {
180+
logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true })
181+
})
182+
})
177183

178-
if (enhancedSecretMatchesKeys.length) {
179-
logError(
180-
logs,
181-
`\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`,
182-
)
183-
logError(
184-
logs,
185-
`\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`,
186-
)
184+
if (enhancedSecretMatchesKeys.length) {
185+
logError(
186+
logs,
187+
`\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`,
188+
)
189+
logError(
190+
logs,
191+
`\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`,
192+
)
193+
}
187194
}
188195

189196
logError(

‎packages/build/src/plugins_core/secrets_scanning/index.ts‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@ const coreStep: CoreStepFunction = async function ({
3939

4040
const passedSecretKeys = (explicitSecretKeys || '').split(',')
4141
const envVars = netlifyConfig.build.environment as Record<string, unknown>
42+
// When the flag is disabled, we may still run the scan if a secrets scan would otherwise take place anyway
43+
// In this case, we hide any output to the user and simply gather the information in our logs
44+
const enhancedScanShouldRunInActiveMode = featureFlags?.enhanced_secret_scan_impacts_builds ?? false
45+
4246
const useMinimalChunks = featureFlags?.secret_scanning_minimal_chunks
4347

4448
systemLog?.({ passedSecretKeys, buildDir })
@@ -56,15 +60,17 @@ const coreStep: CoreStepFunction = async function ({
5660
log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`)
5761
}
5862
const enhancedScanningEnabledInEnv = isEnhancedSecretsScanningEnabled(envVars)
59-
if (enhancedSecretScan && !enhancedScanningEnabledInEnv) {
63+
const enhancedScanConfigured = enhancedSecretScan && enhancedScanningEnabledInEnv
64+
if (enhancedSecretScan && enhancedScanShouldRunInActiveMode && !enhancedScanningEnabledInEnv) {
6065
logSecretsScanSkipMessage(
6166
logs,
6267
'Enhanced secrets detection disabled via ENHANCED_SECRETS_SCAN_ENABLED flag set to false.',
6368
)
6469
}
70+
6571
if (
66-
enhancedSecretScan &&
67-
enhancedScanningEnabledInEnv &&
72+
enhancedScanShouldRunInActiveMode &&
73+
enhancedScanConfigured &&
6874
envVars['ENHANCED_SECRETS_SCAN_OMIT_VALUES'] !== undefined
6975
) {
7076
log(
@@ -75,7 +81,11 @@ const coreStep: CoreStepFunction = async function ({
7581

7682
const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys)
7783

78-
if (keysToSearchFor.length === 0 && !enhancedSecretScan) {
84+
// In passive mode, only run the enhanced scan if we have explicit secret keys
85+
const enhancedScanShouldRun = enhancedScanShouldRunInActiveMode
86+
? enhancedScanConfigured
87+
: enhancedScanConfigured && keysToSearchFor.length > 0
88+
if (keysToSearchFor.length === 0 && !enhancedScanShouldRun) {
7989
logSecretsScanSkipMessage(
8090
logs,
8191
'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.',
@@ -109,7 +119,7 @@ const coreStep: CoreStepFunction = async function ({
109119
keys: keysToSearchFor,
110120
base: buildDir as string,
111121
filePaths,
112-
enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
122+
enhancedScanning: enhancedScanShouldRun,
113123
omitValuesFromEnhancedScan: getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(envVars),
114124
useMinimalChunks,
115125
})
@@ -125,7 +135,8 @@ const coreStep: CoreStepFunction = async function ({
125135
secretsFilesCount: scanResults.scannedFilesCount,
126136
keysToSearchFor,
127137
enhancedPrefixMatches: enhancedSecretMatches.length ? enhancedSecretMatches.map((match) => match.key) : [],
128-
enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
138+
enhancedScanning: enhancedScanShouldRun,
139+
enhancedScanActiveMode: enhancedScanShouldRunInActiveMode,
129140
}
130141

131142
systemLog?.(attributesForLogsAndSpan)
@@ -138,12 +149,17 @@ const coreStep: CoreStepFunction = async function ({
138149
const secretScanResult: SecretScanResult = {
139150
scannedFilesCount: scanResults?.scannedFilesCount ?? 0,
140151
secretsScanMatches: secretMatches ?? [],
141-
enhancedSecretsScanMatches: enhancedSecretMatches ?? [],
152+
enhancedSecretsScanMatches:
153+
enhancedScanShouldRunInActiveMode && enhancedSecretMatches ? enhancedSecretMatches : [],
142154
}
143155
reportValidations({ api, secretScanResult, deployId, systemLog })
144156
}
145157

146-
if (!scanResults || scanResults.matches.length === 0) {
158+
if (
159+
!scanResults ||
160+
scanResults.matches.length === 0 ||
161+
(!enhancedScanShouldRunInActiveMode && !secretMatches?.length)
162+
) {
147163
logSecretsScanSuccessMessage(
148164
logs,
149165
`Secrets scanning complete. ${scanResults?.scannedFilesCount} file(s) scanned. No secrets detected in build output or repo code!`,
@@ -157,6 +173,7 @@ const coreStep: CoreStepFunction = async function ({
157173
logs,
158174
scanResults,
159175
groupedResults: groupScanResultsByKeyAndScanType(scanResults),
176+
enhancedScanShouldRunInActiveMode,
160177
})
161178

162179
const error = new Error(`Secrets scanning found secrets in build.`)

‎packages/build/tests/secrets_scanning/tests.js‎

Lines changed: 121 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -258,11 +258,17 @@ for (const { testPrefix, featureFlags } of [
258258
t.true(output.includes(`No secrets detected in build output or repo code!`))
259259
})
260260

261-
// Enhanced secret scanning
261+
// Enhanced secret scanning with enhanced_secret_scan_impacts_builds enabled
262262

263263
test(testPrefix + 'secrets scanning, enhanced scan should not run when disabled', async (t) => {
264264
const { requests } = await new Fixture('./fixtures/src_scanning_disabled')
265-
.withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test', featureFlags })
265+
.withFlags({
266+
debug: false,
267+
enhancedSecretScan: true,
268+
deployId: 'test',
269+
token: 'test',
270+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
271+
})
266272
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
267273
t.true(requests.length === 0)
268274
})
@@ -278,7 +284,7 @@ for (const { testPrefix, featureFlags } of [
278284
enhancedSecretScan: true,
279285
deployId: 'test',
280286
token: 'test',
281-
featureFlags,
287+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
282288
})
283289
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
284290

@@ -293,11 +299,11 @@ for (const { testPrefix, featureFlags } of [
293299
const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_disabled')
294300
.withFlags({
295301
debug: false,
296-
explicitSecretKeys: '',
302+
explicitSecretKeys: 'ENV_VAR_1',
297303
enhancedSecretScan: true,
298304
deployId: 'test',
299305
token: 'test',
300-
featureFlags,
306+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
301307
})
302308
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
303309
t.true(requests.length === 1)
@@ -316,7 +322,7 @@ for (const { testPrefix, featureFlags } of [
316322
enhancedSecretScan: true,
317323
deployId: 'test',
318324
token: 'test',
319-
featureFlags,
325+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
320326
})
321327
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
322328

@@ -337,7 +343,7 @@ for (const { testPrefix, featureFlags } of [
337343
enhancedSecretScan: false,
338344
deployId: 'test',
339345
token: 'test',
340-
featureFlags,
346+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
341347
})
342348
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
343349

@@ -353,7 +359,7 @@ for (const { testPrefix, featureFlags } of [
353359
enhancedSecretScan: true,
354360
deployId: 'test',
355361
token: 'test',
356-
featureFlags,
362+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
357363
})
358364
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
359365

@@ -367,7 +373,13 @@ for (const { testPrefix, featureFlags } of [
367373

368374
test(testPrefix + 'secrets scanning, should not find secrets in files without known prefixes', async (t) => {
369375
const { requests } = await new Fixture('./fixtures/src_scanning_no_likely_enhanced_scan_secrets', featureFlags)
370-
.withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test' })
376+
.withFlags({
377+
debug: false,
378+
enhancedSecretScan: true,
379+
deployId: 'test',
380+
token: 'test',
381+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
382+
})
371383
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
372384

373385
t.true(requests.length === 1)
@@ -389,7 +401,7 @@ for (const { testPrefix, featureFlags } of [
389401
enhancedSecretScan: true,
390402
deployId: 'test',
391403
token: 'test',
392-
featureFlags,
404+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
393405
})
394406
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
395407

@@ -414,7 +426,7 @@ for (const { testPrefix, featureFlags } of [
414426
enhancedSecretScan: true,
415427
deployId: 'test',
416428
token: 'test',
417-
featureFlags,
429+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
418430
})
419431
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
420432

@@ -443,7 +455,7 @@ for (const { testPrefix, featureFlags } of [
443455
enhancedSecretScan: true,
444456
deployId: 'test',
445457
token: 'test',
446-
featureFlags,
458+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
447459
})
448460
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
449461

@@ -463,12 +475,108 @@ for (const { testPrefix, featureFlags } of [
463475
debug: false,
464476
explicitSecretKeys: '',
465477
enhancedSecretScan: true,
466-
featureFlags,
478+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
467479
})
468480
.runBuildProgrammatic()
469481
// Severity code of 2 is user error
470482
t.is(severityCode, 2)
471483
})
484+
485+
// enhanced scanning enabled, but without impact to builds
486+
487+
test(
488+
testPrefix +
489+
'secrets scanning, should not log enhanced scan info when enhanced_secret_scan_impacts_builds is false',
490+
async (t) => {
491+
const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
492+
.withFlags({
493+
debug: false,
494+
enhancedSecretScan: true,
495+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
496+
deployId: 'test',
497+
token: 'test',
498+
})
499+
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
500+
501+
const normalizedOutput = normalizeOutput(output)
502+
t.false(normalizedOutput.includes('detected as a likely secret'))
503+
},
504+
)
505+
506+
test(
507+
testPrefix +
508+
'secrets scanning, should not fail build when enhanced scan finds likely secrets but enhanced_secret_scan_impacts_builds is false',
509+
async (t) => {
510+
const { severityCode } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
511+
.withFlags({
512+
debug: false,
513+
enhancedSecretScan: true,
514+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
515+
})
516+
.runBuildProgrammatic()
517+
518+
// Severity code of 0 means success, 2 would be user error
519+
t.is(severityCode, 0)
520+
},
521+
)
522+
523+
test(
524+
testPrefix +
525+
'secrets scanning, should not log omit values message when enhanced_secret_scan_impacts_builds is false',
526+
async (t) => {
527+
const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_omitted')
528+
.withFlags({
529+
debug: false,
530+
enhancedSecretScan: true,
531+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
532+
deployId: 'test',
533+
token: 'test',
534+
})
535+
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
536+
537+
t.false(normalizeOutput(output).includes('ENHANCED_SECRETS_SCAN_OMIT_VALUES'))
538+
},
539+
)
540+
541+
test(
542+
testPrefix + 'secrets scanning, should run enhanced scan in passive mode when explicit keys are present',
543+
async (t) => {
544+
const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty')
545+
.withFlags({
546+
debug: false,
547+
explicitSecretKeys: 'ENV_VAR_1',
548+
enhancedSecretScan: true,
549+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
550+
deployId: 'test',
551+
token: 'test',
552+
})
553+
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
554+
555+
t.true(requests.length === 1)
556+
const request = requests[0]
557+
t.is(request.url, '/api/v1/deploys/test/validations_report')
558+
t.truthy(request.body.secrets_scan.scannedFilesCount)
559+
t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches)
560+
},
561+
)
562+
563+
test(
564+
testPrefix + 'secrets scanning, should not run enhanced scan in passive mode when no explicit keys',
565+
async (t) => {
566+
const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
567+
.withFlags({
568+
debug: false,
569+
explicitSecretKeys: '',
570+
enhancedSecretScan: true,
571+
featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
572+
deployId: 'test',
573+
token: 'test',
574+
})
575+
.runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
576+
577+
t.true(requests.length === 0)
578+
},
579+
)
472580
;(featureFlags.secret_scanning_minimal_chunks ? test : test.skip)(
473581
testPrefix + 'does not crash if line in scanned file exceed available memory',
474582
async (t) => {

0 commit comments

Comments
 (0)