From bf6cf095394d7f75e549637f958c62c35ef0ecd4 Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 13:52:53 -0400 Subject: [PATCH 1/7] fix: .test is faster than .match regex --- lib/extractor/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/extractor/index.ts b/lib/extractor/index.ts index 031c2fe6..5994880b 100644 --- a/lib/extractor/index.ts +++ b/lib/extractor/index.ts @@ -228,7 +228,7 @@ function layersWithLatestFileModifications( // if finding a deleted file - trimming to its original file name for excluding it from extractedLayers // + not adding this file if (isWhitedOutFile(filename)) { - removedFilesToIgnore.add(filename.replace(/.wh./, "")); + removedFilesToIgnore.add(filename.replace(/\.wh\./, "")); continue; } // not adding previously found to be whited out files to extractedLayers @@ -248,8 +248,13 @@ function layersWithLatestFileModifications( return extractedLayers; } +/** +* check if a file is 'whited out', which is shown by +* prefixing the filename with a .wh. +* https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ +*/ export function isWhitedOutFile(filename: string) { - return filename.match(/.wh./gm); + return /\.wh\./.test(filename); } function isBufferType(type: FileContent): type is Buffer { From f45e43f8785787ad8890094eb1501322c3ff2b7a Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 14:01:34 -0400 Subject: [PATCH 2/7] chore: we don't need to keep looping --- lib/inputs/file-pattern/static.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/inputs/file-pattern/static.ts b/lib/inputs/file-pattern/static.ts index 3f307fb8..d0de05a1 100644 --- a/lib/inputs/file-pattern/static.ts +++ b/lib/inputs/file-pattern/static.ts @@ -5,25 +5,20 @@ import { ExtractAction, ExtractedLayers } from "../../extractor/types"; import { streamToString } from "../../stream-utils"; import { ManifestFile } from "../../types"; +/** + * Return false if any exclusion pattern matches, + * return true if any inclusion pattern matches + */ function generatePathMatcher( globsInclude: string[], globsExclude: string[], ): (filePath: string) => boolean { return (filePath: string): boolean => { - let exclude = false; - for (const g of globsExclude) { - if (!exclude && minimatch(filePath, g)) { - exclude = true; - } - } - if (!exclude) { - for (const g of globsInclude) { - if (minimatch(filePath, g)) { - return true; - } - } + if (globsExclude.some(glob => minimatch(filePath, glob))) { + return false; } - return false; + + return globsInclude.some(glob => minimatch(filePath, glob)); }; } From 2073d767a009affad8dd790f5fc36d6d56e6a98f Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 14:12:08 -0400 Subject: [PATCH 3/7] chore: unit tests --- test/lib/extractor/index.spec.ts | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/lib/extractor/index.spec.ts b/test/lib/extractor/index.spec.ts index ad9ed146..e25435dd 100644 --- a/test/lib/extractor/index.spec.ts +++ b/test/lib/extractor/index.spec.ts @@ -1,5 +1,6 @@ import { getContentAsString } from "../../../lib/extractor"; import { ExtractAction, ExtractedLayers } from "../../../lib/extractor/types"; +import { isWhitedOutFile } from "../../../lib/extractor"; describe("index", () => { test("getContentAsString() does matches when a pattern is used in the extract action", async () => { @@ -18,3 +19,39 @@ describe("index", () => { expect(result).toEqual("Hello, world!"); }); }); + +describe("isWhitedOutFile", () => { + test("should return true for files containing .wh. in their path", () => { + expect(isWhitedOutFile("/etc/.wh.hosts")).toBe(true); + expect(isWhitedOutFile("/var/lib/.wh.data")).toBe(true); + expect(isWhitedOutFile("/.wh.config")).toBe(true); + }); + + test("should return false for files not containing .wh.", () => { + expect(isWhitedOutFile("/etc/hosts")).toBe(false); + expect(isWhitedOutFile("")).toBe(false); + expect(isWhitedOutFile("/")).toBe(false); + }); + + test("should return false for similar but different patterns", () => { + // make sure the dots are literal and not match all + expect(isWhitedOutFile("/etc/wh.hosts")).toBe(false); + expect(isWhitedOutFile("/etc/.whosts")).toBe(false); + expect(isWhitedOutFile("/etc/whhosts")).toBe(false); + + // dots in wrong places + expect(isWhitedOutFile("/etc/.w.h.hosts")).toBe(false); + expect(isWhitedOutFile("/etc/..wh..hosts")).toBe(false); + + // case sensitive + expect(isWhitedOutFile("/etc/.WH.hosts")).toBe(false); + expect(isWhitedOutFile("/etc/.Wh.hosts")).toBe(false); + }); + + test("should handle .wh. at different positions", () => { + expect(isWhitedOutFile(".wh.start")).toBe(true); + expect(isWhitedOutFile("middle.wh.file")).toBe(true); + expect(isWhitedOutFile("end.wh.")).toBe(true); + expect(isWhitedOutFile("/deeply/nested/path/.wh.present")).toBe(true); + }); +}); \ No newline at end of file From 764d0223822c6b30cf900ab3ca465608cad62510 Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 14:24:05 -0400 Subject: [PATCH 4/7] chore: this code hurt me to read --- lib/analyzer/image-inspector.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/analyzer/image-inspector.ts b/lib/analyzer/image-inspector.ts index ce11c78a..7c9f956e 100644 --- a/lib/analyzer/image-inspector.ts +++ b/lib/analyzer/image-inspector.ts @@ -48,7 +48,6 @@ async function pullWithDockerBinary( password: string | undefined, platform: string | undefined, ): Promise { - let pullAndSaveSuccessful = false; try { if (username || password) { debug( @@ -57,13 +56,13 @@ async function pullWithDockerBinary( } await docker.pullCli(targetImage, { platform }); await docker.save(targetImage, saveLocation); - return (pullAndSaveSuccessful = true); + return true; } catch (err) { debug(`couldn't pull ${targetImage} using docker binary: ${err.message}`); handleDockerPullError(err.stderr, platform); - return pullAndSaveSuccessful; + return false; } } From 83d1925001c59a020e833bf6af6e0fa281ab8276 Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 15:02:24 -0400 Subject: [PATCH 5/7] chore: improve readability / consistency --- lib/analyzer/image-inspector.ts | 8 ++------ lib/extractor/index.ts | 2 +- lib/python-parser/requirements-parser.ts | 6 +++--- test/lib/extractor/index.spec.ts | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/analyzer/image-inspector.ts b/lib/analyzer/image-inspector.ts index 7c9f956e..ad1cb623 100644 --- a/lib/analyzer/image-inspector.ts +++ b/lib/analyzer/image-inspector.ts @@ -54,12 +54,11 @@ async function pullWithDockerBinary( "using local docker binary credentials. the credentials you provided will be ignored", ); } - await docker.pullCli(targetImage, { platform }); + await docker.pullCli(targetImage, {platform}); await docker.save(targetImage, saveLocation); return true; } catch (err) { debug(`couldn't pull ${targetImage} using docker binary: ${err.message}`); - handleDockerPullError(err.stderr, platform); return false; @@ -80,10 +79,7 @@ function handleDockerPullError(err: string, platform?: string) { "manifest unknown", ]; if (unknownManifestConditions.some((value) => err.includes(value))) { - if (platform) { - throw new Error(`The image does not exist for ${platform}`); - } - throw new Error(`The image does not exist for the current platform`); + throw new Error(`The image does not exist for ${platform ?? "the current platform"}`); } if (err.includes("invalid reference format")) { diff --git a/lib/extractor/index.ts b/lib/extractor/index.ts index 5994880b..33ca03e6 100644 --- a/lib/extractor/index.ts +++ b/lib/extractor/index.ts @@ -254,7 +254,7 @@ function layersWithLatestFileModifications( * https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ */ export function isWhitedOutFile(filename: string) { - return /\.wh\./.test(filename); + return filename.includes(".wh."); } function isBufferType(type: FileContent): type is Buffer { diff --git a/lib/python-parser/requirements-parser.ts b/lib/python-parser/requirements-parser.ts index 8fc4d136..a9b7350c 100644 --- a/lib/python-parser/requirements-parser.ts +++ b/lib/python-parser/requirements-parser.ts @@ -15,7 +15,8 @@ export function getRequirements(fileContent: string): PythonRequirement[] { function parseLine(line: string): PythonRequirement | null { line = line.trim(); - if (line.length === 0) { + // there's no point in calling the regex if the line is a comment + if (line.length === 0 || line.startsWith("#")) { return null; } const parsedLine = VERSION_PARSE_REGEX.exec(line); @@ -23,10 +24,9 @@ function parseLine(line: string): PythonRequirement | null { return null; } const { name, extras, specifier, version } = parsedLine.groups; - const correctedSpecifier = specifierValidRange(specifier, version); return { name: name.toLowerCase(), - specifier: correctedSpecifier, + specifier: specifierValidRange(specifier, version), version, extras: parseExtraNames(extras), } as PythonRequirement; diff --git a/test/lib/extractor/index.spec.ts b/test/lib/extractor/index.spec.ts index e25435dd..2455230b 100644 --- a/test/lib/extractor/index.spec.ts +++ b/test/lib/extractor/index.spec.ts @@ -54,4 +54,4 @@ describe("isWhitedOutFile", () => { expect(isWhitedOutFile("end.wh.")).toBe(true); expect(isWhitedOutFile("/deeply/nested/path/.wh.present")).toBe(true); }); -}); \ No newline at end of file +}); From df601b70b7ab5288beb2cea0aa5a777379613710 Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Fri, 8 Aug 2025 15:09:56 -0400 Subject: [PATCH 6/7] chore: formatting --- lib/analyzer/image-inspector.ts | 6 ++++-- lib/extractor/index.ts | 10 +++++----- lib/inputs/file-pattern/static.ts | 6 +++--- test/lib/extractor/index.spec.ts | 7 +++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/analyzer/image-inspector.ts b/lib/analyzer/image-inspector.ts index ad1cb623..b5091e2e 100644 --- a/lib/analyzer/image-inspector.ts +++ b/lib/analyzer/image-inspector.ts @@ -54,7 +54,7 @@ async function pullWithDockerBinary( "using local docker binary credentials. the credentials you provided will be ignored", ); } - await docker.pullCli(targetImage, {platform}); + await docker.pullCli(targetImage, { platform }); await docker.save(targetImage, saveLocation); return true; } catch (err) { @@ -79,7 +79,9 @@ function handleDockerPullError(err: string, platform?: string) { "manifest unknown", ]; if (unknownManifestConditions.some((value) => err.includes(value))) { - throw new Error(`The image does not exist for ${platform ?? "the current platform"}`); + throw new Error( + `The image does not exist for ${platform ?? "the current platform"}`, + ); } if (err.includes("invalid reference format")) { diff --git a/lib/extractor/index.ts b/lib/extractor/index.ts index 33ca03e6..4b5a8ce1 100644 --- a/lib/extractor/index.ts +++ b/lib/extractor/index.ts @@ -248,11 +248,11 @@ function layersWithLatestFileModifications( return extractedLayers; } -/** -* check if a file is 'whited out', which is shown by -* prefixing the filename with a .wh. -* https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ -*/ +/** + * check if a file is 'whited out', which is shown by + * prefixing the filename with a .wh. + * https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ + */ export function isWhitedOutFile(filename: string) { return filename.includes(".wh."); } diff --git a/lib/inputs/file-pattern/static.ts b/lib/inputs/file-pattern/static.ts index d0de05a1..11f0a15d 100644 --- a/lib/inputs/file-pattern/static.ts +++ b/lib/inputs/file-pattern/static.ts @@ -14,11 +14,11 @@ function generatePathMatcher( globsExclude: string[], ): (filePath: string) => boolean { return (filePath: string): boolean => { - if (globsExclude.some(glob => minimatch(filePath, glob))) { + if (globsExclude.some((glob) => minimatch(filePath, glob))) { return false; } - - return globsInclude.some(glob => minimatch(filePath, glob)); + + return globsInclude.some((glob) => minimatch(filePath, glob)); }; } diff --git a/test/lib/extractor/index.spec.ts b/test/lib/extractor/index.spec.ts index 2455230b..3c97190c 100644 --- a/test/lib/extractor/index.spec.ts +++ b/test/lib/extractor/index.spec.ts @@ -1,6 +1,5 @@ -import { getContentAsString } from "../../../lib/extractor"; +import { getContentAsString, isWhitedOutFile } from "../../../lib/extractor"; import { ExtractAction, ExtractedLayers } from "../../../lib/extractor/types"; -import { isWhitedOutFile } from "../../../lib/extractor"; describe("index", () => { test("getContentAsString() does matches when a pattern is used in the extract action", async () => { @@ -38,11 +37,11 @@ describe("isWhitedOutFile", () => { expect(isWhitedOutFile("/etc/wh.hosts")).toBe(false); expect(isWhitedOutFile("/etc/.whosts")).toBe(false); expect(isWhitedOutFile("/etc/whhosts")).toBe(false); - + // dots in wrong places expect(isWhitedOutFile("/etc/.w.h.hosts")).toBe(false); expect(isWhitedOutFile("/etc/..wh..hosts")).toBe(false); - + // case sensitive expect(isWhitedOutFile("/etc/.WH.hosts")).toBe(false); expect(isWhitedOutFile("/etc/.Wh.hosts")).toBe(false); From 85220d37550e7fd74ff8c66912f2af37903657f6 Mon Sep 17 00:00:00 2001 From: Parker Kuivila Date: Mon, 11 Aug 2025 14:33:11 -0400 Subject: [PATCH 7/7] chore: restore wh related (out of scope) --- lib/extractor/index.ts | 9 ++------ test/lib/extractor/index.spec.ts | 38 +------------------------------- 2 files changed, 3 insertions(+), 44 deletions(-) diff --git a/lib/extractor/index.ts b/lib/extractor/index.ts index 4b5a8ce1..031c2fe6 100644 --- a/lib/extractor/index.ts +++ b/lib/extractor/index.ts @@ -228,7 +228,7 @@ function layersWithLatestFileModifications( // if finding a deleted file - trimming to its original file name for excluding it from extractedLayers // + not adding this file if (isWhitedOutFile(filename)) { - removedFilesToIgnore.add(filename.replace(/\.wh\./, "")); + removedFilesToIgnore.add(filename.replace(/.wh./, "")); continue; } // not adding previously found to be whited out files to extractedLayers @@ -248,13 +248,8 @@ function layersWithLatestFileModifications( return extractedLayers; } -/** - * check if a file is 'whited out', which is shown by - * prefixing the filename with a .wh. - * https://www.madebymikal.com/interpreting-whiteout-files-in-docker-image-layers/ - */ export function isWhitedOutFile(filename: string) { - return filename.includes(".wh."); + return filename.match(/.wh./gm); } function isBufferType(type: FileContent): type is Buffer { diff --git a/test/lib/extractor/index.spec.ts b/test/lib/extractor/index.spec.ts index 3c97190c..ad9ed146 100644 --- a/test/lib/extractor/index.spec.ts +++ b/test/lib/extractor/index.spec.ts @@ -1,4 +1,4 @@ -import { getContentAsString, isWhitedOutFile } from "../../../lib/extractor"; +import { getContentAsString } from "../../../lib/extractor"; import { ExtractAction, ExtractedLayers } from "../../../lib/extractor/types"; describe("index", () => { @@ -18,39 +18,3 @@ describe("index", () => { expect(result).toEqual("Hello, world!"); }); }); - -describe("isWhitedOutFile", () => { - test("should return true for files containing .wh. in their path", () => { - expect(isWhitedOutFile("/etc/.wh.hosts")).toBe(true); - expect(isWhitedOutFile("/var/lib/.wh.data")).toBe(true); - expect(isWhitedOutFile("/.wh.config")).toBe(true); - }); - - test("should return false for files not containing .wh.", () => { - expect(isWhitedOutFile("/etc/hosts")).toBe(false); - expect(isWhitedOutFile("")).toBe(false); - expect(isWhitedOutFile("/")).toBe(false); - }); - - test("should return false for similar but different patterns", () => { - // make sure the dots are literal and not match all - expect(isWhitedOutFile("/etc/wh.hosts")).toBe(false); - expect(isWhitedOutFile("/etc/.whosts")).toBe(false); - expect(isWhitedOutFile("/etc/whhosts")).toBe(false); - - // dots in wrong places - expect(isWhitedOutFile("/etc/.w.h.hosts")).toBe(false); - expect(isWhitedOutFile("/etc/..wh..hosts")).toBe(false); - - // case sensitive - expect(isWhitedOutFile("/etc/.WH.hosts")).toBe(false); - expect(isWhitedOutFile("/etc/.Wh.hosts")).toBe(false); - }); - - test("should handle .wh. at different positions", () => { - expect(isWhitedOutFile(".wh.start")).toBe(true); - expect(isWhitedOutFile("middle.wh.file")).toBe(true); - expect(isWhitedOutFile("end.wh.")).toBe(true); - expect(isWhitedOutFile("/deeply/nested/path/.wh.present")).toBe(true); - }); -});