From 3ed91ff38c28599691716fb9b8de23a14f4f3a92 Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Fri, 18 Jul 2025 11:43:21 +0200 Subject: [PATCH 1/2] make sure to account for target world for build cache Signed-off-by: Karthik Ganeshram --- packages/build-tools/src/build.ts | 7 ++++--- packages/build-tools/src/index.ts | 26 +++++++++++++++----------- packages/build-tools/src/utils.ts | 11 +++++++++-- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/build-tools/src/build.ts b/packages/build-tools/src/build.ts index 0d2007b6..b661fa22 100644 --- a/packages/build-tools/src/build.ts +++ b/packages/build-tools/src/build.ts @@ -1,5 +1,5 @@ import { - calculateChecksum, + calculateFileChecksum, fileExists, getExistingBuildData, } from './utils.js'; @@ -9,15 +9,16 @@ export function getBuildDataPath(src: string): string { } export async function ShouldComponentize( -src: string, outputPath: string, componentizeVersion: string, runtimeArgs: string, + src: string, outputPath: string, componentizeVersion: string, runtimeArgs: string, targetWitChecksum: string ) { - const sourceChecksum = await calculateChecksum(src); + const sourceChecksum = await calculateFileChecksum(src); const existingBuildData = await getExistingBuildData(getBuildDataPath(src)); if ( existingBuildData?.version == componentizeVersion && existingBuildData?.checksum === sourceChecksum && existingBuildData?.runtimeArgs === runtimeArgs && + existingBuildData?.targetWitChecksum === targetWitChecksum && (await fileExists(outputPath)) ) { return false; diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index 08cc9e46..2fcb7bb3 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -4,7 +4,8 @@ import { componentize } from '@bytecodealliance/componentize-js'; import { version as componentizeVersion } from '@bytecodealliance/componentize-js'; import { getPackagesWithWasiDeps, processWasiDeps } from './wasiDepsParser.js'; import { - calculateChecksum, + calculateCheckSum, + calculateFileChecksum, saveBuildData, } from './utils.js'; import { getCliArgs } from './cli.js'; @@ -23,15 +24,6 @@ async function main() { let outputPath = CliArgs.output; let runtimeArgs = CliArgs.debug ? '--enable-script-debugging' : ''; - // Small optimization to skip componentization if the source file hasn't changed - if (!(await ShouldComponentize(src, outputPath, componentizeVersion, runtimeArgs))) { - console.log( - 'No changes detected in source file. Skipping componentization.', - ); - return; - } - console.log('Componentizing...'); - // generate wit world string let wasiDeps = getPackagesWithWasiDeps(process.cwd(), new Set(), true); let { witPaths, targetWorlds } = processWasiDeps(wasiDeps); @@ -63,6 +55,17 @@ async function main() { 'combined-wit:combined-wit@0.3.0', ); + // Calcuilate the checksum of the inline wit + let inlineWitChecksum = await calculateCheckSum(inlineWit); + // Small optimization to skip componentization if the source file hasn't changed + if (!(await ShouldComponentize(src, outputPath, componentizeVersion, runtimeArgs, inlineWitChecksum))) { + console.log( + 'No changes detected in source file and target World. Skipping componentization.', + ); + return; + } + console.log('Componentizing...'); + const source = await readFile(src, 'utf8'); const precompiledSource = precompile(source, src, true) as string; @@ -85,9 +88,10 @@ async function main() { // Save the checksum of the input file along with the componentize version await saveBuildData( getBuildDataPath(src), - await calculateChecksum(src), + await calculateFileChecksum(src), componentizeVersion, runtimeArgs, + inlineWitChecksum, ); console.log('Component successfully written.'); diff --git a/packages/build-tools/src/utils.ts b/packages/build-tools/src/utils.ts index dea5fdf8..5b118bd6 100644 --- a/packages/build-tools/src/utils.ts +++ b/packages/build-tools/src/utils.ts @@ -4,7 +4,7 @@ import { access, writeFile } from 'node:fs/promises'; // Function to calculate file checksum -export async function calculateChecksum(filePath: string) { +export async function calculateFileChecksum(filePath: string) { try { const fileBuffer = await readFile(filePath); const hash = createHash('sha256'); @@ -16,6 +16,12 @@ export async function calculateChecksum(filePath: string) { } } +export function calculateCheckSum(data: string | Buffer): string { + const hash = createHash('sha256'); + hash.update(data); + return hash.digest('hex'); +} + // Function to check if a file exists export async function fileExists(filePath: string) { try { @@ -43,13 +49,14 @@ export async function getExistingBuildData(buildDataPath: string) { } export async function saveBuildData( -buildDataPath: string, checksum: string, version: string, runtimeArgs: string, + buildDataPath: string, checksum: string, version: string, runtimeArgs: string, targetWitChecksum: string ) { try { const checksumData = { version, checksum, runtimeArgs, + targetWitChecksum, }; await writeFile(buildDataPath, JSON.stringify(checksumData, null, 2)); } catch (error) { From d9f35227db5030d9a0af5861d3b5a0bdce1610c6 Mon Sep 17 00:00:00 2001 From: Karthik Ganeshram Date: Mon, 21 Jul 2025 21:01:08 +0200 Subject: [PATCH 2/2] cleanups Signed-off-by: Karthik Ganeshram --- packages/build-tools/src/build.ts | 5 +++-- packages/build-tools/src/index.ts | 8 +++----- packages/build-tools/src/utils.ts | 12 +++--------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/packages/build-tools/src/build.ts b/packages/build-tools/src/build.ts index b661fa22..2ae12ff4 100644 --- a/packages/build-tools/src/build.ts +++ b/packages/build-tools/src/build.ts @@ -1,5 +1,6 @@ +import { readFile } from 'node:fs/promises'; import { - calculateFileChecksum, + calculateChecksum, fileExists, getExistingBuildData, } from './utils.js'; @@ -11,7 +12,7 @@ export function getBuildDataPath(src: string): string { export async function ShouldComponentize( src: string, outputPath: string, componentizeVersion: string, runtimeArgs: string, targetWitChecksum: string ) { - const sourceChecksum = await calculateFileChecksum(src); + const sourceChecksum = await calculateChecksum(await readFile(src)); const existingBuildData = await getExistingBuildData(getBuildDataPath(src)); if ( diff --git a/packages/build-tools/src/index.ts b/packages/build-tools/src/index.ts index 2fcb7bb3..24a87e9b 100644 --- a/packages/build-tools/src/index.ts +++ b/packages/build-tools/src/index.ts @@ -4,8 +4,7 @@ import { componentize } from '@bytecodealliance/componentize-js'; import { version as componentizeVersion } from '@bytecodealliance/componentize-js'; import { getPackagesWithWasiDeps, processWasiDeps } from './wasiDepsParser.js'; import { - calculateCheckSum, - calculateFileChecksum, + calculateChecksum, saveBuildData, } from './utils.js'; import { getCliArgs } from './cli.js'; @@ -55,8 +54,7 @@ async function main() { 'combined-wit:combined-wit@0.3.0', ); - // Calcuilate the checksum of the inline wit - let inlineWitChecksum = await calculateCheckSum(inlineWit); + let inlineWitChecksum = await calculateChecksum(inlineWit); // Small optimization to skip componentization if the source file hasn't changed if (!(await ShouldComponentize(src, outputPath, componentizeVersion, runtimeArgs, inlineWitChecksum))) { console.log( @@ -88,7 +86,7 @@ async function main() { // Save the checksum of the input file along with the componentize version await saveBuildData( getBuildDataPath(src), - await calculateFileChecksum(src), + await calculateChecksum(await readFile(src)), componentizeVersion, runtimeArgs, inlineWitChecksum, diff --git a/packages/build-tools/src/utils.ts b/packages/build-tools/src/utils.ts index 5b118bd6..cfc26ea2 100644 --- a/packages/build-tools/src/utils.ts +++ b/packages/build-tools/src/utils.ts @@ -4,23 +4,17 @@ import { access, writeFile } from 'node:fs/promises'; // Function to calculate file checksum -export async function calculateFileChecksum(filePath: string) { +export async function calculateChecksum(content: string | Buffer) { try { - const fileBuffer = await readFile(filePath); const hash = createHash('sha256'); - hash.update(fileBuffer); + hash.update(content); return hash.digest('hex'); } catch (error) { - console.error(`Error calculating checksum for file ${filePath}:`, error); + console.error(`Error calculating checksum:`, error); throw error; } } -export function calculateCheckSum(data: string | Buffer): string { - const hash = createHash('sha256'); - hash.update(data); - return hash.digest('hex'); -} // Function to check if a file exists export async function fileExists(filePath: string) {