refactor(resources): improve embedded resources handling and caching

moazbuilds · moazbuilds · commit 432d553387f2 · 2025-11-22T22:52:32.000+02:00
- Move from TS stub to JSON format for embedded resources metadata
- Add better metadata handling with fallback defaults
- Support cache-only and json-only generation modes
- Include generatedAt timestamp in metadata
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,4 @@ package-lock.json
 
 # MkDocs build output
 site/
+src/shared/resources/embedded-resources.json
diff --git a/scripts/build-binaries.mjs b/scripts/build-binaries.mjs
@@ -43,7 +43,7 @@ console.log(`\n${bold}${cyan}╭────────────────
 console.log(`${bold}${cyan}│${reset}  Building ${bold}CodeMachine${reset} v${mainVersion}  ${bold}${cyan}│${reset}`);
 console.log(`${bold}${cyan}╰────────────────────────────────────────╯${reset}\n`);
 
-await generateEmbeddedResources({ quiet: true, writeStub: false });
+await generateEmbeddedResources({ quiet: true, writeJson: true, writeCache: true });
 console.log(`${green}✓${reset} ${dim}Embedded resources refreshed${reset}`);
 
 // Auto-sync platform package versions before building
diff --git a/scripts/generate-embedded-resources.mjs b/scripts/generate-embedded-resources.mjs
@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
 import { mkdirSync, readdirSync, statSync, readFileSync, writeFileSync } from 'node:fs';
-import { join, dirname, relative } from 'node:path';
+import { join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { brotliCompressSync } from 'node:zlib';
 import { createHash } from 'node:crypto';
@@ -9,7 +9,7 @@ import { homedir, tmpdir } from 'node:os';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const repoRoot = join(__dirname, '..');
 const resourcesDir = join(repoRoot, 'src', 'shared', 'resources');
-const targetPath = join(resourcesDir, 'embedded-resources.ts');
+const targetJsonPath = join(resourcesDir, 'embedded-resources.json');
 const packageJsonPath = join(repoRoot, 'package.json');
 const cacheDirName = 'embedded-resources';
 
@@ -51,7 +51,15 @@ function getCacheBase() {
 
 export async function generateEmbeddedResources(options = {}) {
   const include = options.include ?? DEFAULT_INCLUDE;
-  const shouldWriteStub = options.writeStub ?? true;
+  let shouldWriteJson = options.writeJson;
+  if (shouldWriteJson === undefined && options.writeStub === true) {
+    // Backwards compatibility for callers that previously passed writeStub
+    shouldWriteJson = true;
+  }
+  if (typeof shouldWriteJson !== 'boolean') {
+    shouldWriteJson = true;
+  }
+  const shouldWriteCache = options.writeCache ?? true;
   const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8'));
   const files = [];
 
@@ -84,20 +92,13 @@ export async function generateEmbeddedResources(options = {}) {
   const base64 = compressed.toString('base64');
   const hash = createHash('sha256').update(compressed).digest('hex');
 
-  const fileContents = `// AUTO-GENERATED FILE. DO NOT EDIT.
-// Run scripts/generate-embedded-resources.mjs to regenerate.
-import { Buffer } from 'node:buffer';
-
-export const EMBEDDED_RESOURCES_VERSION = '${packageJson.version}';
-export const EMBEDDED_RESOURCES_HASH = '${hash}';
-export const EMBEDDED_RESOURCES_SIZE = ${compressed.length};
-
-const EMBEDDED_BASE64 = ${JSON.stringify(base64)};
-
-export function getEmbeddedResourceArchive(): Buffer {
-  return Buffer.from(EMBEDDED_BASE64, 'base64');
-}
-`;
+  const archiveRecord = {
+    version: packageJson.version,
+    generatedAt: payload.generatedAt,
+    hash,
+    size: compressed.length,
+    base64,
+  };
 
   const writeCache = () => {
     let cacheBase = getCacheBase();
@@ -106,12 +107,7 @@ export function getEmbeddedResourceArchive(): Buffer {
 
     try {
       mkdirSync(cacheDir, { recursive: true });
-      writeFileSync(cacheFile, JSON.stringify({
-        version: packageJson.version,
-        hash,
-        size: compressed.length,
-        base64,
-      }, null, 2));
+      writeFileSync(cacheFile, JSON.stringify(archiveRecord, null, 2));
       return { cacheFile };
     } catch (error) {
       // Fall back to a temp cache if the primary location is not writable
@@ -120,42 +116,48 @@ export function getEmbeddedResourceArchive(): Buffer {
         cacheDir = join(cacheBase, cacheDirName);
         cacheFile = join(cacheDir, 'embedded-resources.json');
         mkdirSync(cacheDir, { recursive: true });
-        writeFileSync(cacheFile, JSON.stringify({
-          version: packageJson.version,
-          hash,
-          size: compressed.length,
-          base64,
-        }, null, 2));
+        writeFileSync(cacheFile, JSON.stringify(archiveRecord, null, 2));
         return { cacheFile };
       }
       throw error;
     }
   };
 
-  const { cacheFile } = writeCache();
+  const cacheResult = shouldWriteCache ? writeCache() : {};
 
-  if (shouldWriteStub) {
+  if (shouldWriteJson) {
     mkdirSync(resourcesDir, { recursive: true });
-    writeFileSync(targetPath, fileContents);
+    writeFileSync(targetJsonPath, JSON.stringify(archiveRecord, null, 2));
   }
 
   if (!options.quiet) {
     console.log(`[embed] Embedded ${files.length} files (${payloadBuffer.length} bytes, compressed to ${compressed.length} bytes)`);
     console.log(`[embed] Hash: ${hash}`);
-    console.log(`[embed] Cache file written to ${cacheFile}`);
-    if (!shouldWriteStub) {
-      console.log('[embed] Skipped updating src/shared/resources/embedded-resources.ts (cache-only)');
+    if (cacheResult.cacheFile) {
+      console.log(`[embed] Cache file written to ${cacheResult.cacheFile}`);
+    }
+    if (shouldWriteJson) {
+      console.log(`[embed] JSON archive written to ${targetJsonPath}`);
+    } else {
+      console.log('[embed] Skipped writing src/shared/resources/embedded-resources.json');
     }
   }
 
-  return { hash, size: compressed.length, fileCount: files.length, cacheFile };
+  return {
+    hash,
+    size: compressed.length,
+    fileCount: files.length,
+    cacheFile: cacheResult.cacheFile,
+    jsonFile: shouldWriteJson ? targetJsonPath : undefined,
+  };
 }
 
 if (import.meta.main) {
   const args = new Set(process.argv.slice(2));
   const cacheOnly = args.has('--cache-only');
-  const writeStub = args.has('--write-stub') ? true : !cacheOnly;
-  generateEmbeddedResources({ writeStub }).catch((error) => {
+  const writeJson = args.has('--no-json') ? false : args.has('--write-json') ? true : !cacheOnly;
+  const writeCache = args.has('--no-cache') ? false : true;
+  generateEmbeddedResources({ writeJson, writeCache }).catch((error) => {
     console.error('[embed] Failed to generate embedded resources:', error);
     process.exitCode = 1;
   });
diff --git a/src/shared/resources/embedded-loader.ts b/src/shared/resources/embedded-loader.ts
@@ -4,12 +4,7 @@ import { dirname, join } from 'node:path';
 import { homedir, tmpdir } from 'node:os';
 import { brotliDecompressSync } from 'node:zlib';
 
-import {
-  EMBEDDED_RESOURCES_HASH,
-  EMBEDDED_RESOURCES_SIZE,
-  EMBEDDED_RESOURCES_VERSION,
-  getEmbeddedResourceArchive
-} from './embedded-resources.js';
+import { getEmbeddedResourceArchive, getEmbeddedResourceMetadata } from './embedded-resources.js';
 
 type EmbeddedFile = {
   path: string;
@@ -27,15 +22,32 @@ type ArchiveMetadata = {
   version: string;
   hash: string;
   size: number;
+  generatedAt?: string;
 };
 
-const DEFAULT_METADATA: ArchiveMetadata = {
-  version: EMBEDDED_RESOURCES_VERSION,
-  hash: EMBEDDED_RESOURCES_HASH,
-  size: EMBEDDED_RESOURCES_SIZE,
+const FALLBACK_METADATA: ArchiveMetadata = {
+  version: 'unknown',
+  hash: 'unknown',
+  size: 0,
 };
 
-let activeMetadata: ArchiveMetadata = { ...DEFAULT_METADATA };
+let defaultMetadataCache: ArchiveMetadata | null = null;
+
+function getDefaultMetadata(): ArchiveMetadata {
+  if (defaultMetadataCache) {
+    return defaultMetadataCache;
+  }
+
+  try {
+    defaultMetadataCache = getEmbeddedResourceMetadata();
+  } catch {
+    defaultMetadataCache = FALLBACK_METADATA;
+  }
+
+  return defaultMetadataCache;
+}
+
+let activeMetadata: ArchiveMetadata = { ...getDefaultMetadata() };
 let cachedPayload: EmbeddedPayload | null = null;
 let cachedArchiveBuffer: Buffer | null = null;
 
@@ -59,16 +71,20 @@ function tryLoadArchiveFromCache(): Buffer | null {
       hash?: string;
       size?: number;
       base64?: string;
+      generatedAt?: string;
     };
     if (!data?.base64) {
       return null;
     }
+    const buffer = Buffer.from(data.base64, 'base64');
+    const defaults = getDefaultMetadata();
     activeMetadata = {
-      version: data.version ?? DEFAULT_METADATA.version,
-      hash: data.hash ?? DEFAULT_METADATA.hash,
-      size: data.size ?? DEFAULT_METADATA.size,
+      version: data.version ?? defaults.version,
+      hash: data.hash ?? defaults.hash,
+      size: data.size ?? buffer.length,
+      generatedAt: data.generatedAt ?? defaults.generatedAt,
     };
-    return Buffer.from(data.base64, 'base64');
+    return buffer;
   } catch {
     return null;
   }
@@ -90,6 +106,12 @@ function getArchiveBuffer(): Buffer {
     throw new Error('Embedded resource archive is empty');
   }
 
+  // Ensure metadata reflects the currently loaded archive
+  try {
+    activeMetadata = getEmbeddedResourceMetadata();
+  } catch {
+    activeMetadata = { ...getDefaultMetadata(), size: fallback.length };
+  }
   cachedArchiveBuffer = fallback;
   return cachedArchiveBuffer;
 }
@@ -102,6 +124,12 @@ function decodeEmbeddedPayload(): EmbeddedPayload {
   const archive = getArchiveBuffer();
   const decompressed = brotliDecompressSync(archive);
   cachedPayload = JSON.parse(decompressed.toString('utf8')) as EmbeddedPayload;
+  if (cachedPayload.version && cachedPayload.version !== activeMetadata.version) {
+    activeMetadata = { ...activeMetadata, version: cachedPayload.version };
+  }
+  if (cachedPayload.generatedAt && !activeMetadata.generatedAt) {
+    activeMetadata = { ...activeMetadata, generatedAt: cachedPayload.generatedAt };
+  }
   return cachedPayload;
 }
 
diff --git a/src/shared/resources/embedded-resources.ts b/src/shared/resources/embedded-resources.ts
diff --git a/tests/e2e/start-cli.spec.ts b/tests/e2e/start-cli.spec.ts
@@ -1,15 +1,13 @@
 import { spawn, type ChildProcess } from 'node:child_process';
-import { existsSync } from 'node:fs';
 import { mkdtemp, readFile, rm } from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import os from 'node:os';
-import { afterEach, beforeAll, describe, expect, it } from 'bun:test';
+import { afterEach, describe, expect, it } from 'bun:test';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 const projectRoot = path.resolve(__dirname, '..', '..');
-const distEntry = path.join(projectRoot, 'dist', 'index.js');
 const fixturesRoot = path.join(projectRoot, 'tests', 'fixtures');
 const engineFixturesDir = path.join(fixturesRoot, 'codex');
 const activeProcesses = new Set<ChildProcess>();

Original file line number	Diff line number	Diff line change
`@@ -25,3 +25,4 @@ package-lock.json`
`25`	`25`
`26`	`26`	`# MkDocs build output`
`27`	`27`	`site/`
	`28`	`+src/shared/resources/embedded-resources.json`