From d3a14ec6b1517cccbba5e3a6603f8e635f514859 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 28 Mar 2025 18:42:45 +0100 Subject: [PATCH 01/23] Implement gcAndFinalize utility for swingset liveslots --- packages/kernel/src/VatSupervisor.ts | 4 +- packages/kernel/src/services/gc-engine.ts | 20 ++++ packages/kernel/src/utils/gc-finalize.test.ts | 104 ++++++++++++++++++ packages/kernel/src/utils/gc-finalize.ts | 72 ++++++++++++ 4 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 packages/kernel/src/services/gc-engine.ts create mode 100644 packages/kernel/src/utils/gc-finalize.test.ts create mode 100644 packages/kernel/src/utils/gc-finalize.ts diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index e226b0e39..8baf89e7f 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -17,6 +17,7 @@ import { makeSupervisorSyscall } from './services/syscall.ts'; import type { DispatchFn, MakeLiveSlotsFn, GCTools } from './services/types.ts'; import type { VatConfig, VatId, VRef } from './types.ts'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts'; +import { gcAndFinalize } from './utils/gc-finalize.ts'; import { waitUntilQuiescent } from './utils/wait-quiescent.ts'; import type { VatKVStore } from './VatKVStore.ts'; import { makeVatKVStore } from './VatKVStore.ts'; @@ -228,8 +229,7 @@ export class VatSupervisor { WeakRef, FinalizationRegistry, waitUntilQuiescent, - // eslint-disable-next-line no-empty-function - gcAndFinalize: async () => {}, + gcAndFinalize, meterControl: makeDummyMeterControl(), }); diff --git a/packages/kernel/src/services/gc-engine.ts b/packages/kernel/src/services/gc-engine.ts new file mode 100644 index 000000000..b833a42a5 --- /dev/null +++ b/packages/kernel/src/services/gc-engine.ts @@ -0,0 +1,20 @@ +import v8 from 'v8'; +import vm from 'vm'; + +/* global globalThis */ +let bestGC = globalThis.gc; +if (typeof bestGC !== 'function') { + // Node.js v8 wizardry. + v8.setFlagsFromString('--expose_gc'); + bestGC = vm.runInNewContext('gc'); + assert(bestGC); + // We leave --expose_gc turned on, otherwise AVA's shared workers + // may race and disable it before we manage to extract the + // binding. This won't cause 'gc' to be visible to new Compartments + // because SES strips out everything it doesn't recognize. + + // // Hide the gc global from new contexts/workers. + // v8.setFlagsFromString('--no-expose_gc'); +} + +export const engineGC = bestGC; diff --git a/packages/kernel/src/utils/gc-finalize.test.ts b/packages/kernel/src/utils/gc-finalize.test.ts new file mode 100644 index 000000000..26b659aa7 --- /dev/null +++ b/packages/kernel/src/utils/gc-finalize.test.ts @@ -0,0 +1,104 @@ +import { delay } from '@ocap/utils'; +import { describe, it, expect, vi } from 'vitest'; + +import { gcAndFinalize, makeGCAndFinalize } from './gc-finalize.ts'; + +describe('Garbage Collection', () => { + it('should clean up unreachable objects', async () => { + // Set up a WeakRef to track an object + let obj = { test: 'value' }; + const weakRef = new WeakRef(obj); + expect(weakRef.deref()).toBe(obj); + // @ts-expect-error - Remove the reference to the object + obj = null; + expect(weakRef.deref()).toBeDefined(); + await gcAndFinalize(); + expect(weakRef.deref()).toBeUndefined(); + }); + + it('should trigger FinalizationRegistry callbacks', async () => { + // Set up a finalization registry with a callback + const finalizationCallback = vi.fn(); + const registry = new FinalizationRegistry(finalizationCallback); + // Register an object for finalization + let obj = { test: 'finalize me' }; + registry.register(obj, 'test token'); + // Remove reference to the object + // @ts-expect-error - Null assignment + obj = null; + // Trigger garbage collection + await gcAndFinalize(); + // Wait a bit more to ensure finalization callbacks run + await delay(50); + // The callback should have been called at least once + expect(finalizationCallback).toHaveBeenCalled(); + }); + + it('should handle errors gracefully', async () => { + // Create a custom implementation with a failing GC function + const consoleWarnSpy = vi + .spyOn(console, 'warn') + .mockImplementation(vi.fn()); + const mockGcFunction = vi.fn().mockImplementation(() => { + throw new Error('GC failed'); + }); + // Mock globalThis.gc + const originalGc = globalThis.gc; + globalThis.gc = mockGcFunction; + // Create a new gcAndFinalize with our mocked environment + const customGcAndFinalize = makeGCAndFinalize(); + // Should not throw despite GC failing + expect(await customGcAndFinalize()).toBeUndefined(); + // Should log warning + expect(consoleWarnSpy).toHaveBeenCalledWith( + 'GC operation failed:', + expect.any(Error), + ); + // Restore original gc + // Using Object.defineProperty to avoid race condition + Object.defineProperty(globalThis, 'gc', { + value: originalGc, + writable: true, + configurable: true, + }); + consoleWarnSpy.mockRestore(); + }); + + it('should run multiple GC passes', async () => { + // Mock the GC function to verify multiple calls + const mockGcFunction = vi.fn(); + // Mock globalThis.gc + const originalGc = globalThis.gc; + globalThis.gc = mockGcFunction; + // Create a new gcAndFinalize with our mocked environment + const customGcAndFinalize = makeGCAndFinalize(); + await customGcAndFinalize(); + // Should call GC function twice + expect(mockGcFunction).toHaveBeenCalledTimes(2); + // Restore original gc + Object.defineProperty(globalThis, 'gc', { + value: originalGc, + writable: true, + configurable: true, + }); + }); + + it('should work with circular references', async () => { + // Create objects with circular references + type CircularObj = { name: string; ref: CircularObj | null }; + const objA: CircularObj = { name: 'A', ref: null }; + let objB: CircularObj = { name: 'B', ref: null }; + objA.ref = objB; + objB.ref = objA; + // Create a weak reference to track objB + const weakRef = new WeakRef(objB); + expect(weakRef.deref()).toBe(objB); + // Break circular reference and remove our reference + objA.ref = null; + // @ts-expect-error - Null assignment + objB = null; + expect(weakRef.deref()).toBeDefined(); + await gcAndFinalize(); + expect(weakRef.deref()).toBeUndefined(); + }); +}); diff --git a/packages/kernel/src/utils/gc-finalize.ts b/packages/kernel/src/utils/gc-finalize.ts new file mode 100644 index 000000000..8d79afd24 --- /dev/null +++ b/packages/kernel/src/utils/gc-finalize.ts @@ -0,0 +1,72 @@ +import { delay } from '@ocap/utils'; + +/** + * Try to get a GC function for the current environment + * + * @returns A function that triggers GC and finalization when possible + */ +async function getGCFunction(): Promise { + if (typeof globalThis.gc === 'function') { + return globalThis.gc; + } + + // Check if we're in Node.js + if ( + typeof globalThis === 'object' && + Object.prototype.toString.call(globalThis.process) === '[object process]' + ) { + try { + // Dynamic import of Node.js specific module so it's not included in browser builds + const { engineGC } = await import('../services/gc-engine.ts'); + return engineGC; + } catch (error) { + console.debug('Failed to load Node.js GC implementation:', error); + } + } + + return undefined; +} + +/** + * Utility to create a function that performs garbage collection and finalization + * in a cross-environment compatible way. + * + * @returns A function that triggers GC and finalization when possible + */ +function makeGCAndFinalize(): () => Promise { + // Cache the GC function promise + const gcFunctionPromise = getGCFunction(); + + /** + * Function to trigger garbage collection and finalization + */ + return async function gcAndFinalize(): Promise { + try { + const gcFunction = await gcFunctionPromise; + + if (gcFunction) { + // First GC pass + gcFunction(); + // Allow finalization callbacks to run + await delay(0); + // Second GC pass to clean up objects that might have become + // unreachable during finalization + gcFunction(); + // Another tick to ensure finalization completes + await delay(0); + } else { + // No GC function available, log warning and cycle the event loop + console.warn('Deterministic GC not available in this environment'); + await delay(0); + } + } catch (error) { + console.warn('GC operation failed:', error); + } + }; +} + +// Create and export a singleton instance to be used throughout the codebase +export const gcAndFinalize = makeGCAndFinalize(); + +// Still export the factory function for testing or cases where a fresh instance is needed +export { makeGCAndFinalize }; From 29725aca1674d5526fd9a73775012e45f1ad3d7a Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 28 Mar 2025 18:43:06 +0100 Subject: [PATCH 02/23] restructure kernel test vats --- packages/kernel-test/package.json | 4 ++-- packages/kernel-test/src/{ => vats}/message-to-promise-vat.js | 0 .../kernel-test/src/{ => vats}/pass-result-promise-vat.js | 0 packages/kernel-test/src/{ => vats}/pass-result-vat.js | 0 packages/kernel-test/src/{ => vats}/powers-vat.js | 0 packages/kernel-test/src/{ => vats}/promise-arg-vat.js | 0 packages/kernel-test/src/{ => vats}/promise-chain-vat.js | 0 packages/kernel-test/src/{ => vats}/promise-crosswise-vat.js | 0 packages/kernel-test/src/{ => vats}/promise-cycle-vat.js | 0 packages/kernel-test/src/{ => vats}/promise-indirect-vat.js | 0 packages/kernel-test/src/{ => vats}/resolve-pipelined-vat.js | 0 packages/kernel-test/src/{ => vats}/resume-vat.js | 0 packages/kernel-test/src/{ => vats}/vatstore-vat.js | 0 packages/kernel-test/tsconfig.json | 3 ++- packages/nodejs/tsconfig.json | 1 - 15 files changed, 4 insertions(+), 4 deletions(-) rename packages/kernel-test/src/{ => vats}/message-to-promise-vat.js (100%) rename packages/kernel-test/src/{ => vats}/pass-result-promise-vat.js (100%) rename packages/kernel-test/src/{ => vats}/pass-result-vat.js (100%) rename packages/kernel-test/src/{ => vats}/powers-vat.js (100%) rename packages/kernel-test/src/{ => vats}/promise-arg-vat.js (100%) rename packages/kernel-test/src/{ => vats}/promise-chain-vat.js (100%) rename packages/kernel-test/src/{ => vats}/promise-crosswise-vat.js (100%) rename packages/kernel-test/src/{ => vats}/promise-cycle-vat.js (100%) rename packages/kernel-test/src/{ => vats}/promise-indirect-vat.js (100%) rename packages/kernel-test/src/{ => vats}/resolve-pipelined-vat.js (100%) rename packages/kernel-test/src/{ => vats}/resume-vat.js (100%) rename packages/kernel-test/src/{ => vats}/vatstore-vat.js (100%) diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index 34f62d801..75e961043 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -30,10 +30,10 @@ ], "scripts": { "build": "ts-bridge --project tsconfig.build.json --clean && yarn build:vats", - "build:vats": "ocap bundle src", + "build:vats": "ocap bundle src/vats", "build:docs": "typedoc", "changelog:validate": "../../scripts/validate-changelog.sh @ocap/kernel-test", - "clean": "rimraf --glob './*.tsbuildinfo' ./.eslintcache ./coverage ./dist './src/*.bundle'", + "clean": "rimraf --glob './*.tsbuildinfo' ./.eslintcache ./coverage ./dist './src/**/*.bundle'", "lint": "yarn lint:eslint && yarn lint:misc --check && yarn constraints && yarn lint:dependencies", "lint:dependencies": "depcheck", "lint:eslint": "eslint . --cache", diff --git a/packages/kernel-test/src/message-to-promise-vat.js b/packages/kernel-test/src/vats/message-to-promise-vat.js similarity index 100% rename from packages/kernel-test/src/message-to-promise-vat.js rename to packages/kernel-test/src/vats/message-to-promise-vat.js diff --git a/packages/kernel-test/src/pass-result-promise-vat.js b/packages/kernel-test/src/vats/pass-result-promise-vat.js similarity index 100% rename from packages/kernel-test/src/pass-result-promise-vat.js rename to packages/kernel-test/src/vats/pass-result-promise-vat.js diff --git a/packages/kernel-test/src/pass-result-vat.js b/packages/kernel-test/src/vats/pass-result-vat.js similarity index 100% rename from packages/kernel-test/src/pass-result-vat.js rename to packages/kernel-test/src/vats/pass-result-vat.js diff --git a/packages/kernel-test/src/powers-vat.js b/packages/kernel-test/src/vats/powers-vat.js similarity index 100% rename from packages/kernel-test/src/powers-vat.js rename to packages/kernel-test/src/vats/powers-vat.js diff --git a/packages/kernel-test/src/promise-arg-vat.js b/packages/kernel-test/src/vats/promise-arg-vat.js similarity index 100% rename from packages/kernel-test/src/promise-arg-vat.js rename to packages/kernel-test/src/vats/promise-arg-vat.js diff --git a/packages/kernel-test/src/promise-chain-vat.js b/packages/kernel-test/src/vats/promise-chain-vat.js similarity index 100% rename from packages/kernel-test/src/promise-chain-vat.js rename to packages/kernel-test/src/vats/promise-chain-vat.js diff --git a/packages/kernel-test/src/promise-crosswise-vat.js b/packages/kernel-test/src/vats/promise-crosswise-vat.js similarity index 100% rename from packages/kernel-test/src/promise-crosswise-vat.js rename to packages/kernel-test/src/vats/promise-crosswise-vat.js diff --git a/packages/kernel-test/src/promise-cycle-vat.js b/packages/kernel-test/src/vats/promise-cycle-vat.js similarity index 100% rename from packages/kernel-test/src/promise-cycle-vat.js rename to packages/kernel-test/src/vats/promise-cycle-vat.js diff --git a/packages/kernel-test/src/promise-indirect-vat.js b/packages/kernel-test/src/vats/promise-indirect-vat.js similarity index 100% rename from packages/kernel-test/src/promise-indirect-vat.js rename to packages/kernel-test/src/vats/promise-indirect-vat.js diff --git a/packages/kernel-test/src/resolve-pipelined-vat.js b/packages/kernel-test/src/vats/resolve-pipelined-vat.js similarity index 100% rename from packages/kernel-test/src/resolve-pipelined-vat.js rename to packages/kernel-test/src/vats/resolve-pipelined-vat.js diff --git a/packages/kernel-test/src/resume-vat.js b/packages/kernel-test/src/vats/resume-vat.js similarity index 100% rename from packages/kernel-test/src/resume-vat.js rename to packages/kernel-test/src/vats/resume-vat.js diff --git a/packages/kernel-test/src/vatstore-vat.js b/packages/kernel-test/src/vats/vatstore-vat.js similarity index 100% rename from packages/kernel-test/src/vatstore-vat.js rename to packages/kernel-test/src/vats/vatstore-vat.js diff --git a/packages/kernel-test/tsconfig.json b/packages/kernel-test/tsconfig.json index 786cfa485..d0c24bca3 100644 --- a/packages/kernel-test/tsconfig.json +++ b/packages/kernel-test/tsconfig.json @@ -11,7 +11,8 @@ { "path": "../test-utils" }, { "path": "../utils" }, { "path": "../nodejs" }, - { "path": "../kernel" } + { "path": "../kernel" }, + { "path": "../store" } ], "include": [ "../../vitest.config.ts", diff --git a/packages/nodejs/tsconfig.json b/packages/nodejs/tsconfig.json index 01dd4a8f0..e976880f0 100644 --- a/packages/nodejs/tsconfig.json +++ b/packages/nodejs/tsconfig.json @@ -5,7 +5,6 @@ "baseUrl": "./", "isolatedModules": true, "lib": ["ES2022"], - "noEmit": true, "types": ["node", "ses", "vitest"] }, "references": [ From 59003b15c6cf20eb02f44bd61373ed64e2272817 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 28 Mar 2025 20:16:36 +0100 Subject: [PATCH 03/23] reuse test utils and add basic wip gc test --- .github/dependabot.yml | 2 +- packages/kernel-test/package.json | 3 + .../kernel-test/src/gc-integration.test.ts | 49 +++++ packages/kernel-test/src/liveslots.test.ts | 39 +--- packages/kernel-test/src/resume.test.ts | 179 ++---------------- packages/kernel-test/src/supervisor.test.ts | 8 +- packages/kernel-test/src/utils.ts | 137 ++++++++++++++ packages/kernel-test/src/vats/gc-test-vat.js | 39 ++++ packages/kernel-test/src/vatstore.test.ts | 90 +-------- packages/kernel/src/VatSupervisor.ts | 4 +- packages/kernel/src/index.ts | 1 + packages/utils/package.json | 4 +- packages/utils/src/index.ts | 1 + .../src}/wait-quiescent.test.ts | 0 .../src/utils => utils/src}/wait-quiescent.ts | 10 +- yarn.lock | 104 +++++++++- 16 files changed, 376 insertions(+), 294 deletions(-) create mode 100644 packages/kernel-test/src/gc-integration.test.ts create mode 100644 packages/kernel-test/src/utils.ts create mode 100644 packages/kernel-test/src/vats/gc-test-vat.js rename packages/{kernel/src/utils => utils/src}/wait-quiescent.test.ts (100%) rename packages/{kernel/src/utils => utils/src}/wait-quiescent.ts (75%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e1f12aecf..ae51ba99a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,7 +10,7 @@ updates: day: 'monday' time: '06:00' # UTC target-branch: 'main' - versioning-strategy: 'increase-if-necessary' + versioning-strategy: 'widen' open-pull-requests-limit: 10 groups: vite: diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index 75e961043..150b77c58 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -53,8 +53,11 @@ "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", "@ocap/cli": "workspace:^", + "@ocap/kernel": "workspace:^", + "@ocap/nodejs": "workspace:^", "@ocap/store": "workspace:^", "@ocap/streams": "workspace:^", + "@ocap/utils": "workspace:^", "@ts-bridge/cli": "^0.6.3", "@ts-bridge/shims": "^0.1.1", "@typescript-eslint/eslint-plugin": "^8.29.0", diff --git a/packages/kernel-test/src/gc-integration.test.ts b/packages/kernel-test/src/gc-integration.test.ts new file mode 100644 index 000000000..0ee74edad --- /dev/null +++ b/packages/kernel-test/src/gc-integration.test.ts @@ -0,0 +1,49 @@ +import '@ocap/shims/endoify'; +import type { ClusterConfig } from '@ocap/kernel'; +import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; +import { describe, expect, it } from 'vitest'; + +import { + extractVatLogs, + getBundleSpec, + makeKernel, + runTestVats, +} from './utils.ts'; + +const origStdoutWrite = process.stdout.write.bind(process.stdout); +let buffered: string = ''; +// @ts-expect-error Some type def used by lint is just wrong (compiler likes it ok, but lint whines) +process.stdout.write = (buffer: string, encoding, callback): void => { + buffered += buffer; + origStdoutWrite(buffer, encoding, callback); +}; + +// Define a very simple test cluster +const simpleGCTestSubcluster: ClusterConfig = { + bootstrap: 'gcVat', + forceReset: true, + bundles: null, + vats: { + gcVat: { + bundleSpec: getBundleSpec('gc-test-vat'), + parameters: { + name: 'GCVat', + }, + }, + }, +}; + +describe('Simple GC Tests', () => { + it('should create a WeakRef successfully', async () => { + const kernelDatabase = await makeSQLKernelDatabase({ + dbFilename: ':memory:', + }); + const kernel = await makeKernel(kernelDatabase, true); + const bootstrapResult = await runTestVats(kernel, simpleGCTestSubcluster); + expect(bootstrapResult).toBe('gc-test-complete'); + const vatLogs = extractVatLogs(buffered); + expect(vatLogs).toContain( + 'GCVat: WeakRef created and object is accessible', + ); + }, 10000); +}); diff --git a/packages/kernel-test/src/liveslots.test.ts b/packages/kernel-test/src/liveslots.test.ts index f95945fb1..a02ec357f 100644 --- a/packages/kernel-test/src/liveslots.test.ts +++ b/packages/kernel-test/src/liveslots.test.ts @@ -1,15 +1,15 @@ import '@ocap/shims/endoify'; -import { makePromiseKit } from '@endo/promise-kit'; -import { Kernel } from '@ocap/kernel'; +import { Kernel, kunser } from '@ocap/kernel'; import type { ClusterConfig } from '@ocap/kernel'; +import { makeKernel } from '@ocap/nodejs'; +import { waitUntilQuiescent } from '@ocap/utils'; import { MessagePort as NodeMessagePort, MessageChannel as NodeMessageChannel, } from 'node:worker_threads'; import { beforeEach, describe, expect, it } from 'vitest'; -import { kunser } from '../../kernel/src/services/kernel-marshal.ts'; -import { makeKernel } from '../../nodejs/src/kernel/make-kernel.ts'; +import { extractVatLogs, getBundleSpec } from './utils.ts'; const origStdoutWrite = process.stdout.write.bind(process.stdout); let buffered: string = ''; @@ -73,37 +73,12 @@ describe('liveslots promise handling', () => { testName: string, ): Promise<[unknown, string[]]> { buffered = ''; - const bundleSpec = new URL( - `${bundleName}.bundle`, - import.meta.url, - ).toString(); + const bundleSpec = getBundleSpec(bundleName); const bootstrapResultRaw = await kernel.launchSubcluster( makeTestSubcluster(testName, bundleSpec), ); - - const { promise, resolve } = makePromiseKit(); - setTimeout(() => resolve(null), 1000); - await promise; - - const vatLogs = buffered - .split('\n') - .filter((line: string) => line.startsWith('::> ')) - .map((line: string) => line.slice(4)); - - // de-interleave various vats' output to squeeze out interprocess I/O non-determinism in CI - vatLogs.sort((a: string, b: string): number => { - const colonA = a.indexOf(':'); - if (colonA < 0) { - return 0; - } - const prefixA = a.substring(0, colonA); - const colonB = b.indexOf(':'); - if (colonB < 0) { - return 0; - } - const prefixB = b.substring(0, colonB); - return prefixA.localeCompare(prefixB); - }); + await waitUntilQuiescent(1000); + const vatLogs = extractVatLogs(buffered); if (bootstrapResultRaw === undefined) { throw Error(`this can't happen but eslint is stupid`); } diff --git a/packages/kernel-test/src/resume.test.ts b/packages/kernel-test/src/resume.test.ts index 06d2766bb..fb3a672ba 100644 --- a/packages/kernel-test/src/resume.test.ts +++ b/packages/kernel-test/src/resume.test.ts @@ -1,22 +1,16 @@ import '@ocap/shims/endoify'; -import { makePromiseKit } from '@endo/promise-kit'; -import type { - KernelCommand, - KernelCommandReply, - ClusterConfig, -} from '@ocap/kernel'; -import { Kernel } from '@ocap/kernel'; -import type { KernelDatabase } from '@ocap/store'; import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; -import { NodeWorkerDuplexStream } from '@ocap/streams'; -import { - MessagePort as NodeMessagePort, - MessageChannel as NodeMessageChannel, -} from 'node:worker_threads'; +import { waitUntilQuiescent } from '@ocap/utils'; import { describe, expect, it } from 'vitest'; -import { kunser } from '../../kernel/src/services/kernel-marshal.ts'; -import { NodejsVatWorkerService } from '../../nodejs/src/kernel/VatWorkerService.ts'; +import { + extractVatLogs, + getBundleSpec, + makeKernel, + runResume, + runTestVats, + sortLogs, +} from './utils.ts'; const origStdoutWrite = process.stdout.write.bind(process.stdout); let buffered: string = ''; @@ -26,35 +20,24 @@ process.stdout.write = (buffer: string, encoding, callback): void => { origStdoutWrite(buffer, encoding, callback); }; -/** - * Construct a bundle path URL from a bundle name. - * - * @param bundleName - The name of the bundle. - * - * @returns a path string for the named bundle. - */ -function bundleSpec(bundleName: string): string { - return new URL(`${bundleName}.bundle`, import.meta.url).toString(); -} - const testSubcluster = { bootstrap: 'alice', forceReset: true, vats: { alice: { - bundleSpec: bundleSpec('resume-vat'), + bundleSpec: getBundleSpec('resume-vat'), parameters: { name: 'Alice', }, }, bob: { - bundleSpec: bundleSpec('resume-vat'), + bundleSpec: getBundleSpec('resume-vat'), parameters: { name: 'Bob', }, }, carol: { - bundleSpec: bundleSpec('resume-vat'), + bundleSpec: getBundleSpec('resume-vat'), parameters: { name: 'Carol', }, @@ -62,122 +45,6 @@ const testSubcluster = { }, }; -/** - * Handle all the boilerplate to set up a kernel instance. - * - * @param kernelDatabase - The database that will hold the persistent state. - * @param resetStorage - If true, reset the database as part of setting up. - * - * @returns the new kernel instance. - */ -async function makeKernel( - kernelDatabase: KernelDatabase, - resetStorage: boolean, -): Promise { - const kernelPort: NodeMessagePort = new NodeMessageChannel().port1; - const nodeStream = new NodeWorkerDuplexStream< - KernelCommand, - KernelCommandReply - >(kernelPort); - const vatWorkerClient = new NodejsVatWorkerService({}); - const kernel = await Kernel.make( - nodeStream, - vatWorkerClient, - kernelDatabase, - { - resetStorage, - }, - ); - return kernel; -} - -/** - * Take a pass through the JavaScript run loop. - * - * @param delay - Optional delay (in ms) to wait for things to catch up. - */ -async function waitForQuiescence(delay: number = 0): Promise { - const { promise, resolve } = makePromiseKit(); - setTimeout(() => resolve(null), delay); - await promise; -} - -/** - * De-interleave various vats' output to squeeze out interprocess I/O - * non-determinism in CI. - * - * @param logs - An array of log lines. - * - * @returns `logs` sorted by vat. - */ -function sortLogs(logs: string[]): string[] { - logs.sort((a: string, b: string): number => { - const colonA = a.indexOf(':'); - if (colonA < 0) { - return 0; - } - const prefixA = a.substring(0, colonA); - const colonB = b.indexOf(':'); - if (colonB < 0) { - return 0; - } - const prefixB = b.substring(0, colonB); - return prefixA.localeCompare(prefixB); - }); - return logs; -} - -/** - * Convert a raw output buffer into a list of lines suitable for examination. - * - * @param buffer - The raw buffer to convert. - * - * @returns the relevant contents of `buffer`, massaged for use. - */ -function extractVatLogs(buffer: string): string[] { - const result = buffer - .split('\n') - .filter((line: string) => line.startsWith('::> ')) - .map((line: string) => line.slice(4)); - return sortLogs(result); -} - -/** - * Bootstrap the set of test vats. - * - * @param kernel - The kernel to run in. - * @param config - Subcluster configuration telling what vats to run. - * - * @returns the bootstrap result. - */ -async function runBootstrap( - kernel: Kernel, - config: ClusterConfig, -): Promise { - const bootstrapResultRaw = await kernel.launchSubcluster(config); - if (bootstrapResultRaw === undefined) { - throw Error(`this can't happen but eslint is stupid`); - } - return kunser(bootstrapResultRaw); -} - -/** - * Send the `resume message to the root of one of the test vats. - * - * @param kernel - Our kernel. - * @param rootRef - KRef of the object to which the message is sent. - * - * @returns the result returned from `resume`. - */ -async function runResume(kernel: Kernel, rootRef: string): Promise { - const resumeResultRaw = await kernel.queueMessageFromKernel( - rootRef, - 'resume', - [], - ); - return kunser(resumeResultRaw); -} - const bootstrapReference = [ `Alice: saving name`, `Alice: start count: 1`, @@ -232,11 +99,9 @@ const carolResumeReference = [ ]; const reference = sortLogs([ ...bootstrapReference, - ...aliceRestartReference, ...bobRestartReference, ...carolRestartReference, - ...aliceResumeReference, ...bobResumeReference, ...carolResumeReference, @@ -248,24 +113,20 @@ describe('restarting vats', async () => { dbFilename: ':memory:', }); const kernel = await makeKernel(kernelDatabase, true); - buffered = ''; - const bootstrapResult = await runBootstrap(kernel, testSubcluster); + const bootstrapResult = await runTestVats(kernel, testSubcluster); expect(bootstrapResult).toBe('bootstrap Alice'); - - await waitForQuiescence(); + await waitUntilQuiescent(); await kernel.restartVat('v1'); await kernel.restartVat('v2'); await kernel.restartVat('v3'); - const resumeResultA = await runResume(kernel, 'ko1'); expect(resumeResultA).toBe('resume Alice'); const resumeResultB = await runResume(kernel, 'ko2'); expect(resumeResultB).toBe('resume Bob'); const resumeResultC = await runResume(kernel, 'ko3'); expect(resumeResultC).toBe('resume Carol'); - - await waitForQuiescence(1000); + await waitUntilQuiescent(1000); const vatLogs = extractVatLogs(buffered); expect(vatLogs).toStrictEqual(reference); }, 30000); @@ -275,22 +136,18 @@ describe('restarting vats', async () => { dbFilename: ':memory:', }); const kernel1 = await makeKernel(kernelDatabase, true); - buffered = ''; - const bootstrapResult = await runBootstrap(kernel1, testSubcluster); + const bootstrapResult = await runTestVats(kernel1, testSubcluster); expect(bootstrapResult).toBe('bootstrap Alice'); - await waitForQuiescence(); - + await waitUntilQuiescent(); const kernel2 = await makeKernel(kernelDatabase, false); - const resumeResultA = await runResume(kernel2, 'ko1'); expect(resumeResultA).toBe('resume Alice'); const resumeResultB = await runResume(kernel2, 'ko2'); expect(resumeResultB).toBe('resume Bob'); const resumeResultC = await runResume(kernel2, 'ko3'); expect(resumeResultC).toBe('resume Carol'); - - await waitForQuiescence(1000); + await waitUntilQuiescent(1000); const vatLogs = extractVatLogs(buffered); expect(vatLogs).toStrictEqual(reference); }, 30000); diff --git a/packages/kernel-test/src/supervisor.test.ts b/packages/kernel-test/src/supervisor.test.ts index 89bd18506..864dd9a9a 100644 --- a/packages/kernel-test/src/supervisor.test.ts +++ b/packages/kernel-test/src/supervisor.test.ts @@ -1,11 +1,11 @@ import '@ocap/shims/endoify'; -import { VatSupervisor, VatCommandMethod } from '@ocap/kernel'; import type { VatCommand, VatConfig, VatCommandReply } from '@ocap/kernel'; +import { VatSupervisor, VatCommandMethod, kser } from '@ocap/kernel'; import { readFile } from 'fs/promises'; import { join } from 'path'; import { describe, it, expect } from 'vitest'; -import { kser } from '../../kernel/src/services/kernel-marshal.ts'; +import { getBundleSpec } from './utils.ts'; import { TestDuplexStream } from '../../streams/test/stream-mocks.ts'; const makeVatSupervisor = async ({ @@ -32,7 +32,7 @@ const makeVatSupervisor = async ({ throw new Error(`Unexpected URL: ${url}`); } const bundleName = url.split('/').pop() ?? url; - const bundlePath = join(__dirname, bundleName); + const bundlePath = join(__dirname, 'vats', bundleName); const bundleContent = await readFile(bundlePath, 'utf-8'); return { ok: true, @@ -55,7 +55,7 @@ describe('VatSupervisor', () => { const { supervisor } = await makeVatSupervisor({ vatPowers }); const vatConfig: VatConfig = { - bundleSpec: new URL('powers-vat.bundle', import.meta.url).toString(), + bundleSpec: getBundleSpec('powers-vat'), parameters: { bar: 'buzz' }, }; diff --git a/packages/kernel-test/src/utils.ts b/packages/kernel-test/src/utils.ts new file mode 100644 index 000000000..24744a774 --- /dev/null +++ b/packages/kernel-test/src/utils.ts @@ -0,0 +1,137 @@ +// eslint-disable-next-line spaced-comment +/// + +import { Kernel, kunser } from '@ocap/kernel'; +import type { + ClusterConfig, + KernelCommand, + KernelCommandReply, +} from '@ocap/kernel'; +import { NodejsVatWorkerService } from '@ocap/nodejs'; +import type { KernelDatabase } from '@ocap/store'; +import { NodeWorkerDuplexStream } from '@ocap/streams'; +import { waitUntilQuiescent } from '@ocap/utils'; +import { + MessagePort as NodeMessagePort, + MessageChannel as NodeMessageChannel, +} from 'node:worker_threads'; + +/** + * Construct a bundle path URL from a bundle name. + * + * @param bundleName - The name of the bundle. + * + * @returns a path string for the named bundle. + */ +export function getBundleSpec(bundleName: string): string { + return new URL(`./vats/${bundleName}.bundle`, import.meta.url).toString(); +} + +/** + * Run the set of test vats. + * + * @param kernel - The kernel to run in. + * @param config - Subcluster configuration telling what vats to run. + * + * @returns the bootstrap result. + */ +export async function runTestVats( + kernel: Kernel, + config: ClusterConfig, +): Promise { + const bootstrapResultRaw = await kernel.launchSubcluster(config); + await waitUntilQuiescent(); + if (bootstrapResultRaw === undefined) { + throw Error(`this can't happen but eslint is stupid`); + } + return kunser(bootstrapResultRaw); +} + +/** + * Send the `resume message to the root of one of the test vats. + * + * @param kernel - Our kernel. + * @param rootRef - KRef of the object to which the message is sent. + * + * @returns the result returned from `resume`. + */ +export async function runResume( + kernel: Kernel, + rootRef: string, +): Promise { + const resumeResultRaw = await kernel.queueMessageFromKernel( + rootRef, + 'resume', + [], + ); + return kunser(resumeResultRaw); +} + +/** + * Handle all the boilerplate to set up a kernel instance. + * + * @param kernelDatabase - The database that will hold the persistent state. + * @param resetStorage - If true, reset the database as part of setting up. + * + * @returns the new kernel instance. + */ +export async function makeKernel( + kernelDatabase: KernelDatabase, + resetStorage: boolean, +): Promise { + const kernelPort: NodeMessagePort = new NodeMessageChannel().port1; + const nodeStream = new NodeWorkerDuplexStream< + KernelCommand, + KernelCommandReply + >(kernelPort); + const vatWorkerClient = new NodejsVatWorkerService({}); + const kernel = await Kernel.make( + nodeStream, + vatWorkerClient, + kernelDatabase, + { + resetStorage, + }, + ); + return kernel; +} + +/** + * De-interleave various vats' output to squeeze out interprocess I/O + * non-determinism in CI. + * + * @param logs - An array of log lines. + * + * @returns `logs` sorted by vat. + */ +export function sortLogs(logs: string[]): string[] { + logs.sort((a: string, b: string): number => { + const colonA = a.indexOf(':'); + if (colonA < 0) { + return 0; + } + const prefixA = a.substring(0, colonA); + const colonB = b.indexOf(':'); + if (colonB < 0) { + return 0; + } + const prefixB = b.substring(0, colonB); + return prefixA.localeCompare(prefixB); + }); + return logs; +} + +/** + * Convert a raw output buffer into a list of lines suitable for examination. + * + * @param buffer - The raw buffer to convert. + * + * @returns the relevant contents of `buffer`, massaged for use. + */ +export function extractVatLogs(buffer: string): string[] { + const result = buffer + .split('\n') + .filter((line: string) => line.startsWith('::> ')) + .map((line: string) => line.slice(4)); + return sortLogs(result); +} diff --git a/packages/kernel-test/src/vats/gc-test-vat.js b/packages/kernel-test/src/vats/gc-test-vat.js new file mode 100644 index 000000000..94ff26c71 --- /dev/null +++ b/packages/kernel-test/src/vats/gc-test-vat.js @@ -0,0 +1,39 @@ +import { Far } from '@endo/marshal'; + +/** + * Build function for a very simple vat that just tests WeakRef creation. + * + * @param {object} _vatPowers - Special powers granted to this vat. + * @param {object} parameters - Initialization parameters from the vat's config object. + * @returns {object} The root object for the new vat. + */ +export function buildRootObject(_vatPowers, parameters) { + const name = parameters?.name ?? 'anonymous'; + + /** + * Print a message to the test log. + * + * @param {string} message - The message to print. + */ + function tlog(message) { + console.log(`::> ${name}: ${message}`); + } + + return Far('root', { + async bootstrap() { + // Simply create an object and a WeakRef to it + const obj = { value: 'test object' }; + const weakRef = new WeakRef(obj); + + // Verify the WeakRef works + const retrieved = weakRef.deref(); + if (retrieved && retrieved.value === 'test object') { + tlog('WeakRef created and object is accessible'); + } else { + tlog('ERROR: WeakRef failed to retrieve object'); + } + + return 'gc-test-complete'; + }, + }); +} diff --git a/packages/kernel-test/src/vatstore.test.ts b/packages/kernel-test/src/vatstore.test.ts index 74c1e628a..2ea56b7f0 100644 --- a/packages/kernel-test/src/vatstore.test.ts +++ b/packages/kernel-test/src/vatstore.test.ts @@ -1,54 +1,30 @@ import '@ocap/shims/endoify'; -import { makePromiseKit } from '@endo/promise-kit'; -import type { - KernelCommand, - KernelCommandReply, - ClusterConfig, - VatCheckpoint, -} from '@ocap/kernel'; -import { Kernel } from '@ocap/kernel'; -import type { KernelDatabase, VatStore } from '@ocap/store'; +import type { ClusterConfig, VatCheckpoint } from '@ocap/kernel'; +import type { VatStore } from '@ocap/store'; import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; -import { NodeWorkerDuplexStream } from '@ocap/streams'; import deepEqual from 'fast-deep-equal'; -import { - MessagePort as NodeMessagePort, - MessageChannel as NodeMessageChannel, -} from 'node:worker_threads'; import { describe, vi, expect, it } from 'vitest'; -import { kunser } from '../../kernel/src/services/kernel-marshal.ts'; -import { NodejsVatWorkerService } from '../../nodejs/src/kernel/VatWorkerService.ts'; - -/** - * Construct a bundle path URL from a bundle name. - * - * @param bundleName - The name of the bundle. - * - * @returns a path string for the named bundle. - */ -function bundleSpec(bundleName: string): string { - return new URL(`${bundleName}.bundle`, import.meta.url).toString(); -} +import { getBundleSpec, makeKernel, runTestVats } from './utils.ts'; const makeTestSubcluster = (): ClusterConfig => ({ bootstrap: 'alice', forceReset: true, vats: { alice: { - bundleSpec: bundleSpec('vatstore-vat'), + bundleSpec: getBundleSpec('vatstore-vat'), parameters: { name: 'Alice', }, }, bob: { - bundleSpec: bundleSpec('vatstore-vat'), + bundleSpec: getBundleSpec('vatstore-vat'), parameters: { name: 'Bob', }, }, carol: { - bundleSpec: bundleSpec('vatstore-vat'), + bundleSpec: getBundleSpec('vatstore-vat'), parameters: { name: 'Carol', }, @@ -56,58 +32,6 @@ const makeTestSubcluster = (): ClusterConfig => ({ }, }); -/** - * Handle all the boilerplate to set up a kernel instance. - * - * @param kernelDatabase - The database that will hold the persistent state. - * @param resetStorage - If true, reset the database as part of setting up. - * - * @returns the new kernel instance. - */ -async function makeKernel( - kernelDatabase: KernelDatabase, - resetStorage: boolean, -): Promise { - const kernelPort: NodeMessagePort = new NodeMessageChannel().port1; - const nodeStream = new NodeWorkerDuplexStream< - KernelCommand, - KernelCommandReply - >(kernelPort); - const vatWorkerClient = new NodejsVatWorkerService({}); - const kernel = await Kernel.make( - nodeStream, - vatWorkerClient, - kernelDatabase, - { - resetStorage, - }, - ); - return kernel; -} - -/** - * Run the set of test vats. - * - * @param kernel - The kernel to run in. - * @param config - Subcluster configuration telling what vats to run. - * - * @returns the bootstrap result. - */ -async function runTestVats( - kernel: Kernel, - config: ClusterConfig, -): Promise { - const bootstrapResultRaw = await kernel.launchSubcluster(config); - - const { promise, resolve } = makePromiseKit(); - setTimeout(() => resolve(null), 0); - await promise; - if (bootstrapResultRaw === undefined) { - throw Error(`this can't happen but eslint is stupid`); - } - return kunser(bootstrapResultRaw); -} - const emptyMap = new Map(); const emptySet = new Set(); @@ -206,7 +130,6 @@ describe('exercise vatstore', async () => { ); const kernel = await makeKernel(kernelDatabase, true); await runTestVats(kernel, makeTestSubcluster()); - type VSRecord = { key: string; value: string }; const vsContents = kernelDatabase.executeQuery( `SELECT key, value from kv_vatStore where vatID = 'v1'`, @@ -220,7 +143,6 @@ describe('exercise vatstore', async () => { ); expect(vsKv.get('vc.1.sthing')).toBe('{"body":"#3","slots":[]}'); expect(vsKv.get('vc.1.|entryCount')).toBe('1'); - expect(deepEqual(kvUpdates, referenceKVUpdates)).toBe(true); }, 30000); }); diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 8baf89e7f..b67f11c3a 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -9,6 +9,7 @@ import { makeMarshal } from '@endo/marshal'; import type { CapData } from '@endo/marshal'; import { StreamReadError } from '@ocap/errors'; import type { DuplexStream } from '@ocap/streams'; +import { waitUntilQuiescent } from '@ocap/utils'; import type { VatCommand, VatCommandReply } from './messages/index.ts'; import { VatCommandMethod } from './messages/index.ts'; @@ -18,7 +19,6 @@ import type { DispatchFn, MakeLiveSlotsFn, GCTools } from './services/types.ts'; import type { VatConfig, VatId, VRef } from './types.ts'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts'; import { gcAndFinalize } from './utils/gc-finalize.ts'; -import { waitUntilQuiescent } from './utils/wait-quiescent.ts'; import type { VatKVStore } from './VatKVStore.ts'; import { makeVatKVStore } from './VatKVStore.ts'; @@ -236,6 +236,8 @@ export class VatSupervisor { const workerEndowments = { console, assert: globalThis.assert, + WeakRef, + FinalizationRegistry, }; const { bundleSpec, parameters } = vatConfig; diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index af0af1bfe..12f30afaf 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -18,3 +18,4 @@ export { VatConfigStruct, ClusterConfigStruct, } from './types.ts'; +export { kunser, kser } from './services/kernel-marshal.ts'; diff --git a/packages/utils/package.json b/packages/utils/package.json index 2ce25139b..9da62edb4 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -45,7 +45,8 @@ "@endo/captp": "^4.4.5", "@endo/promise-kit": "^1.1.10", "@metamask/superstruct": "^3.2.0", - "@metamask/utils": "^11.4.0" + "@metamask/utils": "^11.4.0", + "setimmediate": "^1.0.5" }, "devDependencies": { "@arethetypeswrong/cli": "^0.17.4", @@ -57,6 +58,7 @@ "@ocap/test-utils": "workspace:^", "@ts-bridge/cli": "^0.6.3", "@ts-bridge/shims": "^0.1.1", + "@types/setimmediate": "^1.0.4", "@typescript-eslint/eslint-plugin": "^8.29.0", "@typescript-eslint/parser": "^8.29.0", "@typescript-eslint/utils": "^8.29.0", diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.ts index c007a403c..3420e2861 100644 --- a/packages/utils/src/index.ts +++ b/packages/utils/src/index.ts @@ -10,3 +10,4 @@ export { isTypedObject, } from './types.ts'; export { fetchValidatedJson } from './fetchValidatedJson.ts'; +export { waitUntilQuiescent } from './wait-quiescent.ts'; diff --git a/packages/kernel/src/utils/wait-quiescent.test.ts b/packages/utils/src/wait-quiescent.test.ts similarity index 100% rename from packages/kernel/src/utils/wait-quiescent.test.ts rename to packages/utils/src/wait-quiescent.test.ts diff --git a/packages/kernel/src/utils/wait-quiescent.ts b/packages/utils/src/wait-quiescent.ts similarity index 75% rename from packages/kernel/src/utils/wait-quiescent.ts rename to packages/utils/src/wait-quiescent.ts index a8553229c..f89ea6e02 100644 --- a/packages/kernel/src/utils/wait-quiescent.ts +++ b/packages/utils/src/wait-quiescent.ts @@ -9,14 +9,20 @@ import { makePromiseKit } from '@endo/promise-kit'; * promise resolves, the holder can be assured that the environment no longer * has agency. * + * @param delay - Optional delay (in ms) to wait for things to catch up. + * * @returns a Promise that can await the compartment becoming quiescent. */ -export async function waitUntilQuiescent(): Promise { +export async function waitUntilQuiescent(delay: number = 0): Promise { // the delivery might cause some number of (native) Promises to be // created and resolved, so we use the IO queue to detect when the // Promise queue is empty. The IO queue (setImmediate and setTimeout) is // lower-priority than the Promise queue on browsers. const { promise: queueEmptyP, resolve } = makePromiseKit(); - setImmediate(() => resolve()); + if (delay > 0) { + setTimeout(() => resolve(), delay); + } else { + setImmediate(() => resolve()); + } return queueEmptyP; } diff --git a/yarn.lock b/yarn.lock index 69d5d1d1f..7e0291507 100644 --- a/yarn.lock +++ b/yarn.lock @@ -240,7 +240,7 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.25.9, @babel/generator@npm:^7.26.3, @babel/generator@npm:^7.26.9": +"@babel/generator@npm:^7.25.9, @babel/generator@npm:^7.26.3": version: 7.27.0 resolution: "@babel/generator@npm:7.27.0" dependencies: @@ -253,6 +253,19 @@ __metadata: languageName: node linkType: hard +"@babel/generator@npm:^7.26.9": + version: 7.26.9 + resolution: "@babel/generator@npm:7.26.9" + dependencies: + "@babel/parser": "npm:^7.26.9" + "@babel/types": "npm:^7.26.9" + "@jridgewell/gen-mapping": "npm:^0.3.5" + "@jridgewell/trace-mapping": "npm:^0.3.25" + jsesc: "npm:^3.0.2" + checksum: 10/95075dd6158a49efcc71d7f2c5d20194fcf245348de7723ca35e37cd5800587f1d4de2be6c4ba87b5f5fbb967c052543c109eaab14b43f6a73eb05ccd9a5bb44 + languageName: node + linkType: hard + "@babel/helper-compilation-targets@npm:^7.26.5": version: 7.26.5 resolution: "@babel/helper-compilation-targets@npm:7.26.5" @@ -380,7 +393,7 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.25.9, @babel/template@npm:^7.26.9": +"@babel/template@npm:^7.25.9": version: 7.27.0 resolution: "@babel/template@npm:7.27.0" dependencies: @@ -391,6 +404,17 @@ __metadata: languageName: node linkType: hard +"@babel/template@npm:^7.26.9": + version: 7.26.9 + resolution: "@babel/template@npm:7.26.9" + dependencies: + "@babel/code-frame": "npm:^7.26.2" + "@babel/parser": "npm:^7.26.9" + "@babel/types": "npm:^7.26.9" + checksum: 10/240288cebac95b1cc1cb045ad143365643da0470e905e11731e63280e43480785bd259924f4aea83898ef68e9fa7c176f5f2d1e8b0a059b27966e8ca0b41a1b6 + languageName: node + linkType: hard + "@babel/traverse@npm:^7.23.2, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.9": version: 7.26.9 resolution: "@babel/traverse@npm:7.26.9" @@ -573,7 +597,7 @@ __metadata: languageName: node linkType: hard -"@endo/common@npm:^1.2.10, @endo/common@npm:^1.2.8, @endo/common@npm:^1.2.9": +"@endo/common@npm:^1.2.10": version: 1.2.10 resolution: "@endo/common@npm:1.2.10" dependencies: @@ -584,6 +608,17 @@ __metadata: languageName: node linkType: hard +"@endo/common@npm:^1.2.8, @endo/common@npm:^1.2.9": + version: 1.2.9 + resolution: "@endo/common@npm:1.2.9" + dependencies: + "@endo/errors": "npm:^1.2.9" + "@endo/eventual-send": "npm:^1.3.0" + "@endo/promise-kit": "npm:^1.1.9" + checksum: 10/37d9bb1fbe6b974f135f755253ce58dd1692ed1849e1f2bdf91726bc62faba7dc4a9f9d527ce759feba08fe6cdb2597dc6bcc0daf8d516a780e856d61654f1d5 + languageName: node + linkType: hard + "@endo/compartment-mapper@npm:^1.6.0": version: 1.6.0 resolution: "@endo/compartment-mapper@npm:1.6.0" @@ -604,7 +639,7 @@ __metadata: languageName: node linkType: hard -"@endo/errors@npm:^1.2.10, @endo/errors@npm:^1.2.8, @endo/errors@npm:^1.2.9": +"@endo/errors@npm:^1.2.10": version: 1.2.10 resolution: "@endo/errors@npm:1.2.10" dependencies: @@ -613,6 +648,15 @@ __metadata: languageName: node linkType: hard +"@endo/errors@npm:^1.2.8, @endo/errors@npm:^1.2.9": + version: 1.2.9 + resolution: "@endo/errors@npm:1.2.9" + dependencies: + ses: "npm:^1.11.0" + checksum: 10/06457df5fa31709683f22fdf6b61e9f45056557308360ee582f3ed659476a44b8960b5c7337283a0c8da3162a4f1087883d228088f34a1fcefcb129cbd5188c6 + languageName: node + linkType: hard + "@endo/evasive-transform@npm:^1.4.0": version: 1.4.0 resolution: "@endo/evasive-transform@npm:1.4.0" @@ -648,7 +692,18 @@ __metadata: languageName: node linkType: hard -"@endo/far@npm:^1.0.0, @endo/far@npm:^1.1.10, @endo/far@npm:^1.1.11, @endo/far@npm:^1.1.9": +"@endo/far@npm:^1.0.0, @endo/far@npm:^1.1.10, @endo/far@npm:^1.1.9": + version: 1.1.10 + resolution: "@endo/far@npm:1.1.10" + dependencies: + "@endo/errors": "npm:^1.2.9" + "@endo/eventual-send": "npm:^1.3.0" + "@endo/pass-style": "npm:^1.4.8" + checksum: 10/028ca6ee4388f238bb452b8922507434867b689b283082209f948fd14059aab2cfde2d7fcf2e47c0f297bb345ecec615100f05b50dfb0e037b8bccd061b39a03 + languageName: node + linkType: hard + +"@endo/far@npm:^1.1.11": version: 1.1.11 resolution: "@endo/far@npm:1.1.11" dependencies: @@ -752,7 +807,7 @@ __metadata: languageName: node linkType: hard -"@endo/promise-kit@npm:^1.1.10, @endo/promise-kit@npm:^1.1.8, @endo/promise-kit@npm:^1.1.9": +"@endo/promise-kit@npm:^1.1.10": version: 1.1.10 resolution: "@endo/promise-kit@npm:1.1.10" dependencies: @@ -761,7 +816,16 @@ __metadata: languageName: node linkType: hard -"@endo/stream@npm:^1.2.10, @endo/stream@npm:^1.2.8, @endo/stream@npm:^1.2.9": +"@endo/promise-kit@npm:^1.1.8, @endo/promise-kit@npm:^1.1.9": + version: 1.1.9 + resolution: "@endo/promise-kit@npm:1.1.9" + dependencies: + ses: "npm:^1.11.0" + checksum: 10/fad342c0573252b57bcd0601d0b47814068333942fdfb4ea1e9dc7980910e46cb6d22fe5715b32571b002cafad9cf085584267bf42d32282768f471d15e665ae + languageName: node + linkType: hard + +"@endo/stream@npm:^1.2.10": version: 1.2.10 resolution: "@endo/stream@npm:1.2.10" dependencies: @@ -772,6 +836,17 @@ __metadata: languageName: node linkType: hard +"@endo/stream@npm:^1.2.8, @endo/stream@npm:^1.2.9": + version: 1.2.9 + resolution: "@endo/stream@npm:1.2.9" + dependencies: + "@endo/eventual-send": "npm:^1.3.0" + "@endo/promise-kit": "npm:^1.1.9" + ses: "npm:^1.11.0" + checksum: 10/5605e0135ad1ac426bafac3d3660642b64943eb0148a4b6a7b91e5d1a3bbd7ff856425207e80e6eee998ba47b2f3a1314cce52e4d7f965fd11bbd3e15a1144be + languageName: node + linkType: hard + "@endo/trampoline@npm:^1.0.3": version: 1.0.3 resolution: "@endo/trampoline@npm:1.0.3" @@ -2015,9 +2090,11 @@ __metadata: "@metamask/eslint-config-typescript": "npm:^14.0.0" "@ocap/cli": "workspace:^" "@ocap/kernel": "workspace:^" + "@ocap/nodejs": "workspace:^" "@ocap/shims": "workspace:^" "@ocap/store": "workspace:^" "@ocap/streams": "workspace:^" + "@ocap/utils": "workspace:^" "@ts-bridge/cli": "npm:^0.6.3" "@ts-bridge/shims": "npm:^0.1.1" "@typescript-eslint/eslint-plugin": "npm:^8.29.0" @@ -2148,7 +2225,7 @@ __metadata: languageName: unknown linkType: soft -"@ocap/nodejs@workspace:packages/nodejs": +"@ocap/nodejs@workspace:^, @ocap/nodejs@workspace:packages/nodejs": version: 0.0.0-use.local resolution: "@ocap/nodejs@workspace:packages/nodejs" dependencies: @@ -2364,6 +2441,7 @@ __metadata: "@ocap/test-utils": "workspace:^" "@ts-bridge/cli": "npm:^0.6.3" "@ts-bridge/shims": "npm:^0.1.1" + "@types/setimmediate": "npm:^1.0.4" "@typescript-eslint/eslint-plugin": "npm:^8.29.0" "@typescript-eslint/parser": "npm:^8.29.0" "@typescript-eslint/utils": "npm:^8.29.0" @@ -2381,6 +2459,7 @@ __metadata: prettier: "npm:^3.5.3" rimraf: "npm:^6.0.1" ses: "npm:^1.12.0" + setimmediate: "npm:^1.0.5" typedoc: "npm:^0.28.1" typescript: "npm:~5.8.2" typescript-eslint: "npm:^8.29.0" @@ -9597,6 +9676,15 @@ __metadata: languageName: node linkType: hard +"ses@npm:^1.11.0": + version: 1.11.0 + resolution: "ses@npm:1.11.0" + dependencies: + "@endo/env-options": "npm:^1.1.8" + checksum: 10/b080fce2f369bd4e3cd7df01704347fc50b426b21269920ad89aeb36de3a3acbf9b849152d27c9fb2c77ea929d5b0f6568e2c2322bda5a4621e9d3d2d70b4cf0 + languageName: node + linkType: hard + "set-function-length@npm:^1.2.2": version: 1.2.2 resolution: "set-function-length@npm:1.2.2" From de88cbade708f486b4af82ac1a4a5e793b1f68ca Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 28 Mar 2025 20:16:55 +0100 Subject: [PATCH 04/23] reset dependabot --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ae51ba99a..e1f12aecf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,7 +10,7 @@ updates: day: 'monday' time: '06:00' # UTC target-branch: 'main' - versioning-strategy: 'widen' + versioning-strategy: 'increase-if-necessary' open-pull-requests-limit: 10 groups: vite: From f42b6cbb54a83b8feb1138d7f22bab2c399fd1af Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 1 Apr 2025 14:16:34 +0200 Subject: [PATCH 05/23] wip --- .../src/garbage-collection.test.ts | 286 ++++++++++++++++++ .../kernel-test/src/gc-integration.test.ts | 49 --- packages/kernel-test/src/liveslots.test.ts | 2 +- packages/kernel-test/src/vats/exporter-vat.js | 136 +++++++++ packages/kernel-test/src/vats/gc-test-vat.js | 39 --- packages/kernel-test/src/vats/importer-vat.js | 127 ++++++++ .../src/vats/message-to-promise-vat.js | 1 + packages/kernel/src/Kernel.ts | 12 +- packages/kernel/src/VatHandle.ts | 1 + packages/kernel/src/VatSupervisor.ts | 1 + packages/kernel/src/index.ts | 3 + packages/kernel/src/services/meter-control.ts | 1 + packages/kernel/src/services/syscall.ts | 2 + 13 files changed, 570 insertions(+), 90 deletions(-) create mode 100644 packages/kernel-test/src/garbage-collection.test.ts delete mode 100644 packages/kernel-test/src/gc-integration.test.ts create mode 100644 packages/kernel-test/src/vats/exporter-vat.js delete mode 100644 packages/kernel-test/src/vats/gc-test-vat.js create mode 100644 packages/kernel-test/src/vats/importer-vat.js diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts new file mode 100644 index 000000000..cba3007d4 --- /dev/null +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -0,0 +1,286 @@ +import '@ocap/shims/endoify'; +import { Kernel, kunser, makeKernelStore } from '@ocap/kernel'; +import type { ClusterConfig, KRef, KernelStore, VatId } from '@ocap/kernel'; +import type { KernelDatabase } from '@ocap/store'; +import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; +import { waitUntilQuiescent } from '@ocap/utils'; +import { expect, beforeEach, afterEach, describe, it } from 'vitest'; + +import { getBundleSpec, makeKernel, runTestVats } from './utils.ts'; + +/** + * Make a test subcluster with vats for GC testing + * + * @returns The test subcluster + */ +function makeTestSubcluster(): ClusterConfig { + return { + bootstrap: 'exporter', + forceReset: true, + bundles: null, + vats: { + exporter: { + bundleSpec: getBundleSpec('exporter-vat'), + parameters: { + name: 'Exporter', + }, + }, + importer: { + bundleSpec: getBundleSpec('importer-vat'), + parameters: { + name: 'Importer', + }, + }, + }, + }; +} + +describe('Garbage Collection E2E Tests', () => { + let kernel: Kernel; + let kernelDatabase: KernelDatabase; + let kernelStore: KernelStore; + let exporterKRef: KRef; + let importerKRef: KRef; + let exporterVatId: VatId; + let importerVatId: VatId; + + beforeEach(async () => { + kernelDatabase = await makeSQLKernelDatabase({ + dbFilename: ':memory:', + }); + kernelStore = makeKernelStore(kernelDatabase); + kernel = await makeKernel(kernelDatabase, true); + await runTestVats(kernel, makeTestSubcluster()); + + const vats = kernel.getVats(); + exporterVatId = vats.find( + (rows) => rows.config.parameters?.name === 'Exporter', + )?.id as VatId; + importerVatId = vats.find( + (rows) => rows.config.parameters?.name === 'Importer', + )?.id as VatId; + exporterKRef = kernelStore.erefToKref(exporterVatId, 'o+0') as KRef; + importerKRef = kernelStore.erefToKref(importerVatId, 'o+0') as KRef; + }); + + afterEach(async () => { + console.log('$$$ DB', kernelDatabase.executeQuery('SELECT * FROM kv')); + }); + + it('objects are tracked with reference counts', async () => { + const objectId = 'test-object'; + // Create an object in the exporter vat + const createObjectData = await kernel.queueMessageFromKernel( + exporterKRef, + 'createObject', + [objectId], + ); + const createObjectRef = createObjectData.slots[0] as KRef; + // Verify initial reference counts from database + const initialRefCounts = kernelStore.getObjectRefCount(createObjectRef); + expect(initialRefCounts.reachable).toBe(1); + expect(initialRefCounts.recognizable).toBe(1); + // Send the object to the importer vat + const objectRef = kunser(createObjectData); + await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ + objectRef, + ]); + await waitUntilQuiescent(); + // Check that the object is reachable from the exporter vat + const exporterReachable = kernelStore.getReachableFlag( + exporterVatId, + createObjectRef, + ); + expect(exporterReachable).toBe(true); + // Check that the object is reachable as a promise from the importer vat + const importerKref = kernelStore.erefToKref(importerVatId, 'p-1') as KRef; + expect(kernelStore.hasCListEntry(importerVatId, importerKref)).toBe(true); + expect(kernelStore.getRefCount(importerKref)).toBe(1); + // Use the object + const useResult = await kernel.queueMessageFromKernel( + importerKRef, + 'useImport', + [], + ); + await waitUntilQuiescent(); + expect(useResult.body).toContain(objectId); + }); + + it('should trigger GC syscalls through bringOutYourDead', async () => { + // 1. Create an object in the exporter vat with a known ID + const objectId = 'test-object'; + const createObjectData = await kernel.queueMessageFromKernel( + exporterKRef, + 'createObject', + [objectId], + ); + const objectRef = createObjectData.slots[0] as KRef; + + // Store initial reference count information + const initialRefCounts = kernelStore.getObjectRefCount(objectRef); + expect(initialRefCounts.reachable).toBe(1); + expect(initialRefCounts.recognizable).toBe(1); + + // 2. Store the reference in the importer vat + await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ + objectRef, + objectId, + ]); + await waitUntilQuiescent(); + + // 3. Verify object is tracked in both vats + const importerHasObject = await kernel.queueMessageFromKernel( + importerKRef, + 'listImportedObjects', + [], + ); + console.log('$$$ importerHasObject', importerHasObject); + expect(importerHasObject.body).toContain(objectId); + + const exporterHasObject = await kernel.queueMessageFromKernel( + exporterKRef, + 'isObjectPresent', + [objectId], + ); + console.log('$$$ exporterHasObject', exporterHasObject); + expect(exporterHasObject.body).toBe('#true'); + + // 4. Make a weak reference to the object in the importer vat + // This should eventually trigger dropImports when GC runs + await kernel.queueMessageFromKernel(importerKRef, 'makeWeak', [objectId]); + await waitUntilQuiescent(); + + // 5. Schedule reap to trigger bringOutYourDead on next crank + kernelStore.scheduleReap(exporterVatId); + + // 6. Run a crank to allow bringOutYourDead to be processed + await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); + await waitUntilQuiescent(100); + + // Check reference counts after dropImports (should be decreased reachable but same recognizable) + const afterWeakRefCounts = kernelStore.getObjectRefCount(objectRef); + expect(afterWeakRefCounts.reachable).toBeLessThan( + initialRefCounts.reachable, + ); + expect(afterWeakRefCounts.recognizable).toBe(initialRefCounts.recognizable); + + // 7. Now completely forget the import in the importer vat + // This should trigger retireImports when GC runs + await kernel.queueMessageFromKernel(importerKRef, 'forgetImport', [ + objectId, + ]); + await waitUntilQuiescent(); + + // 8. Schedule another reap + kernelStore.scheduleReap(importerVatId); + + // 9. Run a crank to allow bringOutYourDead to be processed + await kernel.queueMessageFromKernel(importerKRef, 'noop', []); + await waitUntilQuiescent(100); + + // Check reference counts after retireImports (both should be decreased) + const afterForgetRefCounts = kernelStore.getObjectRefCount(objectRef); + expect(afterForgetRefCounts.reachable).toBeLessThan( + initialRefCounts.reachable, + ); + expect(afterForgetRefCounts.recognizable).toBeLessThan( + initialRefCounts.recognizable, + ); + + // 10. Now forget the object in the exporter vat + // This should trigger retireExports when GC runs + await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ + objectId, + ]); + await waitUntilQuiescent(); + + // 11. Schedule a final reap + kernelStore.scheduleReap(exporterVatId); + + // 12. Run multiple cranks to ensure GC completes + for (let i = 0; i < 3; i++) { + await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); + await waitUntilQuiescent(50); + } + + // Verify the object has been completely removed + const exporterFinalCheck = await kernel.queueMessageFromKernel( + exporterKRef, + 'isObjectPresent', + [objectId], + ); + console.log('$$$ exporterFinalCheck', exporterFinalCheck); + expect(exporterFinalCheck.body).toBe('#false'); + + // Check if reference still exists in the kernel store at all + const refExists = kernelStore.kernelRefExists(objectRef); + expect(refExists).toBe(false); + + // 13. Test abandonExports by creating a new object and forcing its removal + const abandonObjectId = 'abandon-test'; + const abandonObjData = await kernel.queueMessageFromKernel( + exporterKRef, + 'createObject', + [abandonObjectId], + ); + console.log('$$$ abandonObjData', abandonObjData); + const abandonObjRef = abandonObjData.slots[0] as KRef; + + // Store in importer to make it reachable from both vats + await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ + abandonObjRef, + abandonObjectId, + ]); + await waitUntilQuiescent(); + + // Verify it's reachable from both vats + const abandonRefCounts = kernelStore.getObjectRefCount(abandonObjRef); + expect(abandonRefCounts.reachable).toBe(1); + + // Force remove in exporter (this simulates abandonExports) + await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ + abandonObjectId, + ]); + await waitUntilQuiescent(); + + // Schedule reap to trigger abandonExports + kernelStore.scheduleReap(exporterVatId); + + // Run multiple cranks to ensure GC completes + for (let i = 0; i < 3; i++) { + await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); + await waitUntilQuiescent(50); + } + + // Verify object is gone from exporter + const exporterAbandonCheck = await kernel.queueMessageFromKernel( + exporterKRef, + 'isObjectPresent', + [abandonObjectId], + ); + console.log('$$$ exporterAbandonCheck', exporterAbandonCheck); + expect(exporterAbandonCheck.body).toBe('#false'); + + // But it should still be in the importer's list + const importerAbandonCheck = await kernel.queueMessageFromKernel( + importerKRef, + 'listImportedObjects', + [], + ); + console.log('$$$ importerAbandonCheck', importerAbandonCheck); + expect(importerAbandonCheck.body).toContain(abandonObjectId); + + // However, using the object should now fail + try { + await kernel.queueMessageFromKernel(importerKRef, 'useImport', [ + abandonObjectId, + ]); + // Should not reach here + expect(false).toBe(true); + } catch (error) { + // We expect an error + // eslint-disable-next-line vitest/no-conditional-expect + expect(error).toBeDefined(); + } + }); +}); diff --git a/packages/kernel-test/src/gc-integration.test.ts b/packages/kernel-test/src/gc-integration.test.ts deleted file mode 100644 index 0ee74edad..000000000 --- a/packages/kernel-test/src/gc-integration.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import '@ocap/shims/endoify'; -import type { ClusterConfig } from '@ocap/kernel'; -import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; -import { describe, expect, it } from 'vitest'; - -import { - extractVatLogs, - getBundleSpec, - makeKernel, - runTestVats, -} from './utils.ts'; - -const origStdoutWrite = process.stdout.write.bind(process.stdout); -let buffered: string = ''; -// @ts-expect-error Some type def used by lint is just wrong (compiler likes it ok, but lint whines) -process.stdout.write = (buffer: string, encoding, callback): void => { - buffered += buffer; - origStdoutWrite(buffer, encoding, callback); -}; - -// Define a very simple test cluster -const simpleGCTestSubcluster: ClusterConfig = { - bootstrap: 'gcVat', - forceReset: true, - bundles: null, - vats: { - gcVat: { - bundleSpec: getBundleSpec('gc-test-vat'), - parameters: { - name: 'GCVat', - }, - }, - }, -}; - -describe('Simple GC Tests', () => { - it('should create a WeakRef successfully', async () => { - const kernelDatabase = await makeSQLKernelDatabase({ - dbFilename: ':memory:', - }); - const kernel = await makeKernel(kernelDatabase, true); - const bootstrapResult = await runTestVats(kernel, simpleGCTestSubcluster); - expect(bootstrapResult).toBe('gc-test-complete'); - const vatLogs = extractVatLogs(buffered); - expect(vatLogs).toContain( - 'GCVat: WeakRef created and object is accessible', - ); - }, 10000); -}); diff --git a/packages/kernel-test/src/liveslots.test.ts b/packages/kernel-test/src/liveslots.test.ts index a02ec357f..4de20d205 100644 --- a/packages/kernel-test/src/liveslots.test.ts +++ b/packages/kernel-test/src/liveslots.test.ts @@ -279,7 +279,7 @@ describe('liveslots promise handling', () => { expect(vatLogs).toStrictEqual(reference); }, 30000); - it('messageToPromise: send to promise before resolution', async () => { + it.only('messageToPromise: send to promise before resolution', async () => { const [bootstrapResult, vatLogs] = await runTestVats( 'message-to-promise-vat', 'messageToPromise', diff --git a/packages/kernel-test/src/vats/exporter-vat.js b/packages/kernel-test/src/vats/exporter-vat.js new file mode 100644 index 000000000..1293d1307 --- /dev/null +++ b/packages/kernel-test/src/vats/exporter-vat.js @@ -0,0 +1,136 @@ +// import { E } from '@endo/eventual-send'; +import { Far } from '@endo/marshal'; + +/** + * Build function for vats that will run various tests. + * + * @param {*} _vatPowers - Special powers granted to this vat (not used here). + * @param {*} parameters - Initialization parameters from the vat's config object. + * @param {*} _baggage - Root of vat's persistent state (not used here). + * @returns {*} The root object for the new vat. + */ +export function buildRootObject(_vatPowers, parameters, _baggage) { + const name = parameters?.name ?? 'anonymous'; + // const { getSyscall } = vatPowers; + // const syscall = getSyscall(); + + /** + * Print a message to the log. + * + * @param {string} message - The message to print. + */ + function log(message) { + console.log(`${name}: ${message}`); + } + + /** + * Print a message to the log, tagged as part of the test output. + * + * @param {string} message - The message to print. + * @param {...any} args - Additional arguments to print. + */ + function tlog(message, ...args) { + console.log(`::> ${name}: ${message}`, ...args); + } + + // Track objects we've created + const exportedObjects = new Map(); + // Track WeakRefs + const weakRefs = new Map(); + // FinalizationRegistry to know when objects are GC'd + const finalizer = new FinalizationRegistry((id) => { + console.log(`::> ${name}: Object ${id} was finalized`); + weakRefs.delete(id); + }); + + return Far('root', { + bootstrap() { + log(`bootstrap`); + return this; + }, + + createObject(id) { + const obj = Far('SharedObject', { + getValue() { + return id; + }, + }); + exportedObjects.set(id, obj); + tlog(`Created object ${id}`); + return obj; + }, + + getExportedObjectCount() { + return exportedObjects.size; + }, + + // Create a weak reference to an object + makeWeakRef(objId) { + const obj = exportedObjects.get(objId); + if (obj) { + tlog(`Creating weak reference to object ${objId}`); + const ref = new WeakRef(obj); + weakRefs.set(objId, ref); + finalizer.register(obj, objId); + return true; + } + return false; + }, + + // Remove the strong reference, keeping only the weak reference + removeStrongRef(objId) { + if (this.makeWeakRef(objId)) { + tlog(`Removing strong reference to object ${objId}`); + exportedObjects.delete(objId); + return true; + } + return false; + }, + + // Check if an object is still held strongly + isStronglyHeld(objId) { + return exportedObjects.has(objId); + }, + + // Try to access a potentially GC'd object through its weak reference + tryAccessWeakRef(objId) { + const weakRef = weakRefs.get(objId); + if (!weakRef) { + return { status: 'no weak ref' }; + } + + const obj = weakRef.deref(); + if (!obj) { + return { status: 'collected' }; + } + + try { + const value = obj.getValue(); + return { status: 'alive', value }; + } catch (error) { + return { status: 'error', message: String(error) }; + } + }, + + // Check if an object exists in our maps + isObjectPresent(objId) { + return exportedObjects.has(objId); + }, + + // Remove an object from our tracking, allowing it to be GC'd + forgetObject(objId) { + if (exportedObjects.has(objId)) { + tlog(`Forgetting object ${objId}`); + exportedObjects.delete(objId); + return true; + } + tlog(`Cannot forget nonexistent object: ${objId}`); + return false; + }, + + // No-op to help trigger crank cycles + noop() { + return 'noop'; + }, + }); +} diff --git a/packages/kernel-test/src/vats/gc-test-vat.js b/packages/kernel-test/src/vats/gc-test-vat.js deleted file mode 100644 index 94ff26c71..000000000 --- a/packages/kernel-test/src/vats/gc-test-vat.js +++ /dev/null @@ -1,39 +0,0 @@ -import { Far } from '@endo/marshal'; - -/** - * Build function for a very simple vat that just tests WeakRef creation. - * - * @param {object} _vatPowers - Special powers granted to this vat. - * @param {object} parameters - Initialization parameters from the vat's config object. - * @returns {object} The root object for the new vat. - */ -export function buildRootObject(_vatPowers, parameters) { - const name = parameters?.name ?? 'anonymous'; - - /** - * Print a message to the test log. - * - * @param {string} message - The message to print. - */ - function tlog(message) { - console.log(`::> ${name}: ${message}`); - } - - return Far('root', { - async bootstrap() { - // Simply create an object and a WeakRef to it - const obj = { value: 'test object' }; - const weakRef = new WeakRef(obj); - - // Verify the WeakRef works - const retrieved = weakRef.deref(); - if (retrieved && retrieved.value === 'test object') { - tlog('WeakRef created and object is accessible'); - } else { - tlog('ERROR: WeakRef failed to retrieve object'); - } - - return 'gc-test-complete'; - }, - }); -} diff --git a/packages/kernel-test/src/vats/importer-vat.js b/packages/kernel-test/src/vats/importer-vat.js new file mode 100644 index 000000000..b8bae5a02 --- /dev/null +++ b/packages/kernel-test/src/vats/importer-vat.js @@ -0,0 +1,127 @@ +import { E } from '@endo/eventual-send'; +import { Far } from '@endo/marshal'; + +/** + * Build function for vats that will run various tests. + * + * @param {*} _vatPowers - Special powers granted to this vat (not used here). + * @param {*} parameters - Initialization parameters from the vat's config object. + * @param {*} _baggage - Root of vat's persistent state (not used here). + * @returns {*} The root object for the new vat. + */ +export function buildRootObject(_vatPowers, parameters, _baggage) { + const name = parameters?.name ?? 'anonymous'; + // const { getSyscall } = vatPowers; + // const syscall = getSyscall(); + const importedObjects = new Map(); + const weakRefs = new Map(); + const finalizer = new FinalizationRegistry((id) => { + console.log(`::> ${name}: Imported object ${id} was finalized`); + weakRefs.delete(id); + }); + + /** + * Print a message to the log. + * + * @param {string} message - The message to print. + */ + function log(message) { + console.log(`${name}: ${message}`); + } + + /** + * Print a message to the log, tagged as part of the test output. + * + * @param {string} message - The message to print. + * @param {...any} args - Additional arguments to print. + */ + function tlog(message, ...args) { + console.log(`::> ${name}: ${message}`, ...args); + } + + return Far('root', { + bootstrap() { + log(`bootstrap`); + return this; + }, + + storeImport(obj, id = 'default') { + tlog(`Storing import ${id}`, obj); + importedObjects.set(id, obj); + return 'stored'; + }, + + useImport(id = 'default') { + tlog(`useImport ${id}`); + const obj = importedObjects.get(id); + if (!obj) { + throw new Error(`Object not found: ${id}`); + } + tlog(`Using import ${id}`); + return E(obj).getValue(); + }, + + makeWeak(id) { + const obj = importedObjects.get(id); + if (!obj) { + tlog(`Cannot make weak reference to nonexistent object: ${id}`); + return false; + } + + tlog(`Making weak reference to ${id}`); + const ref = new WeakRef(obj); + weakRefs.set(id, ref); + finalizer.register(obj, id); + importedObjects.delete(id); + return true; + }, + + forgetImport(id) { + const had = importedObjects.has(id); + if (had) { + tlog(`Forgetting import ${id}`); + importedObjects.delete(id); + weakRefs.delete(id); + } else { + tlog(`Cannot forget nonexistent import: ${id}`); + } + return had; + }, + + getImportedObjectCount() { + return importedObjects.size; + }, + + listImportedObjects() { + return Array.from(importedObjects.keys()); + }, + + checkAccess(id) { + if (importedObjects.has(id)) { + return { status: 'strong', accessible: true }; + } + + const weakRef = weakRefs.get(id); + if (!weakRef) { + return { status: 'no reference' }; + } + + const obj = weakRef.deref(); + if (!obj) { + return { status: 'collected' }; + } + + try { + const value = E(obj).getValue(); + return { status: 'weak', accessible: true, value }; + } catch (error) { + return { status: 'error', message: String(error) }; + } + }, + + // No-op to help trigger crank cycles + noop() { + return 'noop'; + }, + }); +} diff --git a/packages/kernel-test/src/vats/message-to-promise-vat.js b/packages/kernel-test/src/vats/message-to-promise-vat.js index 17801261d..17e503b7d 100644 --- a/packages/kernel-test/src/vats/message-to-promise-vat.js +++ b/packages/kernel-test/src/vats/message-to-promise-vat.js @@ -47,6 +47,7 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { async bootstrap(vats) { log(`bootstrap start`); tlog(`running test ${test}`); + console.log('$$$ vats', vats); const promise1 = E(vats.bob).setup(); const promise2 = E(promise1).doSomething(); const doneP = promise2.then( diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 5a9aeae09..1dc24dd71 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -181,6 +181,7 @@ export class Kernel { */ async #run(): Promise { for await (const item of this.#runQueueItems()) { + console.log('*** run loop item', item); await this.#deliver(item); } } @@ -922,5 +923,14 @@ export class Kernel { get clusterConfig(): ClusterConfig | null { return this.#mostRecentSubcluster; } + + /** + * Reap all vats. + */ + reapAllVats(): void { + for (const vatID of this.getVatIds()) { + this.#kernelStore.scheduleReap(vatID); + } + } } -// harden(Kernel); // XXX restore this once vitest is able to cope +harden(Kernel); // XXX restore this once vitest is able to cope diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index 63cc083b6..ea169ed9d 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -567,6 +567,7 @@ export class VatHandle { * Make a 'bringOutYourDead' delivery to the vat. */ async deliverBringOutYourDead(): Promise { + console.log('*** deliverBringOutYourDead'); await this.sendVatCommand({ method: VatCommandMethod.deliver, params: ['bringOutYourDead'], diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index b67f11c3a..4415da79a 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -120,6 +120,7 @@ export class VatSupervisor { console.error(`cannot deliver before vat is loaded`); return; } + console.log('*** handleMessage deliver', payload.params); await this.#dispatch(harden(payload.params) as VatDeliveryObject); await Promise.all(this.#syscallsInFlight); this.#syscallsInFlight.length = 0; diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index 12f30afaf..c4b7cda47 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -19,3 +19,6 @@ export { ClusterConfigStruct, } from './types.ts'; export { kunser, kser } from './services/kernel-marshal.ts'; +export { makeKernelStore } from './store/kernel-store.ts'; +export type { KernelStore } from './store/kernel-store.ts'; +export { parseRef } from './store/utils/parse-ref.ts'; diff --git a/packages/kernel/src/services/meter-control.ts b/packages/kernel/src/services/meter-control.ts index dc997e46f..97b1c738c 100644 --- a/packages/kernel/src/services/meter-control.ts +++ b/packages/kernel/src/services/meter-control.ts @@ -63,6 +63,7 @@ export function makeDummyMeterControl(): MeterControl { async function runWithoutMeteringAsync( thunk: () => unknown, ): Promise { + console.log('*** runWithoutMeteringAsync'); meteringDisabled += 1; return Promise.resolve() .then(() => thunk()) diff --git a/packages/kernel/src/services/syscall.ts b/packages/kernel/src/services/syscall.ts index ede62bcf8..18d4b22ba 100644 --- a/packages/kernel/src/services/syscall.ts +++ b/packages/kernel/src/services/syscall.ts @@ -37,10 +37,12 @@ function makeSupervisorSyscall( * @returns the result from performing the syscall. */ function doSyscall(vso: VatSyscallObject): SyscallResult { + console.log('$$$ VAT doSyscall', vso); insistVatSyscallObject(vso); let syscallResult; try { syscallResult = supervisor.executeSyscall(vso); + console.log('$$$ VAT syscallResult', syscallResult); } catch (problem) { console.warn(`supervisor got error during syscall:`, problem); throw problem; From 206306f3f6e843691313eeed15df15e340da2612 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Tue, 1 Apr 2025 21:35:37 +0200 Subject: [PATCH 06/23] testing retireImports and dropImports --- .../src/garbage-collection.test.ts | 208 +++++++++--------- packages/kernel-test/src/utils.ts | 15 ++ packages/kernel-test/src/vats/exporter-vat.js | 42 ++-- packages/kernel-test/src/vats/importer-vat.js | 85 ++++--- packages/kernel/src/VatHandle.ts | 9 +- packages/kernel/src/VatSupervisor.ts | 4 +- packages/kernel/src/utils/gc-finalize.test.ts | 4 +- packages/kernel/src/utils/gc-finalize.ts | 8 +- 8 files changed, 196 insertions(+), 179 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index cba3007d4..d0ca359f9 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -6,7 +6,12 @@ import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; import { waitUntilQuiescent } from '@ocap/utils'; import { expect, beforeEach, afterEach, describe, it } from 'vitest'; -import { getBundleSpec, makeKernel, runTestVats } from './utils.ts'; +import { + getBundleSpec, + makeKernel, + parseReplyBody, + runTestVats, +} from './utils.ts'; /** * Make a test subcluster with vats for GC testing @@ -103,7 +108,7 @@ describe('Garbage Collection E2E Tests', () => { [], ); await waitUntilQuiescent(); - expect(useResult.body).toContain(objectId); + expect(parseReplyBody(useResult.body)).toBe(objectId); }); it('should trigger GC syscalls through bringOutYourDead', async () => { @@ -114,20 +119,26 @@ describe('Garbage Collection E2E Tests', () => { 'createObject', [objectId], ); - const objectRef = createObjectData.slots[0] as KRef; + const createObjectRef = createObjectData.slots[0] as KRef; // Store initial reference count information - const initialRefCounts = kernelStore.getObjectRefCount(objectRef); + const initialRefCounts = kernelStore.getObjectRefCount(createObjectRef); + console.log('Initial ref counts:', createObjectRef, initialRefCounts); expect(initialRefCounts.reachable).toBe(1); expect(initialRefCounts.recognizable).toBe(1); // 2. Store the reference in the importer vat + const objectRef = kunser(createObjectData); await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ objectRef, objectId, ]); await waitUntilQuiescent(); + // Get reference counts after storing in importer + const afterStoreRefCounts = kernelStore.getObjectRefCount(createObjectRef); + console.log('After store ref counts:', objectRef, afterStoreRefCounts); + // 3. Verify object is tracked in both vats const importerHasObject = await kernel.queueMessageFromKernel( importerKRef, @@ -135,7 +146,7 @@ describe('Garbage Collection E2E Tests', () => { [], ); console.log('$$$ importerHasObject', importerHasObject); - expect(importerHasObject.body).toContain(objectId); + expect(parseReplyBody(importerHasObject.body)).toContain(objectId); const exporterHasObject = await kernel.queueMessageFromKernel( exporterKRef, @@ -143,59 +154,56 @@ describe('Garbage Collection E2E Tests', () => { [objectId], ); console.log('$$$ exporterHasObject', exporterHasObject); - expect(exporterHasObject.body).toBe('#true'); + expect(parseReplyBody(exporterHasObject.body)).toBe(true); // 4. Make a weak reference to the object in the importer vat // This should eventually trigger dropImports when GC runs await kernel.queueMessageFromKernel(importerKRef, 'makeWeak', [objectId]); await waitUntilQuiescent(); - // 5. Schedule reap to trigger bringOutYourDead on next crank - kernelStore.scheduleReap(exporterVatId); + console.log('$$$ kernelStore.getGCActions(1)', kernelStore.getGCActions()); + + // // 5. Schedule reap to trigger bringOutYourDead on next crank + kernel.reapAllVats(); - // 6. Run a crank to allow bringOutYourDead to be processed + // // 6. Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); await waitUntilQuiescent(100); - // Check reference counts after dropImports (should be decreased reachable but same recognizable) - const afterWeakRefCounts = kernelStore.getObjectRefCount(objectRef); - expect(afterWeakRefCounts.reachable).toBeLessThan( - initialRefCounts.reachable, - ); - expect(afterWeakRefCounts.recognizable).toBe(initialRefCounts.recognizable); + console.log('$$$ kernelStore.getGCActions(2)', kernelStore.getGCActions()); + + // // Check reference counts after dropImports + const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); + console.log('After weak ref counts:', afterWeakRefCounts); + expect(afterWeakRefCounts.reachable).toBe(0); + expect(afterWeakRefCounts.recognizable).toBe(1); // 7. Now completely forget the import in the importer vat // This should trigger retireImports when GC runs - await kernel.queueMessageFromKernel(importerKRef, 'forgetImport', [ - objectId, - ]); + await kernel.queueMessageFromKernel(importerKRef, 'forgetImport', []); await waitUntilQuiescent(); // 8. Schedule another reap - kernelStore.scheduleReap(importerVatId); + kernel.reapAllVats(); // 9. Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(importerKRef, 'noop', []); await waitUntilQuiescent(100); // Check reference counts after retireImports (both should be decreased) - const afterForgetRefCounts = kernelStore.getObjectRefCount(objectRef); - expect(afterForgetRefCounts.reachable).toBeLessThan( - initialRefCounts.reachable, - ); - expect(afterForgetRefCounts.recognizable).toBeLessThan( - initialRefCounts.recognizable, - ); + const afterForgetRefCounts = kernelStore.getObjectRefCount(createObjectRef); + expect(afterForgetRefCounts.reachable).toBe(0); + expect(afterForgetRefCounts.recognizable).toBe(0); - // 10. Now forget the object in the exporter vat - // This should trigger retireExports when GC runs + // // 10. Now forget the object in the exporter vat + // // This should trigger retireExports when GC runs await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ objectId, ]); await waitUntilQuiescent(); // 11. Schedule a final reap - kernelStore.scheduleReap(exporterVatId); + kernel.reapAllVats(); // 12. Run multiple cranks to ensure GC completes for (let i = 0; i < 3; i++) { @@ -210,77 +218,79 @@ describe('Garbage Collection E2E Tests', () => { [objectId], ); console.log('$$$ exporterFinalCheck', exporterFinalCheck); - expect(exporterFinalCheck.body).toBe('#false'); + expect(parseReplyBody(exporterFinalCheck.body)).toBe(false); // Check if reference still exists in the kernel store at all - const refExists = kernelStore.kernelRefExists(objectRef); - expect(refExists).toBe(false); - - // 13. Test abandonExports by creating a new object and forcing its removal - const abandonObjectId = 'abandon-test'; - const abandonObjData = await kernel.queueMessageFromKernel( - exporterKRef, - 'createObject', - [abandonObjectId], - ); - console.log('$$$ abandonObjData', abandonObjData); - const abandonObjRef = abandonObjData.slots[0] as KRef; - - // Store in importer to make it reachable from both vats - await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ - abandonObjRef, - abandonObjectId, - ]); - await waitUntilQuiescent(); - - // Verify it's reachable from both vats - const abandonRefCounts = kernelStore.getObjectRefCount(abandonObjRef); - expect(abandonRefCounts.reachable).toBe(1); - - // Force remove in exporter (this simulates abandonExports) - await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ - abandonObjectId, - ]); - await waitUntilQuiescent(); - - // Schedule reap to trigger abandonExports - kernelStore.scheduleReap(exporterVatId); - - // Run multiple cranks to ensure GC completes - for (let i = 0; i < 3; i++) { - await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); - await waitUntilQuiescent(50); - } - - // Verify object is gone from exporter - const exporterAbandonCheck = await kernel.queueMessageFromKernel( - exporterKRef, - 'isObjectPresent', - [abandonObjectId], - ); - console.log('$$$ exporterAbandonCheck', exporterAbandonCheck); - expect(exporterAbandonCheck.body).toBe('#false'); - - // But it should still be in the importer's list - const importerAbandonCheck = await kernel.queueMessageFromKernel( - importerKRef, - 'listImportedObjects', - [], - ); - console.log('$$$ importerAbandonCheck', importerAbandonCheck); - expect(importerAbandonCheck.body).toContain(abandonObjectId); - - // However, using the object should now fail - try { - await kernel.queueMessageFromKernel(importerKRef, 'useImport', [ - abandonObjectId, - ]); - // Should not reach here - expect(false).toBe(true); - } catch (error) { - // We expect an error - // eslint-disable-next-line vitest/no-conditional-expect - expect(error).toBeDefined(); - } + // const refExists = kernelStore.kernelRefExists(createObjectRef); + // expect(refExists).toBe(false); + + // // 13. Test abandonExports by creating a new object and forcing its removal + // const abandonObjectId = 'abandon-test'; + // const abandonObjData = await kernel.queueMessageFromKernel( + // exporterKRef, + // 'createObject', + // [abandonObjectId], + // ); + // console.log('$$$ abandonObjData', abandonObjData); + // const abandonObjRef = abandonObjData.slots[0] as KRef; + + // // Store in importer to make it reachable from both vats + // await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ + // abandonObjRef, + // abandonObjectId, + // ]); + // await waitUntilQuiescent(); + + // // Verify it's reachable from both vats + // const abandonRefCounts = kernelStore.getObjectRefCount(abandonObjRef); + // expect(abandonRefCounts.reachable).toBe(1); + + // // Force remove in exporter (this simulates abandonExports) + // await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ + // abandonObjectId, + // ]); + // await waitUntilQuiescent(); + + // // Schedule reap to trigger abandonExports + // kernelStore.scheduleReap(exporterVatId); + + // // Run multiple cranks to ensure GC completes + // for (let i = 0; i < 3; i++) { + // await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); + // await waitUntilQuiescent(50); + // } + + // // Verify object is gone from exporter + // const exporterAbandonCheck = await kernel.queueMessageFromKernel( + // exporterKRef, + // 'isObjectPresent', + // [abandonObjectId], + // ); + // console.log('$$$ exporterAbandonCheck', exporterAbandonCheck); + // expect(parseReplyBody(exporterAbandonCheck.body)).toBe(false); + + // // But it should still be in the importer's list + // const importerAbandonCheck = await kernel.queueMessageFromKernel( + // importerKRef, + // 'listImportedObjects', + // [], + // ); + // console.log('$$$ importerAbandonCheck', importerAbandonCheck); + // expect(parseReplyBody(importerAbandonCheck.body)).toContain( + // abandonObjectId, + // ); + + // // However, using the object should now fail + // try { + // await kernel.queueMessageFromKernel(importerKRef, 'useImport', [ + // abandonObjectId, + // ]); + // // Should not reach here + // expect(false).toBe(true); + // } catch (error) { + // // We expect an error + // // eslint-disable-next-line vitest/no-conditional-expect + // expect(error).toBeDefined(); + // } }); }); diff --git a/packages/kernel-test/src/utils.ts b/packages/kernel-test/src/utils.ts index 24744a774..96bb399b1 100644 --- a/packages/kernel-test/src/utils.ts +++ b/packages/kernel-test/src/utils.ts @@ -135,3 +135,18 @@ export function extractVatLogs(buffer: string): string[] { .map((line: string) => line.slice(4)); return sortLogs(result); } + +/** + * Parse a message body into a JSON object. + * + * @param body - The message body to parse. + * + * @returns The parsed JSON object, or the original body if parsing fails. + */ +export function parseReplyBody(body: string): unknown { + try { + return JSON.parse(body.slice(1)); + } catch { + return body; + } +} diff --git a/packages/kernel-test/src/vats/exporter-vat.js b/packages/kernel-test/src/vats/exporter-vat.js index 1293d1307..2dc5cedc4 100644 --- a/packages/kernel-test/src/vats/exporter-vat.js +++ b/packages/kernel-test/src/vats/exporter-vat.js @@ -60,8 +60,25 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return obj; }, - getExportedObjectCount() { - return exportedObjects.size; + // Check if an object exists in our maps + isObjectPresent(objId) { + return exportedObjects.has(objId); + }, + + // Remove an object from our tracking, allowing it to be GC'd + forgetObject(objId) { + if (exportedObjects.has(objId)) { + tlog(`Forgetting object ${objId}`); + exportedObjects.delete(objId); + return true; + } + tlog(`Cannot forget nonexistent object: ${objId}`); + return false; + }, + + // No-op to help trigger crank cycles + noop() { + return 'noop'; }, // Create a weak reference to an object @@ -111,26 +128,5 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return { status: 'error', message: String(error) }; } }, - - // Check if an object exists in our maps - isObjectPresent(objId) { - return exportedObjects.has(objId); - }, - - // Remove an object from our tracking, allowing it to be GC'd - forgetObject(objId) { - if (exportedObjects.has(objId)) { - tlog(`Forgetting object ${objId}`); - exportedObjects.delete(objId); - return true; - } - tlog(`Cannot forget nonexistent object: ${objId}`); - return false; - }, - - // No-op to help trigger crank cycles - noop() { - return 'noop'; - }, }); } diff --git a/packages/kernel-test/src/vats/importer-vat.js b/packages/kernel-test/src/vats/importer-vat.js index b8bae5a02..6c0459332 100644 --- a/packages/kernel-test/src/vats/importer-vat.js +++ b/packages/kernel-test/src/vats/importer-vat.js @@ -11,14 +11,12 @@ import { Far } from '@endo/marshal'; */ export function buildRootObject(_vatPowers, parameters, _baggage) { const name = parameters?.name ?? 'anonymous'; - // const { getSyscall } = vatPowers; - // const syscall = getSyscall(); + + /** @type {WeakMap} */ + let weakMap = new WeakMap(); + + /** @type {Map} */ const importedObjects = new Map(); - const weakRefs = new Map(); - const finalizer = new FinalizationRegistry((id) => { - console.log(`::> ${name}: Imported object ${id} was finalized`); - weakRefs.delete(id); - }); /** * Print a message to the log. @@ -45,12 +43,27 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return this; }, + /** + * Store an imported object by ID, keeping a strong reference + * and a weak map entry for reverse lookup or GC observation. + * + * @param {object} obj - The imported object to store. + * @param {string} [id] - The ID to store the object under. + * @returns {string} The string 'stored'. + */ storeImport(obj, id = 'default') { tlog(`Storing import ${id}`, obj); importedObjects.set(id, obj); + weakMap.set(obj, id); return 'stored'; }, + /** + * Use the imported object by ID and call its method. + * + * @param {string} id - The ID of the object to use. + * @returns {*} The result of calling the object's method. + */ useImport(id = 'default') { tlog(`useImport ${id}`); const obj = importedObjects.get(id); @@ -61,6 +74,13 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return E(obj).getValue(); }, + /** + * Make the reference to an imported object weak by + * removing the strong reference, keeping only the weak one. + * + * @param {string} id - The ID of the object to make weak. + * @returns {boolean} True if the object was successfully made weak, false if it doesn't exist. + */ makeWeak(id) { const obj = importedObjects.get(id); if (!obj) { @@ -68,24 +88,21 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return false; } - tlog(`Making weak reference to ${id}`); - const ref = new WeakRef(obj); - weakRefs.set(id, ref); - finalizer.register(obj, id); - importedObjects.delete(id); + tlog(`Making weak reference to ${id} (dropping strong ref)`); + importedObjects.delete(id); // remove strong ref + // weakMap still holds a weak ref: GC can now drop it return true; }, - forgetImport(id) { - const had = importedObjects.has(id); - if (had) { - tlog(`Forgetting import ${id}`); - importedObjects.delete(id); - weakRefs.delete(id); - } else { - tlog(`Cannot forget nonexistent import: ${id}`); - } - return had; + /** + * Completely forget about the imported object. + * Once all vats forget it, retireImports() should trigger. + * + * @returns {boolean} True if the object was successfully forgotten, false if it doesn't exist. + */ + forgetImport() { + weakMap = new WeakMap(); + return true; }, getImportedObjectCount() { @@ -96,30 +113,6 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return Array.from(importedObjects.keys()); }, - checkAccess(id) { - if (importedObjects.has(id)) { - return { status: 'strong', accessible: true }; - } - - const weakRef = weakRefs.get(id); - if (!weakRef) { - return { status: 'no reference' }; - } - - const obj = weakRef.deref(); - if (!obj) { - return { status: 'collected' }; - } - - try { - const value = E(obj).getValue(); - return { status: 'weak', accessible: true, value }; - } catch (error) { - return { status: 'error', message: String(error) }; - } - }, - - // No-op to help trigger crank cycles noop() { return 'noop'; }, diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index ea169ed9d..c03e82229 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -321,6 +321,7 @@ export class VatHandle { * @param krefs - The KRefs of the imports to be dropped. */ #handleSyscallDropImports(krefs: KRef[]): void { + console.log('$$$ handleSyscallDropImports', krefs); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an import - meaning this vat received this object from somewhere else @@ -339,6 +340,7 @@ export class VatHandle { * @param krefs - The KRefs of the imports to be retired. */ #handleSyscallRetireImports(krefs: KRef[]): void { + console.log('$$$ handleSyscallRetireImports', krefs); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an import - meaning this vat received this object from somewhere else @@ -364,7 +366,12 @@ export class VatHandle { */ #handleSyscallExportCleanup(krefs: KRef[], checkReachable: boolean): void { const action = checkReachable ? 'retire' : 'abandon'; - + console.log( + '$$$ handleSyscallExportCleanup', + `${action}Export`, + krefs, + checkReachable, + ); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an export - meaning this vat created/owns this object diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 4415da79a..24b634899 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -18,7 +18,7 @@ import { makeSupervisorSyscall } from './services/syscall.ts'; import type { DispatchFn, MakeLiveSlotsFn, GCTools } from './services/types.ts'; import type { VatConfig, VatId, VRef } from './types.ts'; import { ROOT_OBJECT_VREF, isVatConfig } from './types.ts'; -import { gcAndFinalize } from './utils/gc-finalize.ts'; +import { makeGCAndFinalize } from './utils/gc-finalize.ts'; import type { VatKVStore } from './VatKVStore.ts'; import { makeVatKVStore } from './VatKVStore.ts'; @@ -230,7 +230,7 @@ export class VatSupervisor { WeakRef, FinalizationRegistry, waitUntilQuiescent, - gcAndFinalize, + gcAndFinalize: makeGCAndFinalize(), meterControl: makeDummyMeterControl(), }); diff --git a/packages/kernel/src/utils/gc-finalize.test.ts b/packages/kernel/src/utils/gc-finalize.test.ts index 26b659aa7..8df1088e8 100644 --- a/packages/kernel/src/utils/gc-finalize.test.ts +++ b/packages/kernel/src/utils/gc-finalize.test.ts @@ -1,7 +1,9 @@ import { delay } from '@ocap/utils'; import { describe, it, expect, vi } from 'vitest'; -import { gcAndFinalize, makeGCAndFinalize } from './gc-finalize.ts'; +import { makeGCAndFinalize } from './gc-finalize.ts'; + +const gcAndFinalize = makeGCAndFinalize(); describe('Garbage Collection', () => { it('should clean up unreachable objects', async () => { diff --git a/packages/kernel/src/utils/gc-finalize.ts b/packages/kernel/src/utils/gc-finalize.ts index 8d79afd24..45c3d696d 100644 --- a/packages/kernel/src/utils/gc-finalize.ts +++ b/packages/kernel/src/utils/gc-finalize.ts @@ -33,7 +33,7 @@ async function getGCFunction(): Promise { * * @returns A function that triggers GC and finalization when possible */ -function makeGCAndFinalize(): () => Promise { +export function makeGCAndFinalize(): () => Promise { // Cache the GC function promise const gcFunctionPromise = getGCFunction(); @@ -64,9 +64,3 @@ function makeGCAndFinalize(): () => Promise { } }; } - -// Create and export a singleton instance to be used throughout the codebase -export const gcAndFinalize = makeGCAndFinalize(); - -// Still export the factory function for testing or cases where a fresh instance is needed -export { makeGCAndFinalize }; From 77324f613013ff972004a0347c21070049623682 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 15:29:42 +0200 Subject: [PATCH 07/23] Process refCounts and collect garbage --- packages/kernel-test/package.json | 6 +- packages/kernel/src/Kernel.ts | 1 + packages/kernel/src/index.ts | 4 +- packages/kernel/src/store/index.ts | 6 + packages/kernel/src/store/methods/clist.ts | 6 +- packages/kernel/src/store/methods/gc.ts | 184 ++++++++++++------ .../src/store/methods/reachable.test.ts | 27 +++ .../kernel/src/store/methods/reachable.ts | 90 +++++++++ packages/kernel/src/store/methods/vat.ts | 54 ++++- packages/kernel/src/store/types.ts | 3 +- 10 files changed, 314 insertions(+), 67 deletions(-) create mode 100644 packages/kernel/src/store/methods/reachable.test.ts create mode 100644 packages/kernel/src/store/methods/reachable.ts diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index 150b77c58..5531fc58a 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -91,6 +91,10 @@ "@endo/marshal": "^1.6.4", "@endo/promise-kit": "^1.1.10", "@ocap/kernel": "workspace:^", - "@ocap/shims": "workspace:^" + "@ocap/nodejs": "workspace:^", + "@ocap/shims": "workspace:^", + "@ocap/store": "workspace:^", + "@ocap/streams": "workspace:^", + "@ocap/utils": "workspace:^" } } diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 1dc24dd71..8ad65661d 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -183,6 +183,7 @@ export class Kernel { for await (const item of this.#runQueueItems()) { console.log('*** run loop item', item); await this.#deliver(item); + this.#kernelStore.collectGarbage(); } } diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index c4b7cda47..364c69bc2 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -19,6 +19,6 @@ export { ClusterConfigStruct, } from './types.ts'; export { kunser, kser } from './services/kernel-marshal.ts'; -export { makeKernelStore } from './store/kernel-store.ts'; -export type { KernelStore } from './store/kernel-store.ts'; +export { makeKernelStore } from './store/index.ts'; +export type { KernelStore } from './store/index.ts'; export { parseRef } from './store/utils/parse-ref.ts'; diff --git a/packages/kernel/src/store/index.ts b/packages/kernel/src/store/index.ts index dea1e268d..e4ead1652 100644 --- a/packages/kernel/src/store/index.ts +++ b/packages/kernel/src/store/index.ts @@ -64,6 +64,7 @@ import { getIdMethods } from './methods/id.ts'; import { getObjectMethods } from './methods/object.ts'; import { getPromiseMethods } from './methods/promise.ts'; import { getQueueMethods } from './methods/queue.ts'; +import { getReachableMethods } from './methods/reachable.ts'; import { getRefCountMethods } from './methods/refcount.ts'; import { getVatMethods } from './methods/vat.ts'; import type { StoreContext } from './types.ts'; @@ -120,6 +121,8 @@ export function makeKernelStore(kdb: KernelDatabase) { // Garbage collection gcActions: provideCachedStoredValue('gcActions', '[]'), reapQueue: provideCachedStoredValue('reapQueue', '[]'), + // TODO: Store terminated vats in DB and fetch from there + terminatedVats: [], }; const id = getIdMethods(context); @@ -130,6 +133,7 @@ export function makeKernelStore(kdb: KernelDatabase) { const cList = getCListMethods(context); const queue = getQueueMethods(context); const vat = getVatMethods(context); + const reachable = getReachableMethods(context); /** * Create a new VatStore for a vat. @@ -159,6 +163,7 @@ export function makeKernelStore(kdb: KernelDatabase) { function reset(): void { kdb.clear(); context.maybeFreeKrefs.clear(); + context.terminatedVats = []; context.runQueue = provideStoredQueue('run', true); context.gcActions = provideCachedStoredValue('gcActions', '[]'); context.reapQueue = provideCachedStoredValue('reapQueue', '[]'); @@ -182,6 +187,7 @@ export function makeKernelStore(kdb: KernelDatabase) { ...object, ...promise, ...gc, + ...reachable, ...cList, ...vat, makeVatStore, diff --git a/packages/kernel/src/store/methods/clist.ts b/packages/kernel/src/store/methods/clist.ts index 6cab68ec0..efc752599 100644 --- a/packages/kernel/src/store/methods/clist.ts +++ b/packages/kernel/src/store/methods/clist.ts @@ -1,8 +1,8 @@ import { Fail } from '@endo/errors'; import { getBaseMethods } from './base.ts'; -import { getGCMethods } from './gc.ts'; import { getObjectMethods } from './object.ts'; +import { getReachableMethods } from './reachable.ts'; import { getRefCountMethods } from './refcount.ts'; import type { EndpointId, KRef, ERef } from '../../types.ts'; import type { StoreContext } from '../types.ts'; @@ -22,7 +22,7 @@ import { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getCListMethods(ctx: StoreContext) { const { getSlotKey } = getBaseMethods(ctx.kv); - const { clearReachableFlag } = getGCMethods(ctx); + const { clearReachableFlag } = getReachableMethods(ctx); const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); const { kernelRefExists, refCountKey } = getRefCountMethods(ctx); @@ -218,7 +218,7 @@ export function getCListMethods(ctx: StoreContext) { { isExport = false, onlyRecognizable = false, - }: { isExport?: boolean; onlyRecognizable?: boolean }, + }: { isExport?: boolean; onlyRecognizable?: boolean } = {}, ): boolean { kref || Fail`decrementRefCount called with empty kref`; diff --git a/packages/kernel/src/store/methods/gc.ts b/packages/kernel/src/store/methods/gc.ts index 3f2c6753f..efa7d85a4 100644 --- a/packages/kernel/src/store/methods/gc.ts +++ b/packages/kernel/src/store/methods/gc.ts @@ -1,9 +1,14 @@ +import { Fail } from '@endo/errors'; + import { getBaseMethods } from './base.ts'; +import { getCListMethods } from './clist.ts'; import { getObjectMethods } from './object.ts'; +import { getPromiseMethods } from './promise.ts'; +import { getReachableMethods } from './reachable.ts'; import { getRefCountMethods } from './refcount.ts'; +import { getVatMethods } from './vat.ts'; import type { VatId, - EndpointId, KRef, GCAction, RunQueueItemBringOutYourDead, @@ -14,12 +19,7 @@ import { RunQueueItemType, } from '../../types.ts'; import type { StoreContext } from '../types.ts'; -import { insistKernelType } from '../utils/kernel-slots.ts'; -import { parseRef } from '../utils/parse-ref.ts'; -import { - buildReachableAndVatSlot, - parseReachableAndVatSlot, -} from '../utils/reachable.ts'; +import { insistKernelType, parseKernelSlot } from '../utils/kernel-slots.ts'; /** * Create a store for garbage collection. @@ -30,9 +30,12 @@ import { // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getGCMethods(ctx: StoreContext) { const { getSlotKey } = getBaseMethods(ctx.kv); - const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); - const { kernelRefExists } = getRefCountMethods(ctx); - + const { getRefCount } = getRefCountMethods(ctx); + const { getObjectRefCount, deleteKernelObject } = getObjectMethods(ctx); + const { getKernelPromise, deleteKernelPromise } = getPromiseMethods(ctx); + const { decrementRefCount } = getCListMethods(ctx); + const { getImporters } = getVatMethods(ctx); + const { getReachableFlag, getReachableAndVatSlot } = getReachableMethods(ctx); /** * Get the set of GC actions to perform. * @@ -71,49 +74,6 @@ export function getGCMethods(ctx: StoreContext) { setGCActions(actions); } - /** - * Check if a kernel object is reachable. - * - * @param endpointId - The endpoint for which the reachable flag is being checked. - * @param kref - The kref. - * @returns True if the kernel object is reachable, false otherwise. - */ - function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { - const key = getSlotKey(endpointId, kref); - const data = ctx.kv.getRequired(key); - const { isReachable } = parseReachableAndVatSlot(data); - return isReachable; - } - - /** - * Clear the reachable flag for a given endpoint and kref. - * - * @param endpointId - The endpoint for which the reachable flag is being cleared. - * @param kref - The kref. - */ - function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { - const key = getSlotKey(endpointId, kref); - const { isReachable, vatSlot } = parseReachableAndVatSlot( - ctx.kv.getRequired(key), - ); - ctx.kv.set(key, buildReachableAndVatSlot(false, vatSlot)); - const { direction, isPromise } = parseRef(vatSlot); - // decrement 'reachable' part of refcount, but only for object imports - if ( - isReachable && - !isPromise && - direction === 'import' && - kernelRefExists(kref) - ) { - const counts = getObjectRefCount(kref); - counts.reachable -= 1; - setObjectRefCount(kref, counts); - if (counts.reachable === 0) { - ctx.maybeFreeKrefs.add(kref); - } - } - } - /** * Schedule a vat for reaping. * @@ -142,16 +102,124 @@ export function getGCMethods(ctx: StoreContext) { return undefined; } + /** + * Retires kernel objects by notifying importers and removing the objects. + * + * @param koids - Array of kernel object IDs to retire. + */ + function retireKernelObjects(koids: KRef[]): void { + Array.isArray(koids) || Fail`retireExports given non-Array ${koids}`; + const newActions: GCAction[] = []; + for (const koid of koids) { + const importers = getImporters(koid); + for (const vatID of importers) { + newActions.push(`${vatID} retireImport ${koid}`); + } + deleteKernelObject(koid); + } + addGCActions(newActions); + } + + /** + * Processes reference counts for kernel resources and performs garbage collection actions + * for resources that are no longer referenced or should be retired. + */ + function collectGarbage(): void { + const actions: Set = new Set(); + for (const kref of ctx.maybeFreeKrefs.values()) { + const { type } = parseKernelSlot(kref); + if (type === 'promise') { + const kpid = kref; + const kp = getKernelPromise(kpid); + const refCount = getRefCount(kpid); + if (refCount === 0) { + if (kp.state === 'fulfilled' || kp.state === 'rejected') { + // https://github.com/Agoric/agoric-sdk/issues/9888 don't assume promise is settled + for (const slot of kp.value?.slots ?? []) { + // Note: the following decrement can result in an addition to the + // maybeFreeKrefs set, which we are in the midst of iterating. + // TC39 went to a lot of trouble to ensure that this is kosher. + decrementRefCount(slot); + } + } + deleteKernelPromise(kpid); + } + } + + if (type === 'object') { + const { reachable, recognizable } = getObjectRefCount(kref); + if (reachable === 0) { + // We avoid ownerOfKernelObject(), which will report + // 'undefined' if the owner is dead (and being slowly + // deleted). Message delivery should use that, but not us. + const ownerKey = `${kref}.owner`; + let ownerVatID = ctx.kv.get(ownerKey); + const terminated = ctx.terminatedVats.includes(ownerVatID as VatId); + + // Some objects that are still owned, but the owning vat + // might still alive, or might be terminated and in the + // process of being deleted. These two clauses are + // mutually exclusive. + if (ownerVatID && !terminated) { + const vatConsidersReachable = getReachableFlag(ownerVatID, kref); + if (vatConsidersReachable) { + // the reachable count is zero, but the vat doesn't realize it + actions.add(`${ownerVatID} dropExport ${kref}`); + } + if (recognizable === 0) { + // TODO: rethink this assert + // assert.equal(vatConsidersReachable, false, `${kref} is reachable but not recognizable`); + actions.add(`${ownerVatID} retireExport ${kref}`); + } + } else if (ownerVatID && terminated) { + // When we're slowly deleting a vat, and one of its + // exports becomes unreferenced, we obviously must not + // send dropExports or retireExports into the dead vat. + // We fast-forward the abandonment that slow-deletion + // would have done, then treat the object as orphaned. + + const { vatSlot } = getReachableAndVatSlot(ownerVatID, kref); + // delete directly, not orphanKernelObject(), which + // would re-submit to maybeFreeKrefs + ctx.kv.delete(ownerKey); + ctx.kv.delete(getSlotKey(ownerVatID, kref)); + ctx.kv.delete(getSlotKey(ownerVatID, vatSlot)); + // now fall through to the orphaned case + ownerVatID = undefined; + } + + // Now handle objects which were orphaned. NOTE: this + // includes objects which were owned by a terminated (but + // not fully deleted) vat, where `ownerVatID` was cleared + // in the last line of that previous clause (the + // fall-through case). Don't try to change this `if + // (!ownerVatID)` into an `else if`: the two clauses are + // *not* mutually-exclusive. + if (!ownerVatID) { + // orphaned and unreachable, so retire it. If the kref + // is recognizable, then we need retireKernelObjects() + // to scan for importers and send retireImports (and + // delete), else we can call deleteKernelObject directly + if (recognizable) { + retireKernelObjects([kref]); + } else { + deleteKernelObject(kref); + } + } + } + } + } + addGCActions([...actions]); + ctx.maybeFreeKrefs.clear(); + } + return { - // GC actions getGCActions, setGCActions, addGCActions, - // Reachability tracking - getReachableFlag, - clearReachableFlag, - // Reaping scheduleReap, nextReapAction, + retireKernelObjects, + collectGarbage, }; } diff --git a/packages/kernel/src/store/methods/reachable.test.ts b/packages/kernel/src/store/methods/reachable.test.ts new file mode 100644 index 000000000..80ebb69c2 --- /dev/null +++ b/packages/kernel/src/store/methods/reachable.test.ts @@ -0,0 +1,27 @@ +import { describe, it, expect, beforeEach } from 'vitest'; + +import { makeMapKernelDatabase } from '../../../test/storage.ts'; +import { makeKernelStore } from '../index.ts'; + +describe('GC methods', () => { + let kernelStore: ReturnType; + + beforeEach(() => { + kernelStore = makeKernelStore(makeMapKernelDatabase()); + }); + + describe('reachability tracking', () => { + it('manages reachable flags', () => { + const ko1 = kernelStore.initKernelObject('v1'); + kernelStore.addClistEntry('v1', ko1, 'o-1'); + + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true); + + kernelStore.clearReachableFlag('v1', ko1); + expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false); + + const refCounts = kernelStore.getObjectRefCount(ko1); + expect(refCounts.reachable).toBe(0); + }); + }); +}); diff --git a/packages/kernel/src/store/methods/reachable.ts b/packages/kernel/src/store/methods/reachable.ts new file mode 100644 index 000000000..6a8a72714 --- /dev/null +++ b/packages/kernel/src/store/methods/reachable.ts @@ -0,0 +1,90 @@ +import type { EndpointId, KRef } from 'src/types'; + +import { getBaseMethods } from './base.ts'; +import { getObjectMethods } from './object.ts'; +import { getRefCountMethods } from './refcount.ts'; +import type { StoreContext } from '../types'; +import { parseRef } from '../utils/parse-ref.ts'; +import { + parseReachableAndVatSlot, + buildReachableAndVatSlot, +} from '../utils/reachable.ts'; + +/** + * Get the reachable methods that provide functionality for managing reachable flags. + * + * @param ctx - The store context. + * @returns The reachable methods. + */ +// eslint-disable-next-line @typescript-eslint/explicit-function-return-type +export function getReachableMethods(ctx: StoreContext) { + const { getSlotKey } = getBaseMethods(ctx.kv); + const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx); + const { kernelRefExists } = getRefCountMethods(ctx); + + /** + * Check if a kernel object is reachable. + * + * @param endpointId - The endpoint for which the reachable flag is being checked. + * @param kref - The kref. + * @returns True if the kernel object is reachable, false otherwise. + */ + function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean { + const key = getSlotKey(endpointId, kref); + const data = ctx.kv.getRequired(key); + const { isReachable } = parseReachableAndVatSlot(data); + return isReachable; + } + + /** + * Get the reachable and vat slot for a given vat and kernel slot. + * + * @param endpointId - The vat ID. + * @param kref - The kernel slot. + * @returns The reachable and vat slot. + */ + function getReachableAndVatSlot( + endpointId: EndpointId, + kref: KRef, + ): { + isReachable: boolean; + vatSlot: string; + } { + const key = getSlotKey(endpointId, kref); + const data = ctx.kv.getRequired(key); + return parseReachableAndVatSlot(data); + } + + /** + * Clear the reachable flag for a given endpoint and kref. + * + * @param endpointId - The endpoint for which the reachable flag is being cleared. + * @param kref - The kref. + */ + function clearReachableFlag(endpointId: EndpointId, kref: KRef): void { + const key = getSlotKey(endpointId, kref); + const { isReachable, vatSlot } = getReachableAndVatSlot(endpointId, kref); + ctx.kv.set(key, buildReachableAndVatSlot(false, vatSlot)); + const { direction, isPromise } = parseRef(vatSlot); + // decrement 'reachable' part of refcount, but only for object imports + if ( + isReachable && + !isPromise && + direction === 'import' && + kernelRefExists(kref) + ) { + const counts = getObjectRefCount(kref); + counts.reachable -= 1; + setObjectRefCount(kref, counts); + if (counts.reachable === 0) { + ctx.maybeFreeKrefs.add(kref); + } + } + } + + return { + getReachableFlag, + getReachableAndVatSlot, + clearReachableFlag, + }; +} diff --git a/packages/kernel/src/store/methods/vat.ts b/packages/kernel/src/store/methods/vat.ts index faae4787d..5db3e5c8c 100644 --- a/packages/kernel/src/store/methods/vat.ts +++ b/packages/kernel/src/store/methods/vat.ts @@ -1,6 +1,8 @@ import { getBaseMethods } from './base.ts'; -import type { EndpointId, VatConfig, VatId } from '../../types.ts'; +import type { EndpointId, KRef, VatConfig, VatId } from '../../types.ts'; import type { StoreContext } from '../types.ts'; +import { parseRef } from '../utils/parse-ref.ts'; +import { parseReachableAndVatSlot } from '../utils/reachable.ts'; type VatRecord = { vatID: VatId; @@ -19,7 +21,7 @@ const VAT_CONFIG_BASE_LEN = VAT_CONFIG_BASE.length; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getVatMethods(ctx: StoreContext) { const { kv } = ctx; - const { getPrefixedKeys } = getBaseMethods(ctx.kv); + const { getPrefixedKeys, getSlotKey } = getBaseMethods(ctx.kv); /** * Delete all persistent state associated with an endpoint. @@ -50,6 +52,17 @@ export function getVatMethods(ctx: StoreContext) { } } + /** + * Get all vat IDs from the store. + * + * @returns an array of vat IDs. + */ + function getVatIDs(): VatId[] { + return Array.from(getPrefixedKeys(VAT_CONFIG_BASE)).map((vatKey) => + vatKey.slice(VAT_CONFIG_BASE_LEN), + ); + } + /** * Fetch the stored configuration for a vat. * @@ -82,11 +95,48 @@ export function getVatMethods(ctx: StoreContext) { kv.delete(`${VAT_CONFIG_BASE}${vatID}`); } + /** + * Checks if a vat imports the specified kernel slot. + * + * @param vatID - The ID of the vat to check. + * @param kernelSlot - The kernel slot reference. + * @returns True if the vat imports the kernel slot, false otherwise. + */ + function importsKernelSlot(vatID: VatId, kernelSlot: KRef): boolean { + const data = ctx.kv.get(getSlotKey(vatID, kernelSlot)); + if (data) { + const { vatSlot } = parseReachableAndVatSlot(data); + const { direction } = parseRef(vatSlot); + if (direction === 'import') { + return true; + } + } + return false; + } + + /** + * Gets all vats that import a specific kernel object. + * + * @param koid - The kernel object ID. + * @returns An array of vat IDs that import the kernel object. + */ + function getImporters(koid: KRef): VatId[] { + const importers = []; + importers.push( + ...getVatIDs().filter((vatID) => importsKernelSlot(vatID, koid)), + ); + importers.sort(); + return importers; + } + return { deleteEndpoint, getAllVatRecords, getVatConfig, setVatConfig, deleteVatConfig, + getVatIDs, + importsKernelSlot, + getImporters, }; } diff --git a/packages/kernel/src/store/types.ts b/packages/kernel/src/store/types.ts index 3a7910a75..673130a92 100644 --- a/packages/kernel/src/store/types.ts +++ b/packages/kernel/src/store/types.ts @@ -1,6 +1,6 @@ import type { KVStore } from '@ocap/store'; -import type { KRef } from '../types.ts'; +import type { KRef, VatId } from '../types.ts'; export type StoreContext = { kv: KVStore; @@ -13,6 +13,7 @@ export type StoreContext = { maybeFreeKrefs: Set; gcActions: StoredValue; reapQueue: StoredValue; + terminatedVats: VatId[]; }; export type StoredValue = { From 770c4ba7bbfecf3c2a4e21301446164d2ec727dc Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:25:15 +0200 Subject: [PATCH 08/23] fix tests --- .../src/garbage-collection.test.ts | 136 +++--------------- packages/kernel-test/src/vats/exporter-vat.js | 60 +------- packages/kernel-test/src/vats/importer-vat.js | 24 ++-- .../src/vats/message-to-promise-vat.js | 1 - packages/kernel-test/vitest.config.ts | 1 + packages/kernel/package.json | 1 + packages/kernel/src/Kernel.ts | 31 ++-- packages/kernel/src/VatHandle.ts | 27 ++-- packages/kernel/src/VatSupervisor.ts | 3 - packages/kernel/src/index.test.ts | 4 + packages/kernel/src/services/meter-control.ts | 1 - packages/kernel/src/services/syscall.ts | 2 - packages/kernel/src/store/index.test.ts | 6 + packages/kernel/src/store/methods/vat.test.ts | 56 ++++++++ yarn.lock | 1 + 15 files changed, 139 insertions(+), 215 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index d0ca359f9..71bbe7764 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -4,7 +4,7 @@ import type { ClusterConfig, KRef, KernelStore, VatId } from '@ocap/kernel'; import type { KernelDatabase } from '@ocap/store'; import { makeSQLKernelDatabase } from '@ocap/store/sqlite/nodejs'; import { waitUntilQuiescent } from '@ocap/utils'; -import { expect, beforeEach, afterEach, describe, it } from 'vitest'; +import { expect, beforeEach, describe, it } from 'vitest'; import { getBundleSpec, @@ -68,10 +68,6 @@ describe('Garbage Collection E2E Tests', () => { importerKRef = kernelStore.erefToKref(importerVatId, 'o+0') as KRef; }); - afterEach(async () => { - console.log('$$$ DB', kernelDatabase.executeQuery('SELECT * FROM kv')); - }); - it('objects are tracked with reference counts', async () => { const objectId = 'test-object'; // Create an object in the exporter vat @@ -112,7 +108,7 @@ describe('Garbage Collection E2E Tests', () => { }); it('should trigger GC syscalls through bringOutYourDead', async () => { - // 1. Create an object in the exporter vat with a known ID + // Create an object in the exporter vat with a known ID const objectId = 'test-object'; const createObjectData = await kernel.queueMessageFromKernel( exporterKRef, @@ -123,11 +119,10 @@ describe('Garbage Collection E2E Tests', () => { // Store initial reference count information const initialRefCounts = kernelStore.getObjectRefCount(createObjectRef); - console.log('Initial ref counts:', createObjectRef, initialRefCounts); expect(initialRefCounts.reachable).toBe(1); expect(initialRefCounts.recognizable).toBe(1); - // 2. Store the reference in the importer vat + // Store the reference in the importer vat const objectRef = kunser(createObjectData); await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ objectRef, @@ -135,17 +130,12 @@ describe('Garbage Collection E2E Tests', () => { ]); await waitUntilQuiescent(); - // Get reference counts after storing in importer - const afterStoreRefCounts = kernelStore.getObjectRefCount(createObjectRef); - console.log('After store ref counts:', objectRef, afterStoreRefCounts); - - // 3. Verify object is tracked in both vats + // Verify object is tracked in both vats const importerHasObject = await kernel.queueMessageFromKernel( importerKRef, 'listImportedObjects', [], ); - console.log('$$$ importerHasObject', importerHasObject); expect(parseReplyBody(importerHasObject.body)).toContain(objectId); const exporterHasObject = await kernel.queueMessageFromKernel( @@ -153,40 +143,34 @@ describe('Garbage Collection E2E Tests', () => { 'isObjectPresent', [objectId], ); - console.log('$$$ exporterHasObject', exporterHasObject); expect(parseReplyBody(exporterHasObject.body)).toBe(true); - // 4. Make a weak reference to the object in the importer vat + // Make a weak reference to the object in the importer vat // This should eventually trigger dropImports when GC runs await kernel.queueMessageFromKernel(importerKRef, 'makeWeak', [objectId]); await waitUntilQuiescent(); - console.log('$$$ kernelStore.getGCActions(1)', kernelStore.getGCActions()); - - // // 5. Schedule reap to trigger bringOutYourDead on next crank - kernel.reapAllVats(); - - // // 6. Run a crank to allow bringOutYourDead to be processed - await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); - await waitUntilQuiescent(100); + // Schedule reap to trigger bringOutYourDead on next crank + kernel.reapVats((vatId) => vatId === importerVatId); - console.log('$$$ kernelStore.getGCActions(2)', kernelStore.getGCActions()); + // Run a crank to allow bringOutYourDead to be processed + await kernel.queueMessageFromKernel(importerKRef, 'noop', []); + await waitUntilQuiescent(); - // // Check reference counts after dropImports + // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); - console.log('After weak ref counts:', afterWeakRefCounts); expect(afterWeakRefCounts.reachable).toBe(0); expect(afterWeakRefCounts.recognizable).toBe(1); - // 7. Now completely forget the import in the importer vat + // Now completely forget the import in the importer vat // This should trigger retireImports when GC runs await kernel.queueMessageFromKernel(importerKRef, 'forgetImport', []); await waitUntilQuiescent(); - // 8. Schedule another reap - kernel.reapAllVats(); + // Schedule another reap + kernel.reapVats((vatId) => vatId === importerVatId); - // 9. Run a crank to allow bringOutYourDead to be processed + // Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(importerKRef, 'noop', []); await waitUntilQuiescent(100); @@ -195,21 +179,19 @@ describe('Garbage Collection E2E Tests', () => { expect(afterForgetRefCounts.reachable).toBe(0); expect(afterForgetRefCounts.recognizable).toBe(0); - // // 10. Now forget the object in the exporter vat - // // This should trigger retireExports when GC runs + // Now forget the object in the exporter vat + // This should trigger retireExports when GC runs await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ objectId, ]); await waitUntilQuiescent(); - // 11. Schedule a final reap - kernel.reapAllVats(); + // Schedule a final reap + kernel.reapVats((vatId) => vatId === exporterVatId); - // 12. Run multiple cranks to ensure GC completes - for (let i = 0; i < 3; i++) { - await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); - await waitUntilQuiescent(50); - } + // Run a crank to ensure GC completes + await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); + await waitUntilQuiescent(50); // Verify the object has been completely removed const exporterFinalCheck = await kernel.queueMessageFromKernel( @@ -217,80 +199,6 @@ describe('Garbage Collection E2E Tests', () => { 'isObjectPresent', [objectId], ); - console.log('$$$ exporterFinalCheck', exporterFinalCheck); expect(parseReplyBody(exporterFinalCheck.body)).toBe(false); - - // Check if reference still exists in the kernel store at all - // const refExists = kernelStore.kernelRefExists(createObjectRef); - // expect(refExists).toBe(false); - - // // 13. Test abandonExports by creating a new object and forcing its removal - // const abandonObjectId = 'abandon-test'; - // const abandonObjData = await kernel.queueMessageFromKernel( - // exporterKRef, - // 'createObject', - // [abandonObjectId], - // ); - // console.log('$$$ abandonObjData', abandonObjData); - // const abandonObjRef = abandonObjData.slots[0] as KRef; - - // // Store in importer to make it reachable from both vats - // await kernel.queueMessageFromKernel(importerKRef, 'storeImport', [ - // abandonObjRef, - // abandonObjectId, - // ]); - // await waitUntilQuiescent(); - - // // Verify it's reachable from both vats - // const abandonRefCounts = kernelStore.getObjectRefCount(abandonObjRef); - // expect(abandonRefCounts.reachable).toBe(1); - - // // Force remove in exporter (this simulates abandonExports) - // await kernel.queueMessageFromKernel(exporterKRef, 'forgetObject', [ - // abandonObjectId, - // ]); - // await waitUntilQuiescent(); - - // // Schedule reap to trigger abandonExports - // kernelStore.scheduleReap(exporterVatId); - - // // Run multiple cranks to ensure GC completes - // for (let i = 0; i < 3; i++) { - // await kernel.queueMessageFromKernel(exporterKRef, 'noop', []); - // await waitUntilQuiescent(50); - // } - - // // Verify object is gone from exporter - // const exporterAbandonCheck = await kernel.queueMessageFromKernel( - // exporterKRef, - // 'isObjectPresent', - // [abandonObjectId], - // ); - // console.log('$$$ exporterAbandonCheck', exporterAbandonCheck); - // expect(parseReplyBody(exporterAbandonCheck.body)).toBe(false); - - // // But it should still be in the importer's list - // const importerAbandonCheck = await kernel.queueMessageFromKernel( - // importerKRef, - // 'listImportedObjects', - // [], - // ); - // console.log('$$$ importerAbandonCheck', importerAbandonCheck); - // expect(parseReplyBody(importerAbandonCheck.body)).toContain( - // abandonObjectId, - // ); - - // // However, using the object should now fail - // try { - // await kernel.queueMessageFromKernel(importerKRef, 'useImport', [ - // abandonObjectId, - // ]); - // // Should not reach here - // expect(false).toBe(true); - // } catch (error) { - // // We expect an error - // // eslint-disable-next-line vitest/no-conditional-expect - // expect(error).toBeDefined(); - // } }); }); diff --git a/packages/kernel-test/src/vats/exporter-vat.js b/packages/kernel-test/src/vats/exporter-vat.js index 2dc5cedc4..959d4148e 100644 --- a/packages/kernel-test/src/vats/exporter-vat.js +++ b/packages/kernel-test/src/vats/exporter-vat.js @@ -1,4 +1,3 @@ -// import { E } from '@endo/eventual-send'; import { Far } from '@endo/marshal'; /** @@ -33,22 +32,15 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { console.log(`::> ${name}: ${message}`, ...args); } - // Track objects we've created const exportedObjects = new Map(); - // Track WeakRefs - const weakRefs = new Map(); - // FinalizationRegistry to know when objects are GC'd - const finalizer = new FinalizationRegistry((id) => { - console.log(`::> ${name}: Object ${id} was finalized`); - weakRefs.delete(id); - }); return Far('root', { bootstrap() { log(`bootstrap`); - return this; + return `bootstrap-${name}`; }, + // Create an object in our maps createObject(id) { const obj = Far('SharedObject', { getValue() { @@ -80,53 +72,5 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { noop() { return 'noop'; }, - - // Create a weak reference to an object - makeWeakRef(objId) { - const obj = exportedObjects.get(objId); - if (obj) { - tlog(`Creating weak reference to object ${objId}`); - const ref = new WeakRef(obj); - weakRefs.set(objId, ref); - finalizer.register(obj, objId); - return true; - } - return false; - }, - - // Remove the strong reference, keeping only the weak reference - removeStrongRef(objId) { - if (this.makeWeakRef(objId)) { - tlog(`Removing strong reference to object ${objId}`); - exportedObjects.delete(objId); - return true; - } - return false; - }, - - // Check if an object is still held strongly - isStronglyHeld(objId) { - return exportedObjects.has(objId); - }, - - // Try to access a potentially GC'd object through its weak reference - tryAccessWeakRef(objId) { - const weakRef = weakRefs.get(objId); - if (!weakRef) { - return { status: 'no weak ref' }; - } - - const obj = weakRef.deref(); - if (!obj) { - return { status: 'collected' }; - } - - try { - const value = obj.getValue(); - return { status: 'alive', value }; - } catch (error) { - return { status: 'error', message: String(error) }; - } - }, }); } diff --git a/packages/kernel-test/src/vats/importer-vat.js b/packages/kernel-test/src/vats/importer-vat.js index 6c0459332..b4e573d99 100644 --- a/packages/kernel-test/src/vats/importer-vat.js +++ b/packages/kernel-test/src/vats/importer-vat.js @@ -40,12 +40,12 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return Far('root', { bootstrap() { log(`bootstrap`); - return this; + return `bootstrap-${name}`; }, /** - * Store an imported object by ID, keeping a strong reference - * and a weak map entry for reverse lookup or GC observation. + * Store an imported object by ID + * keeping a strong reference and a weak map entry. * * @param {object} obj - The imported object to store. * @param {string} [id] - The ID to store the object under. @@ -87,10 +87,8 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { tlog(`Cannot make weak reference to nonexistent object: ${id}`); return false; } - tlog(`Making weak reference to ${id} (dropping strong ref)`); - importedObjects.delete(id); // remove strong ref - // weakMap still holds a weak ref: GC can now drop it + importedObjects.delete(id); return true; }, @@ -105,14 +103,20 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { return true; }, - getImportedObjectCount() { - return importedObjects.size; - }, - + /** + * List all imported objects. + * + * @returns {string[]} An array of all imported object IDs. + */ listImportedObjects() { return Array.from(importedObjects.keys()); }, + /** + * No-op method. + * + * @returns {string} The string 'noop'. + */ noop() { return 'noop'; }, diff --git a/packages/kernel-test/src/vats/message-to-promise-vat.js b/packages/kernel-test/src/vats/message-to-promise-vat.js index 17e503b7d..17801261d 100644 --- a/packages/kernel-test/src/vats/message-to-promise-vat.js +++ b/packages/kernel-test/src/vats/message-to-promise-vat.js @@ -47,7 +47,6 @@ export function buildRootObject(_vatPowers, parameters, _baggage) { async bootstrap(vats) { log(`bootstrap start`); tlog(`running test ${test}`); - console.log('$$$ vats', vats); const promise1 = E(vats.bob).setup(); const promise2 = E(promise1).doSomething(); const doneP = promise2.then( diff --git a/packages/kernel-test/vitest.config.ts b/packages/kernel-test/vitest.config.ts index 1f60917fc..d7ea99af1 100644 --- a/packages/kernel-test/vitest.config.ts +++ b/packages/kernel-test/vitest.config.ts @@ -8,6 +8,7 @@ const config = mergeConfig( test: { name: 'kernel-test', pool: 'forks', + exclude: ['./src/utils.ts'], }, }), ); diff --git a/packages/kernel/package.json b/packages/kernel/package.json index bc1396324..2e553e09e 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -47,6 +47,7 @@ "@endo/far": "^1.1.11", "@endo/import-bundle": "^1.4.0", "@endo/marshal": "^1.6.4", + "@endo/nat": "^5.1.0", "@endo/pass-style": "^1.5.0", "@endo/promise-kit": "^1.1.10", "@metamask/superstruct": "^3.2.0", diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 8ad65661d..57363cfdc 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -181,7 +181,6 @@ export class Kernel { */ async #run(): Promise { for await (const item of this.#runQueueItems()) { - console.log('*** run loop item', item); await this.#deliver(item); this.#kernelStore.collectGarbage(); } @@ -597,7 +596,11 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} dropExports`, krefs); const vat = this.#getVat(vatId); - await vat.deliverDropExports(krefs); + const vrefs: VRef[] = krefs + .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) + .filter((ref): ref is VRef => typeof ref === 'string'); + log(`@@@@ deliver ${vatId} dropExports`, vrefs); + await vat.deliverDropExports(vrefs); log(`@@@@ done ${vatId} dropExports`, krefs); break; } @@ -605,7 +608,11 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} retireExports`, krefs); const vat = this.#getVat(vatId); - await vat.deliverRetireExports(krefs); + const vrefs: VRef[] = krefs + .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) + .filter((ref): ref is VRef => typeof ref === 'string'); + log(`@@@@ deliver ${vatId} retireExports`, vrefs); + await vat.deliverRetireExports(vrefs); log(`@@@@ done ${vatId} retireExports`, krefs); break; } @@ -613,7 +620,11 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} retireImports`, krefs); const vat = this.#getVat(vatId); - await vat.deliverRetireImports(krefs); + const vrefs: VRef[] = krefs + .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) + .filter((ref): ref is VRef => typeof ref === 'string'); + log(`@@@@ deliver ${vatId} retireImports`, vrefs); + await vat.deliverRetireImports(vrefs); log(`@@@@ done ${vatId} retireImports`, krefs); break; } @@ -926,12 +937,16 @@ export class Kernel { } /** - * Reap all vats. + * Reap vats that match the filter. + * + * @param filter - A function that returns true if the vat should be reaped. */ - reapAllVats(): void { + reapVats(filter: (vatId: VatId) => boolean = () => true): void { for (const vatID of this.getVatIds()) { - this.#kernelStore.scheduleReap(vatID); + if (filter(vatID)) { + this.#kernelStore.scheduleReap(vatID); + } } } } -harden(Kernel); // XXX restore this once vitest is able to cope +// harden(Kernel); // XXX restore this once vitest is able to cope diff --git a/packages/kernel/src/VatHandle.ts b/packages/kernel/src/VatHandle.ts index c03e82229..8bbeb244b 100644 --- a/packages/kernel/src/VatHandle.ts +++ b/packages/kernel/src/VatHandle.ts @@ -321,7 +321,6 @@ export class VatHandle { * @param krefs - The KRefs of the imports to be dropped. */ #handleSyscallDropImports(krefs: KRef[]): void { - console.log('$$$ handleSyscallDropImports', krefs); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an import - meaning this vat received this object from somewhere else @@ -340,7 +339,6 @@ export class VatHandle { * @param krefs - The KRefs of the imports to be retired. */ #handleSyscallRetireImports(krefs: KRef[]): void { - console.log('$$$ handleSyscallRetireImports', krefs); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an import - meaning this vat received this object from somewhere else @@ -366,12 +364,6 @@ export class VatHandle { */ #handleSyscallExportCleanup(krefs: KRef[], checkReachable: boolean): void { const action = checkReachable ? 'retire' : 'abandon'; - console.log( - '$$$ handleSyscallExportCleanup', - `${action}Export`, - krefs, - checkReachable, - ); for (const kref of krefs) { const { direction, isPromise } = parseRef(kref); // We validate it's an export - meaning this vat created/owns this object @@ -537,36 +529,36 @@ export class VatHandle { /** * Make a 'dropExports' delivery to the vat. * - * @param krefs - The KRefs of the exports to be dropped. + * @param vrefs - The VRefs of the exports to be dropped. */ - async deliverDropExports(krefs: KRef[]): Promise { + async deliverDropExports(vrefs: VRef[]): Promise { await this.sendVatCommand({ method: VatCommandMethod.deliver, - params: ['dropExports', krefs], + params: ['dropExports', vrefs], }); } /** * Make a 'retireExports' delivery to the vat. * - * @param krefs - The KRefs of the exports to be retired. + * @param vrefs - The VRefs of the exports to be retired. */ - async deliverRetireExports(krefs: KRef[]): Promise { + async deliverRetireExports(vrefs: VRef[]): Promise { await this.sendVatCommand({ method: VatCommandMethod.deliver, - params: ['retireExports', krefs], + params: ['retireExports', vrefs], }); } /** * Make a 'retireImports' delivery to the vat. * - * @param krefs - The KRefs of the imports to be retired. + * @param vrefs - The VRefs of the imports to be retired. */ - async deliverRetireImports(krefs: KRef[]): Promise { + async deliverRetireImports(vrefs: VRef[]): Promise { await this.sendVatCommand({ method: VatCommandMethod.deliver, - params: ['retireImports', krefs], + params: ['retireImports', vrefs], }); } @@ -574,7 +566,6 @@ export class VatHandle { * Make a 'bringOutYourDead' delivery to the vat. */ async deliverBringOutYourDead(): Promise { - console.log('*** deliverBringOutYourDead'); await this.sendVatCommand({ method: VatCommandMethod.deliver, params: ['bringOutYourDead'], diff --git a/packages/kernel/src/VatSupervisor.ts b/packages/kernel/src/VatSupervisor.ts index 24b634899..5b0edf136 100644 --- a/packages/kernel/src/VatSupervisor.ts +++ b/packages/kernel/src/VatSupervisor.ts @@ -120,7 +120,6 @@ export class VatSupervisor { console.error(`cannot deliver before vat is loaded`); return; } - console.log('*** handleMessage deliver', payload.params); await this.#dispatch(harden(payload.params) as VatDeliveryObject); await Promise.all(this.#syscallsInFlight); this.#syscallsInFlight.length = 0; @@ -237,8 +236,6 @@ export class VatSupervisor { const workerEndowments = { console, assert: globalThis.assert, - WeakRef, - FinalizationRegistry, }; const { bundleSpec, parameters } = vatConfig; diff --git a/packages/kernel/src/index.test.ts b/packages/kernel/src/index.test.ts index 279072fc9..e55ac1253 100644 --- a/packages/kernel/src/index.test.ts +++ b/packages/kernel/src/index.test.ts @@ -24,6 +24,10 @@ describe('index', () => { 'isVatId', 'isVatWorkerServiceCommand', 'isVatWorkerServiceReply', + 'kser', + 'kunser', + 'makeKernelStore', + 'parseRef', ]); }); }); diff --git a/packages/kernel/src/services/meter-control.ts b/packages/kernel/src/services/meter-control.ts index 97b1c738c..dc997e46f 100644 --- a/packages/kernel/src/services/meter-control.ts +++ b/packages/kernel/src/services/meter-control.ts @@ -63,7 +63,6 @@ export function makeDummyMeterControl(): MeterControl { async function runWithoutMeteringAsync( thunk: () => unknown, ): Promise { - console.log('*** runWithoutMeteringAsync'); meteringDisabled += 1; return Promise.resolve() .then(() => thunk()) diff --git a/packages/kernel/src/services/syscall.ts b/packages/kernel/src/services/syscall.ts index 18d4b22ba..ede62bcf8 100644 --- a/packages/kernel/src/services/syscall.ts +++ b/packages/kernel/src/services/syscall.ts @@ -37,12 +37,10 @@ function makeSupervisorSyscall( * @returns the result from performing the syscall. */ function doSyscall(vso: VatSyscallObject): SyscallResult { - console.log('$$$ VAT doSyscall', vso); insistVatSyscallObject(vso); let syscallResult; try { syscallResult = supervisor.executeSyscall(vso); - console.log('$$$ VAT syscallResult', syscallResult); } catch (problem) { console.warn(`supervisor got error during syscall:`, problem); throw problem; diff --git a/packages/kernel/src/store/index.test.ts b/packages/kernel/src/store/index.test.ts index ba7587c1b..852af38a2 100644 --- a/packages/kernel/src/store/index.test.ts +++ b/packages/kernel/src/store/index.test.ts @@ -59,6 +59,7 @@ describe('kernel store', () => { 'allocateErefForKref', 'clear', 'clearReachableFlag', + 'collectGarbage', 'decRefCount', 'decrementRefCount', 'deleteClistEntry', @@ -75,6 +76,7 @@ describe('kernel store', () => { 'forgetKref', 'getAllVatRecords', 'getGCActions', + 'getImporters', 'getKernelPromise', 'getKernelPromiseMessageQueue', 'getNextObjectId', @@ -85,10 +87,13 @@ describe('kernel store', () => { 'getOwner', 'getPromisesByDecider', 'getQueueLength', + 'getReachableAndVatSlot', 'getReachableFlag', 'getRefCount', 'getVatConfig', + 'getVatIDs', 'hasCListEntry', + 'importsKernelSlot', 'incRefCount', 'incrementRefCount', 'initEndpoint', @@ -102,6 +107,7 @@ describe('kernel store', () => { 'refCountKey', 'reset', 'resolveKernelPromise', + 'retireKernelObjects', 'runQueueLength', 'scheduleReap', 'setGCActions', diff --git a/packages/kernel/src/store/methods/vat.test.ts b/packages/kernel/src/store/methods/vat.test.ts index a462f3a4f..40f16cce6 100644 --- a/packages/kernel/src/store/methods/vat.test.ts +++ b/packages/kernel/src/store/methods/vat.test.ts @@ -186,4 +186,60 @@ describe('vat store methods', () => { expect(mockGetPrefixedKeys).toHaveBeenCalledWith(`clk.${endpointId}.`); }); }); + + describe('getVatIDs', () => { + it('returns all vat IDs from storage', () => { + mockGetPrefixedKeys.mockReturnValue([ + `vatConfig.${vatID1}`, + `vatConfig.${vatID2}`, + `vatConfig.v3`, + ]); + + const result = vatMethods.getVatIDs(); + + expect(result).toStrictEqual([vatID1, vatID2, 'v3']); + expect(mockGetPrefixedKeys).toHaveBeenCalledWith('vatConfig.'); + }); + + it('returns an empty array when no vats are configured', () => { + mockGetPrefixedKeys.mockReturnValue([]); + + const result = vatMethods.getVatIDs(); + + expect(result).toStrictEqual([]); + expect(mockGetPrefixedKeys).toHaveBeenCalledWith('vatConfig.'); + }); + }); + + describe('getImporters', () => { + it('handles case with no vats', () => { + const kernelObject = 'ko123'; + + // Mock empty array of vat IDs + mockGetPrefixedKeys.mockImplementation((prefix) => { + if (prefix === 'vatConfig.') { + return []; + } + return []; + }); + + // This shouldn't be called + const mockImportsKernelSlot = vi.fn(); + + // Replace the importsKernelSlot method for this test + const originalImportsKernelSlot = vatMethods.importsKernelSlot; + vatMethods.importsKernelSlot = mockImportsKernelSlot; + + try { + const result = vatMethods.getImporters(kernelObject); + + expect(result).toStrictEqual([]); + expect(mockGetPrefixedKeys).toHaveBeenCalledWith('vatConfig.'); + expect(mockImportsKernelSlot).not.toHaveBeenCalled(); + } finally { + // Restore original method + vatMethods.importsKernelSlot = originalImportsKernelSlot; + } + }); + }); }); diff --git a/yarn.lock b/yarn.lock index 7e0291507..370e55a1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2131,6 +2131,7 @@ __metadata: "@endo/far": "npm:^1.1.11" "@endo/import-bundle": "npm:^1.4.0" "@endo/marshal": "npm:^1.6.4" + "@endo/nat": "npm:^5.1.0" "@endo/pass-style": "npm:^1.5.0" "@endo/promise-kit": "npm:^1.1.10" "@metamask/auto-changelog": "npm:^5.0.1" From 96975d66bfdcd03b958be295c4df6900f92e06cd Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:26:35 +0200 Subject: [PATCH 09/23] fix other tests --- packages/kernel-test/src/liveslots.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel-test/src/liveslots.test.ts b/packages/kernel-test/src/liveslots.test.ts index 4de20d205..a02ec357f 100644 --- a/packages/kernel-test/src/liveslots.test.ts +++ b/packages/kernel-test/src/liveslots.test.ts @@ -279,7 +279,7 @@ describe('liveslots promise handling', () => { expect(vatLogs).toStrictEqual(reference); }, 30000); - it.only('messageToPromise: send to promise before resolution', async () => { + it('messageToPromise: send to promise before resolution', async () => { const [bootstrapResult, vatLogs] = await runTestVats( 'message-to-promise-vat', 'messageToPromise', From 18a9e6f7c553555ade15c200b3021ded1e880c80 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:31:18 +0200 Subject: [PATCH 10/23] fix test config --- packages/kernel-test/src/garbage-collection.test.ts | 3 +-- packages/kernel-test/vitest.config.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 71bbe7764..eac9ae022 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -22,7 +22,6 @@ function makeTestSubcluster(): ClusterConfig { return { bootstrap: 'exporter', forceReset: true, - bundles: null, vats: { exporter: { bundleSpec: getBundleSpec('exporter-vat'), @@ -40,7 +39,7 @@ function makeTestSubcluster(): ClusterConfig { }; } -describe('Garbage Collection E2E Tests', () => { +describe('Garbage Collection', () => { let kernel: Kernel; let kernelDatabase: KernelDatabase; let kernelStore: KernelStore; diff --git a/packages/kernel-test/vitest.config.ts b/packages/kernel-test/vitest.config.ts index d7ea99af1..1f60917fc 100644 --- a/packages/kernel-test/vitest.config.ts +++ b/packages/kernel-test/vitest.config.ts @@ -8,7 +8,6 @@ const config = mergeConfig( test: { name: 'kernel-test', pool: 'forks', - exclude: ['./src/utils.ts'], }, }), ); From 9120a313dd3bea808d9b9df3fbd730e3ff4ccee6 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:44:12 +0200 Subject: [PATCH 11/23] fix lint --- packages/kernel/package.json | 10 ++++++++++ packages/kernel/src/store/methods/reachable.ts | 5 ++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 2e553e09e..ba82580c2 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -42,6 +42,7 @@ "test:watch": "vitest --config vitest.config.ts" }, "dependencies": { +<<<<<<< HEAD "@agoric/swingset-liveslots": "0.10.3-u19.2", "@endo/errors": "^1.2.10", "@endo/far": "^1.1.11", @@ -50,6 +51,15 @@ "@endo/nat": "^5.1.0", "@endo/pass-style": "^1.5.0", "@endo/promise-kit": "^1.1.10", +======= + "@agoric/swingset-liveslots": "0.10.3-dev-3d64bee.0", + "@endo/errors": "^1.2.8", + "@endo/far": "^1.1.9", + "@endo/import-bundle": "^1.3.1", + "@endo/marshal": "^1.6.2", + "@endo/pass-style": "^1.4.7", + "@endo/promise-kit": "^1.1.6", +>>>>>>> 538c21e6 (fix lint) "@metamask/superstruct": "^3.2.0", "@metamask/utils": "^11.4.0", "@ocap/errors": "workspace:^", diff --git a/packages/kernel/src/store/methods/reachable.ts b/packages/kernel/src/store/methods/reachable.ts index 6a8a72714..fc5c9edd3 100644 --- a/packages/kernel/src/store/methods/reachable.ts +++ b/packages/kernel/src/store/methods/reachable.ts @@ -1,9 +1,8 @@ -import type { EndpointId, KRef } from 'src/types'; - import { getBaseMethods } from './base.ts'; import { getObjectMethods } from './object.ts'; import { getRefCountMethods } from './refcount.ts'; -import type { StoreContext } from '../types'; +import type { EndpointId, KRef } from '../../types.ts'; +import type { StoreContext } from '../types.ts'; import { parseRef } from '../utils/parse-ref.ts'; import { parseReachableAndVatSlot, From 03d9935d39aadb020e5be2d3b57cbfcc44cb31ef Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:45:53 +0200 Subject: [PATCH 12/23] lockfile --- packages/kernel/package.json | 10 ---- yarn.lock | 98 +++--------------------------------- 2 files changed, 7 insertions(+), 101 deletions(-) diff --git a/packages/kernel/package.json b/packages/kernel/package.json index ba82580c2..2e553e09e 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -42,7 +42,6 @@ "test:watch": "vitest --config vitest.config.ts" }, "dependencies": { -<<<<<<< HEAD "@agoric/swingset-liveslots": "0.10.3-u19.2", "@endo/errors": "^1.2.10", "@endo/far": "^1.1.11", @@ -51,15 +50,6 @@ "@endo/nat": "^5.1.0", "@endo/pass-style": "^1.5.0", "@endo/promise-kit": "^1.1.10", -======= - "@agoric/swingset-liveslots": "0.10.3-dev-3d64bee.0", - "@endo/errors": "^1.2.8", - "@endo/far": "^1.1.9", - "@endo/import-bundle": "^1.3.1", - "@endo/marshal": "^1.6.2", - "@endo/pass-style": "^1.4.7", - "@endo/promise-kit": "^1.1.6", ->>>>>>> 538c21e6 (fix lint) "@metamask/superstruct": "^3.2.0", "@metamask/utils": "^11.4.0", "@ocap/errors": "workspace:^", diff --git a/yarn.lock b/yarn.lock index 370e55a1e..d335a8f75 100644 --- a/yarn.lock +++ b/yarn.lock @@ -240,7 +240,7 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.25.9, @babel/generator@npm:^7.26.3": +"@babel/generator@npm:^7.25.9, @babel/generator@npm:^7.26.3, @babel/generator@npm:^7.26.9": version: 7.27.0 resolution: "@babel/generator@npm:7.27.0" dependencies: @@ -253,19 +253,6 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.26.9": - version: 7.26.9 - resolution: "@babel/generator@npm:7.26.9" - dependencies: - "@babel/parser": "npm:^7.26.9" - "@babel/types": "npm:^7.26.9" - "@jridgewell/gen-mapping": "npm:^0.3.5" - "@jridgewell/trace-mapping": "npm:^0.3.25" - jsesc: "npm:^3.0.2" - checksum: 10/95075dd6158a49efcc71d7f2c5d20194fcf245348de7723ca35e37cd5800587f1d4de2be6c4ba87b5f5fbb967c052543c109eaab14b43f6a73eb05ccd9a5bb44 - languageName: node - linkType: hard - "@babel/helper-compilation-targets@npm:^7.26.5": version: 7.26.5 resolution: "@babel/helper-compilation-targets@npm:7.26.5" @@ -393,7 +380,7 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.25.9": +"@babel/template@npm:^7.25.9, @babel/template@npm:^7.26.9": version: 7.27.0 resolution: "@babel/template@npm:7.27.0" dependencies: @@ -404,17 +391,6 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.26.9": - version: 7.26.9 - resolution: "@babel/template@npm:7.26.9" - dependencies: - "@babel/code-frame": "npm:^7.26.2" - "@babel/parser": "npm:^7.26.9" - "@babel/types": "npm:^7.26.9" - checksum: 10/240288cebac95b1cc1cb045ad143365643da0470e905e11731e63280e43480785bd259924f4aea83898ef68e9fa7c176f5f2d1e8b0a059b27966e8ca0b41a1b6 - languageName: node - linkType: hard - "@babel/traverse@npm:^7.23.2, @babel/traverse@npm:^7.25.9, @babel/traverse@npm:^7.26.9": version: 7.26.9 resolution: "@babel/traverse@npm:7.26.9" @@ -597,7 +573,7 @@ __metadata: languageName: node linkType: hard -"@endo/common@npm:^1.2.10": +"@endo/common@npm:^1.2.10, @endo/common@npm:^1.2.8, @endo/common@npm:^1.2.9": version: 1.2.10 resolution: "@endo/common@npm:1.2.10" dependencies: @@ -608,17 +584,6 @@ __metadata: languageName: node linkType: hard -"@endo/common@npm:^1.2.8, @endo/common@npm:^1.2.9": - version: 1.2.9 - resolution: "@endo/common@npm:1.2.9" - dependencies: - "@endo/errors": "npm:^1.2.9" - "@endo/eventual-send": "npm:^1.3.0" - "@endo/promise-kit": "npm:^1.1.9" - checksum: 10/37d9bb1fbe6b974f135f755253ce58dd1692ed1849e1f2bdf91726bc62faba7dc4a9f9d527ce759feba08fe6cdb2597dc6bcc0daf8d516a780e856d61654f1d5 - languageName: node - linkType: hard - "@endo/compartment-mapper@npm:^1.6.0": version: 1.6.0 resolution: "@endo/compartment-mapper@npm:1.6.0" @@ -639,7 +604,7 @@ __metadata: languageName: node linkType: hard -"@endo/errors@npm:^1.2.10": +"@endo/errors@npm:^1.2.10, @endo/errors@npm:^1.2.8, @endo/errors@npm:^1.2.9": version: 1.2.10 resolution: "@endo/errors@npm:1.2.10" dependencies: @@ -648,15 +613,6 @@ __metadata: languageName: node linkType: hard -"@endo/errors@npm:^1.2.8, @endo/errors@npm:^1.2.9": - version: 1.2.9 - resolution: "@endo/errors@npm:1.2.9" - dependencies: - ses: "npm:^1.11.0" - checksum: 10/06457df5fa31709683f22fdf6b61e9f45056557308360ee582f3ed659476a44b8960b5c7337283a0c8da3162a4f1087883d228088f34a1fcefcb129cbd5188c6 - languageName: node - linkType: hard - "@endo/evasive-transform@npm:^1.4.0": version: 1.4.0 resolution: "@endo/evasive-transform@npm:1.4.0" @@ -692,18 +648,7 @@ __metadata: languageName: node linkType: hard -"@endo/far@npm:^1.0.0, @endo/far@npm:^1.1.10, @endo/far@npm:^1.1.9": - version: 1.1.10 - resolution: "@endo/far@npm:1.1.10" - dependencies: - "@endo/errors": "npm:^1.2.9" - "@endo/eventual-send": "npm:^1.3.0" - "@endo/pass-style": "npm:^1.4.8" - checksum: 10/028ca6ee4388f238bb452b8922507434867b689b283082209f948fd14059aab2cfde2d7fcf2e47c0f297bb345ecec615100f05b50dfb0e037b8bccd061b39a03 - languageName: node - linkType: hard - -"@endo/far@npm:^1.1.11": +"@endo/far@npm:^1.0.0, @endo/far@npm:^1.1.10, @endo/far@npm:^1.1.11, @endo/far@npm:^1.1.9": version: 1.1.11 resolution: "@endo/far@npm:1.1.11" dependencies: @@ -807,7 +752,7 @@ __metadata: languageName: node linkType: hard -"@endo/promise-kit@npm:^1.1.10": +"@endo/promise-kit@npm:^1.1.10, @endo/promise-kit@npm:^1.1.8, @endo/promise-kit@npm:^1.1.9": version: 1.1.10 resolution: "@endo/promise-kit@npm:1.1.10" dependencies: @@ -816,16 +761,7 @@ __metadata: languageName: node linkType: hard -"@endo/promise-kit@npm:^1.1.8, @endo/promise-kit@npm:^1.1.9": - version: 1.1.9 - resolution: "@endo/promise-kit@npm:1.1.9" - dependencies: - ses: "npm:^1.11.0" - checksum: 10/fad342c0573252b57bcd0601d0b47814068333942fdfb4ea1e9dc7980910e46cb6d22fe5715b32571b002cafad9cf085584267bf42d32282768f471d15e665ae - languageName: node - linkType: hard - -"@endo/stream@npm:^1.2.10": +"@endo/stream@npm:^1.2.10, @endo/stream@npm:^1.2.8, @endo/stream@npm:^1.2.9": version: 1.2.10 resolution: "@endo/stream@npm:1.2.10" dependencies: @@ -836,17 +772,6 @@ __metadata: languageName: node linkType: hard -"@endo/stream@npm:^1.2.8, @endo/stream@npm:^1.2.9": - version: 1.2.9 - resolution: "@endo/stream@npm:1.2.9" - dependencies: - "@endo/eventual-send": "npm:^1.3.0" - "@endo/promise-kit": "npm:^1.1.9" - ses: "npm:^1.11.0" - checksum: 10/5605e0135ad1ac426bafac3d3660642b64943eb0148a4b6a7b91e5d1a3bbd7ff856425207e80e6eee998ba47b2f3a1314cce52e4d7f965fd11bbd3e15a1144be - languageName: node - linkType: hard - "@endo/trampoline@npm:^1.0.3": version: 1.0.3 resolution: "@endo/trampoline@npm:1.0.3" @@ -9677,15 +9602,6 @@ __metadata: languageName: node linkType: hard -"ses@npm:^1.11.0": - version: 1.11.0 - resolution: "ses@npm:1.11.0" - dependencies: - "@endo/env-options": "npm:^1.1.8" - checksum: 10/b080fce2f369bd4e3cd7df01704347fc50b426b21269920ad89aeb36de3a3acbf9b849152d27c9fb2c77ea929d5b0f6568e2c2322bda5a4621e9d3d2d70b4cf0 - languageName: node - linkType: hard - "set-function-length@npm:^1.2.2": version: 1.2.2 resolution: "set-function-length@npm:1.2.2" From 496540d187a9fb48ecedf57f825d7adc199eb36e Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 20:54:46 +0200 Subject: [PATCH 13/23] add some time --- packages/kernel-test/src/garbage-collection.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index eac9ae022..9e32154b7 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -154,7 +154,7 @@ describe('Garbage Collection', () => { // Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(); + await waitUntilQuiescent(100); // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); From f7afb6827a1af86eefdac8eb2e64d2cd26a66260 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Wed, 2 Apr 2025 21:34:45 +0200 Subject: [PATCH 14/23] increase wait --- packages/kernel-test/src/garbage-collection.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 9e32154b7..1132cce14 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -154,7 +154,7 @@ describe('Garbage Collection', () => { // Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(100); + await waitUntilQuiescent(1000); // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); @@ -171,7 +171,7 @@ describe('Garbage Collection', () => { // Run a crank to allow bringOutYourDead to be processed await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(100); + await waitUntilQuiescent(1000); // Check reference counts after retireImports (both should be decreased) const afterForgetRefCounts = kernelStore.getObjectRefCount(createObjectRef); From 106f50bddf7eae9c419be69457807c24de602d5b Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 3 Apr 2025 00:51:02 +0200 Subject: [PATCH 15/23] increase test time --- packages/kernel-test/src/garbage-collection.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 1132cce14..194a9bf0a 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -199,5 +199,5 @@ describe('Garbage Collection', () => { [objectId], ); expect(parseReplyBody(exporterFinalCheck.body)).toBe(false); - }); + }, 30000); }); From 7eb468d3f277c4b4bfd54fd383ffdf23579cf51d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 3 Apr 2025 01:13:18 +0200 Subject: [PATCH 16/23] try multiple noop --- .../kernel-test/src/garbage-collection.test.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index 194a9bf0a..bbac9f3a9 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -152,9 +152,11 @@ describe('Garbage Collection', () => { // Schedule reap to trigger bringOutYourDead on next crank kernel.reapVats((vatId) => vatId === importerVatId); - // Run a crank to allow bringOutYourDead to be processed - await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(1000); + // Run 3 cranks to allow bringOutYourDead to be processed + for (let i = 0; i < 3; i++) { + await kernel.queueMessageFromKernel(importerKRef, 'noop', []); + await waitUntilQuiescent(300); + } // Check reference counts after dropImports const afterWeakRefCounts = kernelStore.getObjectRefCount(createObjectRef); @@ -169,9 +171,11 @@ describe('Garbage Collection', () => { // Schedule another reap kernel.reapVats((vatId) => vatId === importerVatId); - // Run a crank to allow bringOutYourDead to be processed - await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(1000); + // Run 3 cranks to allow bringOutYourDead to be processed + for (let i = 0; i < 3; i++) { + await kernel.queueMessageFromKernel(importerKRef, 'noop', []); + await waitUntilQuiescent(300); + } // Check reference counts after retireImports (both should be decreased) const afterForgetRefCounts = kernelStore.getObjectRefCount(createObjectRef); @@ -199,5 +203,5 @@ describe('Garbage Collection', () => { [objectId], ); expect(parseReplyBody(exporterFinalCheck.body)).toBe(false); - }, 30000); + }, 40000); }); From c156848ce0a4f97af060e16ba01d68f755d001a3 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 3 Apr 2025 01:31:41 +0200 Subject: [PATCH 17/23] try running kernel-tests on threads --- packages/kernel-test/vitest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kernel-test/vitest.config.ts b/packages/kernel-test/vitest.config.ts index 1f60917fc..0c4cdfa23 100644 --- a/packages/kernel-test/vitest.config.ts +++ b/packages/kernel-test/vitest.config.ts @@ -7,7 +7,7 @@ const config = mergeConfig( defineProject({ test: { name: 'kernel-test', - pool: 'forks', + pool: 'threads', }, }), ); From 56bd046f9272f4c3a88548f8857569a93854c18d Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 3 Apr 2025 14:48:48 +0200 Subject: [PATCH 18/23] apply suggestion --- .../src/garbage-collection.test.ts | 4 +-- packages/kernel/src/Kernel.ts | 12 ++----- packages/kernel/src/store/index.test.ts | 1 + .../kernel/src/store/methods/clist.test.ts | 34 +++++++++++++++++++ packages/kernel/src/store/methods/clist.ts | 14 ++++++++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/packages/kernel-test/src/garbage-collection.test.ts b/packages/kernel-test/src/garbage-collection.test.ts index bbac9f3a9..923987ff3 100644 --- a/packages/kernel-test/src/garbage-collection.test.ts +++ b/packages/kernel-test/src/garbage-collection.test.ts @@ -155,7 +155,7 @@ describe('Garbage Collection', () => { // Run 3 cranks to allow bringOutYourDead to be processed for (let i = 0; i < 3; i++) { await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(300); + await waitUntilQuiescent(500); } // Check reference counts after dropImports @@ -174,7 +174,7 @@ describe('Garbage Collection', () => { // Run 3 cranks to allow bringOutYourDead to be processed for (let i = 0; i < 3; i++) { await kernel.queueMessageFromKernel(importerKRef, 'noop', []); - await waitUntilQuiescent(300); + await waitUntilQuiescent(500); } // Check reference counts after retireImports (both should be decreased) diff --git a/packages/kernel/src/Kernel.ts b/packages/kernel/src/Kernel.ts index 57363cfdc..53ea9e4da 100644 --- a/packages/kernel/src/Kernel.ts +++ b/packages/kernel/src/Kernel.ts @@ -596,9 +596,7 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} dropExports`, krefs); const vat = this.#getVat(vatId); - const vrefs: VRef[] = krefs - .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) - .filter((ref): ref is VRef => typeof ref === 'string'); + const vrefs = this.#kernelStore.krefsToExistingErefs(vatId, krefs); log(`@@@@ deliver ${vatId} dropExports`, vrefs); await vat.deliverDropExports(vrefs); log(`@@@@ done ${vatId} dropExports`, krefs); @@ -608,9 +606,7 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} retireExports`, krefs); const vat = this.#getVat(vatId); - const vrefs: VRef[] = krefs - .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) - .filter((ref): ref is VRef => typeof ref === 'string'); + const vrefs = this.#kernelStore.krefsToExistingErefs(vatId, krefs); log(`@@@@ deliver ${vatId} retireExports`, vrefs); await vat.deliverRetireExports(vrefs); log(`@@@@ done ${vatId} retireExports`, krefs); @@ -620,9 +616,7 @@ export class Kernel { const { vatId, krefs } = item; log(`@@@@ deliver ${vatId} retireImports`, krefs); const vat = this.#getVat(vatId); - const vrefs: VRef[] = krefs - .map((kref) => this.#kernelStore.krefToEref(vatId, kref)) - .filter((ref): ref is VRef => typeof ref === 'string'); + const vrefs = this.#kernelStore.krefsToExistingErefs(vatId, krefs); log(`@@@@ deliver ${vatId} retireImports`, vrefs); await vat.deliverRetireImports(vrefs); log(`@@@@ done ${vatId} retireImports`, krefs); diff --git a/packages/kernel/src/store/index.test.ts b/packages/kernel/src/store/index.test.ts index 852af38a2..bde06b622 100644 --- a/packages/kernel/src/store/index.test.ts +++ b/packages/kernel/src/store/index.test.ts @@ -101,6 +101,7 @@ describe('kernel store', () => { 'initKernelPromise', 'kernelRefExists', 'krefToEref', + 'krefsToExistingErefs', 'kv', 'makeVatStore', 'nextReapAction', diff --git a/packages/kernel/src/store/methods/clist.test.ts b/packages/kernel/src/store/methods/clist.test.ts index c48b9ca7b..a6e49a6f1 100644 --- a/packages/kernel/src/store/methods/clist.test.ts +++ b/packages/kernel/src/store/methods/clist.test.ts @@ -169,6 +169,40 @@ describe('clist-methods', () => { }); }); + describe('krefsToExistingErefs', () => { + it('returns the ERefs for existing KRefs', () => { + const endpointId: EndpointId = 'v1'; + const kref1: KRef = 'ko1'; + const kref2: KRef = 'ko2'; + const eref1: ERef = 'o-1'; + const eref2: ERef = 'o-2'; + + clistMethods.addClistEntry(endpointId, kref1, eref1); + clistMethods.addClistEntry(endpointId, kref2, eref2); + + expect( + clistMethods.krefsToExistingErefs(endpointId, [kref1, kref2]), + ).toStrictEqual([eref1, eref2]); + }); + + it('returns an empty array for non-existent KRefs', () => { + const endpointId: EndpointId = 'v1'; + const kref: KRef = 'ko1'; + + expect( + clistMethods.krefsToExistingErefs(endpointId, [kref]), + ).toStrictEqual([]); + }); + + it('returns an empty array for empty KRef array', () => { + const endpointId: EndpointId = 'v1'; + + expect(clistMethods.krefsToExistingErefs(endpointId, [])).toStrictEqual( + [], + ); + }); + }); + describe('incrementRefCount', () => { it('increments promise reference counts', () => { const kref: KRef = 'kp1'; diff --git a/packages/kernel/src/store/methods/clist.ts b/packages/kernel/src/store/methods/clist.ts index efc752599..cf23ad12b 100644 --- a/packages/kernel/src/store/methods/clist.ts +++ b/packages/kernel/src/store/methods/clist.ts @@ -136,6 +136,19 @@ export function getCListMethods(ctx: StoreContext) { return vatSlot; } + /** + * Look up the ERef that and endpoint's c-list maps a KRef to. + * + * @param endpointId - The endpoint in question. + * @param krefs - The KRefs to look up. + * @returns The given endpoint's ERefs corresponding to `krefs` + */ + function krefsToExistingErefs(endpointId: EndpointId, krefs: KRef[]): ERef[] { + return krefs + .map((kref) => krefToEref(endpointId, kref)) + .filter((eref): eref is ERef => Boolean(eref)); + } + /** * Remove an entry from an endpoint's c-list given an eref. * @@ -263,6 +276,7 @@ export function getCListMethods(ctx: StoreContext) { krefToEref, forgetEref, forgetKref, + krefsToExistingErefs, // Refcount management incrementRefCount, decrementRefCount, From 8d4e7755e1fdc7322ed96394a8f07ff1020d3f87 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Thu, 3 Apr 2025 15:04:53 +0200 Subject: [PATCH 19/23] revert to forks and remove questionable tests --- packages/kernel-test/vitest.config.ts | 2 +- packages/kernel/src/utils/gc-finalize.test.ts | 49 ------------------- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/packages/kernel-test/vitest.config.ts b/packages/kernel-test/vitest.config.ts index 0c4cdfa23..1f60917fc 100644 --- a/packages/kernel-test/vitest.config.ts +++ b/packages/kernel-test/vitest.config.ts @@ -7,7 +7,7 @@ const config = mergeConfig( defineProject({ test: { name: 'kernel-test', - pool: 'threads', + pool: 'forks', }, }), ); diff --git a/packages/kernel/src/utils/gc-finalize.test.ts b/packages/kernel/src/utils/gc-finalize.test.ts index 8df1088e8..0c279cc2a 100644 --- a/packages/kernel/src/utils/gc-finalize.test.ts +++ b/packages/kernel/src/utils/gc-finalize.test.ts @@ -36,55 +36,6 @@ describe('Garbage Collection', () => { expect(finalizationCallback).toHaveBeenCalled(); }); - it('should handle errors gracefully', async () => { - // Create a custom implementation with a failing GC function - const consoleWarnSpy = vi - .spyOn(console, 'warn') - .mockImplementation(vi.fn()); - const mockGcFunction = vi.fn().mockImplementation(() => { - throw new Error('GC failed'); - }); - // Mock globalThis.gc - const originalGc = globalThis.gc; - globalThis.gc = mockGcFunction; - // Create a new gcAndFinalize with our mocked environment - const customGcAndFinalize = makeGCAndFinalize(); - // Should not throw despite GC failing - expect(await customGcAndFinalize()).toBeUndefined(); - // Should log warning - expect(consoleWarnSpy).toHaveBeenCalledWith( - 'GC operation failed:', - expect.any(Error), - ); - // Restore original gc - // Using Object.defineProperty to avoid race condition - Object.defineProperty(globalThis, 'gc', { - value: originalGc, - writable: true, - configurable: true, - }); - consoleWarnSpy.mockRestore(); - }); - - it('should run multiple GC passes', async () => { - // Mock the GC function to verify multiple calls - const mockGcFunction = vi.fn(); - // Mock globalThis.gc - const originalGc = globalThis.gc; - globalThis.gc = mockGcFunction; - // Create a new gcAndFinalize with our mocked environment - const customGcAndFinalize = makeGCAndFinalize(); - await customGcAndFinalize(); - // Should call GC function twice - expect(mockGcFunction).toHaveBeenCalledTimes(2); - // Restore original gc - Object.defineProperty(globalThis, 'gc', { - value: originalGc, - writable: true, - configurable: true, - }); - }); - it('should work with circular references', async () => { // Create objects with circular references type CircularObj = { name: string; ref: CircularObj | null }; From 7f732d5e054cd5d08ddb9681a57bf79c2dff5299 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Fri, 4 Apr 2025 19:59:56 +0200 Subject: [PATCH 20/23] clean --- packages/kernel-test/src/vats/exporter-vat.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/kernel-test/src/vats/exporter-vat.js b/packages/kernel-test/src/vats/exporter-vat.js index 959d4148e..16dd998f5 100644 --- a/packages/kernel-test/src/vats/exporter-vat.js +++ b/packages/kernel-test/src/vats/exporter-vat.js @@ -10,8 +10,6 @@ import { Far } from '@endo/marshal'; */ export function buildRootObject(_vatPowers, parameters, _baggage) { const name = parameters?.name ?? 'anonymous'; - // const { getSyscall } = vatPowers; - // const syscall = getSyscall(); /** * Print a message to the log. From d1d6e383e57820f1daf8a7c07c8fe5e989005d7f Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Apr 2025 15:17:34 +0200 Subject: [PATCH 21/23] thresholds --- vitest.config.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vitest.config.ts b/vitest.config.ts index b10b089d5..6a186a730 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -88,10 +88,10 @@ export default defineConfig({ lines: 81.03, }, 'packages/kernel/**': { - statements: 84.83, - functions: 90.99, - branches: 71.22, - lines: 84.94, + statements: 86.59, + functions: 92.88, + branches: 71.29, + lines: 86.56, }, 'packages/nodejs/**': { statements: 72.91, From b31f5da1db034a380e63d044f90cb37cd9439cc7 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Apr 2025 15:28:58 +0200 Subject: [PATCH 22/23] fix deps --- packages/kernel-test/package.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/kernel-test/package.json b/packages/kernel-test/package.json index 5531fc58a..7353b12ea 100644 --- a/packages/kernel-test/package.json +++ b/packages/kernel-test/package.json @@ -53,11 +53,6 @@ "@metamask/eslint-config-nodejs": "^14.0.0", "@metamask/eslint-config-typescript": "^14.0.0", "@ocap/cli": "workspace:^", - "@ocap/kernel": "workspace:^", - "@ocap/nodejs": "workspace:^", - "@ocap/store": "workspace:^", - "@ocap/streams": "workspace:^", - "@ocap/utils": "workspace:^", "@ts-bridge/cli": "^0.6.3", "@ts-bridge/shims": "^0.1.1", "@typescript-eslint/eslint-plugin": "^8.29.0", From bbf59307c5d95f8d09e4a40c0f155e13c06bf4d9 Mon Sep 17 00:00:00 2001 From: Dimitris Marlagkoutsos Date: Mon, 7 Apr 2025 15:32:43 +0200 Subject: [PATCH 23/23] remove unused --- packages/kernel/package.json | 1 - yarn.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/kernel/package.json b/packages/kernel/package.json index 2e553e09e..bc1396324 100644 --- a/packages/kernel/package.json +++ b/packages/kernel/package.json @@ -47,7 +47,6 @@ "@endo/far": "^1.1.11", "@endo/import-bundle": "^1.4.0", "@endo/marshal": "^1.6.4", - "@endo/nat": "^5.1.0", "@endo/pass-style": "^1.5.0", "@endo/promise-kit": "^1.1.10", "@metamask/superstruct": "^3.2.0", diff --git a/yarn.lock b/yarn.lock index d335a8f75..99136a271 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2056,7 +2056,6 @@ __metadata: "@endo/far": "npm:^1.1.11" "@endo/import-bundle": "npm:^1.4.0" "@endo/marshal": "npm:^1.6.4" - "@endo/nat": "npm:^5.1.0" "@endo/pass-style": "npm:^1.5.0" "@endo/promise-kit": "npm:^1.1.10" "@metamask/auto-changelog": "npm:^5.0.1"