Implement proper terminated vat cleanup

Author: sirtimid

packages/kernel/src/Kernel.ts

@@ -181,6 +181,7 @@ export class Kernel {
    */
   async #run(): Promise<void> {
     for await (const item of this.#runQueueItems()) {
+      this.#kernelStore.nextTerminatedVatCleanup();
       await this.#deliver(item);
       this.#kernelStore.collectGarbage();
     }
@@ -832,6 +833,7 @@ export class Kernel {
   async terminateVat(vatId: VatId): Promise<void> {
     await this.#stopVat(vatId, true);
     this.#kernelStore.deleteVatConfig(vatId);
+    this.#kernelStore.markVatAsTerminated(vatId);
   }
 
   /**

packages/kernel/src/store/index.ts

@@ -121,8 +121,7 @@ export function makeKernelStore(kdb: KernelDatabase) {
     // Garbage collection
     gcActions: provideCachedStoredValue('gcActions', '[]'),
     reapQueue: provideCachedStoredValue('reapQueue', '[]'),
-    // TODO: Store terminated vats in DB and fetch from there
-    terminatedVats: [],
+    terminatedVats: provideCachedStoredValue('vats.terminated', '[]'),
   };
 
   const id = getIdMethods(context);
@@ -152,7 +151,6 @@ export function makeKernelStore(kdb: KernelDatabase) {
    * @param vatId - The vat whose state is to be deleted.
    */
   function deleteVat(vatId: VatId): void {
-    vat.deleteEndpoint(vatId);
     vat.deleteVatConfig(vatId);
     kdb.deleteVatStore(vatId);
   }
@@ -163,10 +161,10 @@ export function makeKernelStore(kdb: KernelDatabase) {
   function reset(): void {
     kdb.clear();
     context.maybeFreeKrefs.clear();
-    context.terminatedVats = [];
     context.runQueue = provideStoredQueue('run', true);
     context.gcActions = provideCachedStoredValue('gcActions', '[]');
     context.reapQueue = provideCachedStoredValue('reapQueue', '[]');
+    context.terminatedVats = provideCachedStoredValue('vats.terminated', '[]');
     context.nextObjectId = provideCachedStoredValue('nextObjectId', '1');
     context.nextPromiseId = provideCachedStoredValue('nextPromiseId', '1');
     context.nextVatId = provideCachedStoredValue('nextVatId', '1');

packages/kernel/src/store/methods/gc.ts

@@ -34,7 +34,7 @@ export function getGCMethods(ctx: StoreContext) {
   const { getObjectRefCount, deleteKernelObject } = getObjectMethods(ctx);
   const { getKernelPromise, deleteKernelPromise } = getPromiseMethods(ctx);
   const { decrementRefCount } = getCListMethods(ctx);
-  const { getImporters } = getVatMethods(ctx);
+  const { getImporters, isVatTerminated } = getVatMethods(ctx);
   const { getReachableFlag, getReachableAndVatSlot } = getReachableMethods(ctx);
   /**
    * Get the set of GC actions to perform.
@@ -154,7 +154,7 @@ export function getGCMethods(ctx: StoreContext) {
           // deleted). Message delivery should use that, but not us.
           const ownerKey = `${kref}.owner`;
           let ownerVatID = ctx.kv.get(ownerKey);
-          const terminated = ctx.terminatedVats.includes(ownerVatID as VatId);
+          const terminated = isVatTerminated(ownerVatID as VatId);
 
           // Some objects that are still owned, but the owning vat
           // might still alive, or might be terminated and in the

packages/kernel/src/store/methods/vat.ts

@@ -1,6 +1,8 @@
 import { getBaseMethods } from './base.ts';
+import { getCListMethods } from './clist.ts';
+import { getReachableMethods } from './reachable.ts';
 import type { EndpointId, KRef, VatConfig, VatId } from '../../types.ts';
-import type { StoreContext } from '../types.ts';
+import type { StoreContext, VatCleanupWork } from '../types.ts';
 import { parseRef } from '../utils/parse-ref.ts';
 import { parseReachableAndVatSlot } from '../utils/reachable.ts';
 
@@ -22,6 +24,8 @@ const VAT_CONFIG_BASE_LEN = VAT_CONFIG_BASE.length;
 export function getVatMethods(ctx: StoreContext) {
   const { kv } = ctx;
   const { getPrefixedKeys, getSlotKey } = getBaseMethods(ctx.kv);
+  const { decrementRefCount } = getCListMethods(ctx);
+  const { clearReachableFlag } = getReachableMethods(ctx);
 
   /**
    * Delete all persistent state associated with an endpoint.
@@ -129,6 +133,156 @@ export function getVatMethods(ctx: StoreContext) {
     return importers;
   }
 
+  /**
+   * Get the list of terminated vats.
+   *
+   * @returns an array of terminated vat IDs.
+   */
+  function getTerminatedVats(): VatId[] {
+    return JSON.parse(ctx.terminatedVats.get() ?? '[]');
+  }
+
+  /**
+   * Check if a vat is terminated.
+   *
+   * @param vatID - The ID of the vat to check.
+   * @returns True if the vat is terminated, false otherwise.
+   */
+  function isVatTerminated(vatID: VatId): boolean {
+    return getTerminatedVats().includes(vatID);
+  }
+
+  /**
+   * Add a vat to the list of terminated vats.
+   *
+   * @param vatID - The ID of the vat to add.
+   */
+  function markVatAsTerminated(vatID: VatId): void {
+    const terminatedVats = getTerminatedVats();
+    if (!terminatedVats.includes(vatID)) {
+      terminatedVats.push(vatID);
+      ctx.terminatedVats.set(JSON.stringify(terminatedVats));
+    }
+  }
+
+  /**
+   * Remove a vat from the list of terminated vats.
+   *
+   * @param vatID - The ID of the vat to remove.
+   */
+  function forgetTerminatedVat(vatID: VatId): void {
+    const terminatedVats = getTerminatedVats().filter((id) => id !== vatID);
+    ctx.terminatedVats.set(JSON.stringify(terminatedVats));
+  }
+
+  /**
+   * Cleanup a terminated vat.
+   *
+   * @param vatID - The ID of the vat to cleanup.
+   * @returns The work done during the cleanup.
+   */
+  function cleanupTerminatedVat(vatID: VatId): VatCleanupWork {
+    const work = {
+      exports: 0,
+      imports: 0,
+      promises: 0,
+      kv: 0,
+    };
+
+    if (!isVatTerminated(vatID)) {
+      return work;
+    }
+
+    const clistPrefix = `${vatID}.c.`;
+    const exportPrefix = `${clistPrefix}o+`;
+    const importPrefix = `${clistPrefix}o-`;
+    const promisePrefix = `${clistPrefix}p`;
+
+    // First, clean up exports (objects exported by the terminated vat)
+    for (const key of getPrefixedKeys(exportPrefix)) {
+      const vref = key.slice(clistPrefix.length);
+      const kref = ctx.kv.get(key);
+      if (kref) {
+        const vatKey = getSlotKey(vatID, vref);
+        const kernelKey = getSlotKey(vatID, kref);
+        // Clear the reachable flag
+        clearReachableFlag(vatID, kref);
+        // Delete the c-list entries
+        ctx.kv.delete(kernelKey);
+        ctx.kv.delete(vatKey);
+        // Delete the owner entry
+        ctx.kv.delete(`${kref}.owner`);
+        // Add to maybeFreeKrefs for GC processing
+        ctx.maybeFreeKrefs.add(kref);
+        work.exports += 1;
+      }
+    }
+
+    // Next, clean up imports (objects imported by the terminated vat)
+    for (const key of getPrefixedKeys(importPrefix)) {
+      const vref = key.slice(clistPrefix.length);
+      const kref = ctx.kv.get(key);
+      if (kref) {
+        const vatKey = getSlotKey(vatID, vref);
+        const kernelKey = getSlotKey(vatID, kref);
+        // Clear the reachable flag
+        clearReachableFlag(vatID, kref);
+        // Decrement ref count for the import
+        decrementRefCount(kref, {
+          isExport: false,
+          onlyRecognizable: true,
+        });
+        // Delete the c-list entries
+        ctx.kv.delete(kernelKey);
+        ctx.kv.delete(vatKey);
+        work.imports += 1;
+      }
+    }
+
+    // Clean up promises
+    for (const key of getPrefixedKeys(promisePrefix)) {
+      const vref = key.slice(clistPrefix.length);
+      const kref = ctx.kv.get(key);
+      if (kref) {
+        const vatKey = getSlotKey(vatID, vref);
+        const kernelKey = getSlotKey(vatID, kref);
+        // Decrement refcount for the promise
+        decrementRefCount(kref);
+        // Delete the c-list entries
+        ctx.kv.delete(kernelKey);
+        ctx.kv.delete(vatKey);
+        work.promises += 1;
+      }
+    }
+
+    // Finally, clean up any remaining KV entries for this vat
+    for (const key of getPrefixedKeys(`${vatID}.`)) {
+      ctx.kv.delete(key);
+      work.kv += 1;
+    }
+
+    // Clean up any remaining c-list entries and vat-specific counters
+    deleteEndpoint(vatID);
+
+    // Remove the vat from the terminated vats list
+    forgetTerminatedVat(vatID);
+
+    // Log the cleanup work done
+    console.log(`Cleaned up terminated vat ${vatID}:`, work);
+
+    return work;
+  }
+
+  /**
+   * Get the next terminated vat to cleanup.
+   *
+   * @returns The work done during the cleanup.
+   */
+  function nextTerminatedVatCleanup(): VatCleanupWork | undefined {
+    const vatID = getTerminatedVats()?.[0];
+    return vatID ? cleanupTerminatedVat(vatID) : undefined;
+  }
+
   return {
     deleteEndpoint,
     getAllVatRecords,
@@ -138,5 +292,11 @@ export function getVatMethods(ctx: StoreContext) {
     getVatIDs,
     importsKernelSlot,
     getImporters,
+    getTerminatedVats,
+    markVatAsTerminated,
+    forgetTerminatedVat,
+    isVatTerminated,
+    cleanupTerminatedVat,
+    nextTerminatedVatCleanup,
   };
 }

packages/kernel/src/store/types.ts

@@ -1,6 +1,6 @@
 import type { KVStore } from '@ocap/store';
 
-import type { KRef, VatId } from '../types.ts';
+import type { KRef } from '../types.ts';
 
 export type StoreContext = {
   kv: KVStore;
@@ -13,7 +13,7 @@ export type StoreContext = {
   maybeFreeKrefs: Set<KRef>;
   gcActions: StoredValue;
   reapQueue: StoredValue;
-  terminatedVats: VatId[];
+  terminatedVats: StoredValue;
 };
 
 export type StoredValue = {
@@ -27,3 +27,10 @@ export type StoredQueue = {
   dequeue(): object | undefined;
   delete(): void;
 };
+
+export type VatCleanupWork = {
+  exports: number;
+  imports: number;
+  promises: number;
+  kv: number;
+};