Process refCounts and collect garbage

Author: sirtimid

packages/kernel-test/package.json

@@ -53,11 +53,6 @@
     "@metamask/eslint-config-nodejs": "^14.0.0",
     "@metamask/eslint-config-typescript": "^14.0.0",
     "@ocap/cli": "workspace:^",
-    "@ocap/kernel": "workspace:^",
-    "@ocap/nodejs": "workspace:^",
-    "@ocap/store": "workspace:^",
-    "@ocap/streams": "workspace:^",
-    "@ocap/utils": "workspace:^",
     "@ts-bridge/cli": "^0.6.2",
     "@ts-bridge/shims": "^0.1.1",
     "@typescript-eslint/eslint-plugin": "^8.26.1",
@@ -91,6 +86,10 @@
     "@endo/marshal": "^1.6.2",
     "@endo/promise-kit": "^1.1.6",
     "@ocap/kernel": "workspace:^",
-    "@ocap/shims": "workspace:^"
+    "@ocap/nodejs": "workspace:^",
+    "@ocap/shims": "workspace:^",
+    "@ocap/store": "workspace:^",
+    "@ocap/streams": "workspace:^",
+    "@ocap/utils": "workspace:^"
   }
 }

packages/kernel/src/Kernel.ts

@@ -183,6 +183,7 @@ export class Kernel {
     for await (const item of this.#runQueueItems()) {
       console.log('*** run loop item', item);
       await this.#deliver(item);
+      this.#kernelStore.collectGarbage();
     }
   }
 

packages/kernel/src/index.ts

@@ -19,6 +19,6 @@ export {
   ClusterConfigStruct,
 } from './types.ts';
 export { kunser, kser } from './services/kernel-marshal.ts';
-export { makeKernelStore } from './store/kernel-store.ts';
-export type { KernelStore } from './store/kernel-store.ts';
+export { makeKernelStore } from './store/index.ts';
+export type { KernelStore } from './store/index.ts';
 export { parseRef } from './store/utils/parse-ref.ts';

packages/kernel/src/store/index.ts

@@ -64,6 +64,7 @@ import { getIdMethods } from './methods/id.ts';
 import { getObjectMethods } from './methods/object.ts';
 import { getPromiseMethods } from './methods/promise.ts';
 import { getQueueMethods } from './methods/queue.ts';
+import { getReachableMethods } from './methods/reachable.ts';
 import { getRefCountMethods } from './methods/refcount.ts';
 import { getVatMethods } from './methods/vat.ts';
 import type { StoreContext } from './types.ts';
@@ -120,6 +121,8 @@ export function makeKernelStore(kdb: KernelDatabase) {
     // Garbage collection
     gcActions: provideCachedStoredValue('gcActions', '[]'),
     reapQueue: provideCachedStoredValue('reapQueue', '[]'),
+    // TODO: Store terminated vats in DB and fetch from there
+    terminatedVats: [],
   };
 
   const id = getIdMethods(context);
@@ -130,6 +133,7 @@ export function makeKernelStore(kdb: KernelDatabase) {
   const cList = getCListMethods(context);
   const queue = getQueueMethods(context);
   const vat = getVatMethods(context);
+  const reachable = getReachableMethods(context);
 
   /**
    * Create a new VatStore for a vat.
@@ -159,6 +163,7 @@ export function makeKernelStore(kdb: KernelDatabase) {
   function reset(): void {
     kdb.clear();
     context.maybeFreeKrefs.clear();
+    context.terminatedVats = [];
     context.runQueue = provideStoredQueue('run', true);
     context.gcActions = provideCachedStoredValue('gcActions', '[]');
     context.reapQueue = provideCachedStoredValue('reapQueue', '[]');
@@ -182,6 +187,7 @@ export function makeKernelStore(kdb: KernelDatabase) {
     ...object,
     ...promise,
     ...gc,
+    ...reachable,
     ...cList,
     ...vat,
     makeVatStore,

packages/kernel/src/store/methods/clist.ts

@@ -1,8 +1,8 @@
 import { Fail } from '@endo/errors';
 
 import { getBaseMethods } from './base.ts';
-import { getGCMethods } from './gc.ts';
 import { getObjectMethods } from './object.ts';
+import { getReachableMethods } from './reachable.ts';
 import { getRefCountMethods } from './refcount.ts';
 import type { EndpointId, KRef, ERef } from '../../types.ts';
 import type { StoreContext } from '../types.ts';
@@ -22,7 +22,7 @@ import {
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export function getCListMethods(ctx: StoreContext) {
   const { getSlotKey } = getBaseMethods(ctx.kv);
-  const { clearReachableFlag } = getGCMethods(ctx);
+  const { clearReachableFlag } = getReachableMethods(ctx);
   const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx);
   const { kernelRefExists, refCountKey } = getRefCountMethods(ctx);
 
@@ -218,7 +218,7 @@ export function getCListMethods(ctx: StoreContext) {
     {
       isExport = false,
       onlyRecognizable = false,
-    }: { isExport?: boolean; onlyRecognizable?: boolean },
+    }: { isExport?: boolean; onlyRecognizable?: boolean } = {},
   ): boolean {
     kref || Fail`decrementRefCount called with empty kref`;
 

packages/kernel/src/store/methods/gc.ts

@@ -1,9 +1,14 @@
+import { Fail } from '@endo/errors';
+
 import { getBaseMethods } from './base.ts';
+import { getCListMethods } from './clist.ts';
 import { getObjectMethods } from './object.ts';
+import { getPromiseMethods } from './promise.ts';
+import { getReachableMethods } from './reachable.ts';
 import { getRefCountMethods } from './refcount.ts';
+import { getVatMethods } from './vat.ts';
 import type {
   VatId,
-  EndpointId,
   KRef,
   GCAction,
   RunQueueItemBringOutYourDead,
@@ -14,12 +19,7 @@ import {
   RunQueueItemType,
 } from '../../types.ts';
 import type { StoreContext } from '../types.ts';
-import { insistKernelType } from '../utils/kernel-slots.ts';
-import { parseRef } from '../utils/parse-ref.ts';
-import {
-  buildReachableAndVatSlot,
-  parseReachableAndVatSlot,
-} from '../utils/reachable.ts';
+import { insistKernelType, parseKernelSlot } from '../utils/kernel-slots.ts';
 
 /**
  * Create a store for garbage collection.
@@ -30,9 +30,12 @@ import {
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 export function getGCMethods(ctx: StoreContext) {
   const { getSlotKey } = getBaseMethods(ctx.kv);
-  const { getObjectRefCount, setObjectRefCount } = getObjectMethods(ctx);
-  const { kernelRefExists } = getRefCountMethods(ctx);
-
+  const { getRefCount } = getRefCountMethods(ctx);
+  const { getObjectRefCount, deleteKernelObject } = getObjectMethods(ctx);
+  const { getKernelPromise, deleteKernelPromise } = getPromiseMethods(ctx);
+  const { decrementRefCount } = getCListMethods(ctx);
+  const { getImporters } = getVatMethods(ctx);
+  const { getReachableFlag, getReachableAndVatSlot } = getReachableMethods(ctx);
   /**
    * Get the set of GC actions to perform.
    *
@@ -71,49 +74,6 @@ export function getGCMethods(ctx: StoreContext) {
     setGCActions(actions);
   }
 
-  /**
-   * Check if a kernel object is reachable.
-   *
-   * @param endpointId - The endpoint for which the reachable flag is being checked.
-   * @param kref - The kref.
-   * @returns True if the kernel object is reachable, false otherwise.
-   */
-  function getReachableFlag(endpointId: EndpointId, kref: KRef): boolean {
-    const key = getSlotKey(endpointId, kref);
-    const data = ctx.kv.getRequired(key);
-    const { isReachable } = parseReachableAndVatSlot(data);
-    return isReachable;
-  }
-
-  /**
-   * Clear the reachable flag for a given endpoint and kref.
-   *
-   * @param endpointId - The endpoint for which the reachable flag is being cleared.
-   * @param kref - The kref.
-   */
-  function clearReachableFlag(endpointId: EndpointId, kref: KRef): void {
-    const key = getSlotKey(endpointId, kref);
-    const { isReachable, vatSlot } = parseReachableAndVatSlot(
-      ctx.kv.getRequired(key),
-    );
-    ctx.kv.set(key, buildReachableAndVatSlot(false, vatSlot));
-    const { direction, isPromise } = parseRef(vatSlot);
-    // decrement 'reachable' part of refcount, but only for object imports
-    if (
-      isReachable &&
-      !isPromise &&
-      direction === 'import' &&
-      kernelRefExists(kref)
-    ) {
-      const counts = getObjectRefCount(kref);
-      counts.reachable -= 1;
-      setObjectRefCount(kref, counts);
-      if (counts.reachable === 0) {
-        ctx.maybeFreeKrefs.add(kref);
-      }
-    }
-  }
-
   /**
    * Schedule a vat for reaping.
    *
@@ -142,16 +102,124 @@ export function getGCMethods(ctx: StoreContext) {
     return undefined;
   }
 
+  /**
+   * Retires kernel objects by notifying importers and removing the objects.
+   *
+   * @param koids - Array of kernel object IDs to retire.
+   */
+  function retireKernelObjects(koids: KRef[]): void {
+    Array.isArray(koids) || Fail`retireExports given non-Array ${koids}`;
+    const newActions: GCAction[] = [];
+    for (const koid of koids) {
+      const importers = getImporters(koid);
+      for (const vatID of importers) {
+        newActions.push(`${vatID} retireImport ${koid}`);
+      }
+      deleteKernelObject(koid);
+    }
+    addGCActions(newActions);
+  }
+
+  /**
+   * Processes reference counts for kernel resources and performs garbage collection actions
+   * for resources that are no longer referenced or should be retired.
+   */
+  function collectGarbage(): void {
+    const actions: Set<GCAction> = new Set();
+    for (const kref of ctx.maybeFreeKrefs.values()) {
+      const { type } = parseKernelSlot(kref);
+      if (type === 'promise') {
+        const kpid = kref;
+        const kp = getKernelPromise(kpid);
+        const refCount = getRefCount(kpid);
+        if (refCount === 0) {
+          if (kp.state === 'fulfilled' || kp.state === 'rejected') {
+            // https://github.com/Agoric/agoric-sdk/issues/9888 don't assume promise is settled
+            for (const slot of kp.value?.slots ?? []) {
+              // Note: the following decrement can result in an addition to the
+              // maybeFreeKrefs set, which we are in the midst of iterating.
+              // TC39 went to a lot of trouble to ensure that this is kosher.
+              decrementRefCount(slot);
+            }
+          }
+          deleteKernelPromise(kpid);
+        }
+      }
+
+      if (type === 'object') {
+        const { reachable, recognizable } = getObjectRefCount(kref);
+        if (reachable === 0) {
+          // We avoid ownerOfKernelObject(), which will report
+          // 'undefined' if the owner is dead (and being slowly
+          // deleted). Message delivery should use that, but not us.
+          const ownerKey = `${kref}.owner`;
+          let ownerVatID = ctx.kv.get(ownerKey);
+          const terminated = ctx.terminatedVats.includes(ownerVatID as VatId);
+
+          // Some objects that are still owned, but the owning vat
+          // might still alive, or might be terminated and in the
+          // process of being deleted. These two clauses are
+          // mutually exclusive.
+          if (ownerVatID && !terminated) {
+            const vatConsidersReachable = getReachableFlag(ownerVatID, kref);
+            if (vatConsidersReachable) {
+              // the reachable count is zero, but the vat doesn't realize it
+              actions.add(`${ownerVatID} dropExport ${kref}`);
+            }
+            if (recognizable === 0) {
+              // TODO: rethink this assert
+              // assert.equal(vatConsidersReachable, false, `${kref} is reachable but not recognizable`);
+              actions.add(`${ownerVatID} retireExport ${kref}`);
+            }
+          } else if (ownerVatID && terminated) {
+            // When we're slowly deleting a vat, and one of its
+            // exports becomes unreferenced, we obviously must not
+            // send dropExports or retireExports into the dead vat.
+            // We fast-forward the abandonment that slow-deletion
+            // would have done, then treat the object as orphaned.
+
+            const { vatSlot } = getReachableAndVatSlot(ownerVatID, kref);
+            // delete directly, not orphanKernelObject(), which
+            // would re-submit to maybeFreeKrefs
+            ctx.kv.delete(ownerKey);
+            ctx.kv.delete(getSlotKey(ownerVatID, kref));
+            ctx.kv.delete(getSlotKey(ownerVatID, vatSlot));
+            // now fall through to the orphaned case
+            ownerVatID = undefined;
+          }
+
+          // Now handle objects which were orphaned. NOTE: this
+          // includes objects which were owned by a terminated (but
+          // not fully deleted) vat, where `ownerVatID` was cleared
+          // in the last line of that previous clause (the
+          // fall-through case). Don't try to change this `if
+          // (!ownerVatID)` into an `else if`: the two clauses are
+          // *not* mutually-exclusive.
+          if (!ownerVatID) {
+            // orphaned and unreachable, so retire it. If the kref
+            // is recognizable, then we need retireKernelObjects()
+            // to scan for importers and send retireImports (and
+            // delete), else we can call deleteKernelObject directly
+            if (recognizable) {
+              retireKernelObjects([kref]);
+            } else {
+              deleteKernelObject(kref);
+            }
+          }
+        }
+      }
+    }
+    addGCActions([...actions]);
+    ctx.maybeFreeKrefs.clear();
+  }
+
   return {
-    // GC actions
     getGCActions,
     setGCActions,
     addGCActions,
-    // Reachability tracking
-    getReachableFlag,
-    clearReachableFlag,
-    // Reaping
     scheduleReap,
     nextReapAction,
+    retireKernelObjects,
+    collectGarbage,
   };
 }

packages/kernel/src/store/methods/reachable.test.ts

@@ -0,0 +1,27 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+
+import { makeMapKernelDatabase } from '../../../test/storage.ts';
+import { makeKernelStore } from '../index.ts';
+
+describe('GC methods', () => {
+  let kernelStore: ReturnType<typeof makeKernelStore>;
+
+  beforeEach(() => {
+    kernelStore = makeKernelStore(makeMapKernelDatabase());
+  });
+
+  describe('reachability tracking', () => {
+    it('manages reachable flags', () => {
+      const ko1 = kernelStore.initKernelObject('v1');
+      kernelStore.addClistEntry('v1', ko1, 'o-1');
+
+      expect(kernelStore.getReachableFlag('v1', ko1)).toBe(true);
+
+      kernelStore.clearReachableFlag('v1', ko1);
+      expect(kernelStore.getReachableFlag('v1', ko1)).toBe(false);
+
+      const refCounts = kernelStore.getObjectRefCount(ko1);
+      expect(refCounts.reachable).toBe(0);
+    });
+  });
+});