Release v0.3.4.

Fix retrieve_entities task pagination/count consistency so deleted-filtered entities do not inflate totals and task pages return stable, complete results across offsets.

Author: markmhendrickson

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "neotoma",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "MCP server for structured personal data memory with unified source ingestion",
   "main": "dist/index.js",
   "type": "module",

package.json

@@ -60,7 +60,7 @@ export async function queryEntities(
   // Build query for entities
   let entityQuery = db
     .from("entities")
-    .select("id, entity_type, canonical_name, user_id, merged_to_entity_id, merged_at");
+    .select("id, entity_type, canonical_name, user_id, merged_to_entity_id, merged_at, created_at");
 
   // Filter by user if provided
   if (userId) {
@@ -82,38 +82,28 @@ export async function queryEntities(
     entityQuery = entityQuery.in("id", filterEntityIds);
   }
 
-  entityQuery = entityQuery.range(offset, offset + limit - 1);
+  // Always use a deterministic order so pagination is stable across calls.
+  entityQuery = entityQuery.order("id", { ascending: true });
 
-  const { data: entitiesData, error: entitiesError } = await entityQuery;
+  const { data: baseEntitiesData, error: entitiesError } = await entityQuery;
 
   if (entitiesError) {
     throw new Error(`Failed to query entities: ${entitiesError.message}`);
   }
 
-  if (!entitiesData || entitiesData.length === 0) {
+  if (!baseEntitiesData || baseEntitiesData.length === 0) {
     return [];
   }
 
-  // Get snapshots for entities
-  let entities = entitiesData;
-  const entityIds = entities.map((e: any) => e.id);
-  const { data: snapshots, error: snapshotsError } = await db
-    .from("entity_snapshots")
-    .select("*")
-    .in("entity_id", entityIds);
-
-  if (snapshotsError) {
-    throw new Error(`Failed to query snapshots: ${snapshotsError.message}`);
-  }
-
-  // Filter deleted entities unless explicitly requested
-  let filteredEntityIds = entityIds;
+  // Filter deleted entities unless explicitly requested, then paginate.
+  let entities = baseEntitiesData;
   if (!includeDeleted) {
+    const allEntityIds = entities.map((e: any) => e.id);
     // Check for deletion observations (highest priority with _deleted: true)
     const { data: deletionObservations } = await db
       .from("observations")
       .select("entity_id, source_priority, observed_at, fields")
-      .in("entity_id", entityIds)
+      .in("entity_id", allEntityIds)
       .order("source_priority", { ascending: false })
       .order("observed_at", { ascending: false });
 
@@ -144,12 +134,24 @@ export async function queryEntities(
     }
 
     // Filter out deleted entities
-    filteredEntityIds = entityIds.filter((id: string) => !deletedEntityIds.has(id));
     entities = entities.filter((e: any) => !deletedEntityIds.has(e.id));
+  }
 
-    if (entities.length === 0) {
-      return [];
-    }
+  entities = entities.slice(offset, offset + limit);
+  if (entities.length === 0) {
+    return [];
+  }
+
+  // Get snapshots for entities on the selected page
+  const entityIds = entities.map((e: any) => e.id);
+  const filteredEntityIds = entityIds;
+  const { data: snapshots, error: snapshotsError } = await db
+    .from("entity_snapshots")
+    .select("*")
+    .in("entity_id", entityIds);
+
+  if (snapshotsError) {
+    throw new Error(`Failed to query snapshots: ${snapshotsError.message}`);
   }
 
   // Combine entities with snapshots

src/services/entity_queries.ts

@@ -14,6 +14,63 @@ interface QueryEntitiesParams {
   offset?: number;
 }
 
+async function countVisibleEntities(params: {
+  userId: string;
+  entityType?: string;
+  includeMerged?: boolean;
+}): Promise<number> {
+  const { userId, entityType, includeMerged = false } = params;
+
+  let entityIdQuery = db.from("entities").select("id").eq("user_id", userId);
+  if (entityType) {
+    entityIdQuery = entityIdQuery.eq("entity_type", entityType);
+  }
+  if (!includeMerged) {
+    entityIdQuery = entityIdQuery.is("merged_to_entity_id", null);
+  }
+
+  const { data: entityRows, error: entityError } = await entityIdQuery;
+  if (entityError) {
+    throw new Error(`Failed to query entity ids for count: ${entityError.message}`);
+  }
+  if (!entityRows || entityRows.length === 0) {
+    return 0;
+  }
+
+  const entityIds = entityRows.map((row: { id: string }) => row.id);
+  const deletedEntityIds = new Set<string>();
+  const chunkSize = 500;
+
+  for (let i = 0; i < entityIds.length; i += chunkSize) {
+    const chunk = entityIds.slice(i, i + chunkSize);
+    const { data: deletionObservations, error: observationsError } = await db
+      .from("observations")
+      .select("entity_id, source_priority, observed_at, fields")
+      .in("entity_id", chunk)
+      .order("source_priority", { ascending: false })
+      .order("observed_at", { ascending: false });
+
+    if (observationsError) {
+      throw new Error(`Failed to query deletion observations for count: ${observationsError.message}`);
+    }
+
+    const highestByEntity = new Map<string, any>();
+    for (const obs of deletionObservations || []) {
+      if (!highestByEntity.has(obs.entity_id)) {
+        highestByEntity.set(obs.entity_id, obs);
+      }
+    }
+
+    for (const [entityId, obs] of highestByEntity.entries()) {
+      if (obs.fields?._deleted === true) {
+        deletedEntityIds.add(entityId);
+      }
+    }
+  }
+
+  return entityIds.length - deletedEntityIds.size;
+}
+
 export async function queryEntitiesWithCount(params: QueryEntitiesParams): Promise<{
   entities: EntityWithProvenance[];
   total: number;
@@ -83,19 +140,7 @@ export async function queryEntitiesWithCount(params: QueryEntitiesParams): Promi
       offset,
     });
 
-    let countQuery = db.from("entities").select("*", { count: "exact", head: true });
-    countQuery = countQuery.eq("user_id", userId);
-    if (entityType) {
-      countQuery = countQuery.eq("entity_type", entityType);
-    }
-    if (!includeMerged) {
-      countQuery = countQuery.is("merged_to_entity_id", null);
-    }
-    const { count, error: countError } = await countQuery;
-    if (countError) {
-      throw new Error(`Failed to count entities: ${countError.message}`);
-    }
-    total = count || 0;
+    total = await countVisibleEntities({ userId, entityType, includeMerged });
   }
 
   return {

src/shared/action_handlers/entity_handlers.ts

@@ -2,6 +2,8 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest";
 import { db } from "../../src/db.js";
 import { TestIdTracker } from "../helpers/cleanup_helpers.js";
 import { verifyEntityExists, computeEntitySnapshot } from "../helpers/database_verifiers.js";
+import { queryEntities } from "../../src/services/entity_queries.js";
+import { queryEntitiesWithCount } from "../../src/shared/action_handlers/entity_handlers.js";
 
 describe("MCP entity actions - parameter variations", () => {
   const tracker = new TestIdTracker();
@@ -212,6 +214,99 @@ describe("MCP entity actions - parameter variations", () => {
 
       expect(entities!.length).toBeGreaterThan(0);
     });
+
+    it("should paginate after filtering deleted task entities", async () => {
+      const localUserId = `test-user-entity-pagination-${Date.now()}`;
+      const sourceId = `src_pagination_${Date.now()}`;
+      const deletedEntityId = `ent_000_deleted_${Date.now()}`;
+      const activeEntityId = `ent_999_active_${Date.now()}`;
+
+      const { error: sourceError } = await db.from("sources").insert({
+        id: sourceId,
+        user_id: localUserId,
+        content_hash: `pagination_deleted_filter_${Date.now()}`,
+        storage_url: "file:///test/minimal.txt",
+        mime_type: "text/plain",
+        file_size: 0,
+      });
+      expect(sourceError).toBeNull();
+
+      tracker.trackSource(sourceId);
+      tracker.trackEntity(deletedEntityId);
+      tracker.trackEntity(activeEntityId);
+
+      const { error: entityError } = await db.from("entities").insert([
+        {
+          id: deletedEntityId,
+          entity_type: "task",
+          canonical_name: "Deleted task for pagination filter test",
+          user_id: localUserId,
+        },
+        {
+          id: activeEntityId,
+          entity_type: "task",
+          canonical_name: "Active task for pagination filter test",
+          user_id: localUserId,
+        },
+      ]);
+      expect(entityError).toBeNull();
+
+      const { error: observationError } = await db.from("observations").insert([
+        {
+          entity_id: deletedEntityId,
+          entity_type: "task",
+          source_id: sourceId,
+          fields: { title: "Deleted task" },
+          user_id: localUserId,
+          source_priority: 100,
+          schema_version: "1.0",
+          observed_at: new Date(Date.now() - 2000).toISOString(),
+        },
+        {
+          entity_id: activeEntityId,
+          entity_type: "task",
+          source_id: sourceId,
+          fields: { title: "Active task" },
+          user_id: localUserId,
+          source_priority: 100,
+          schema_version: "1.0",
+          observed_at: new Date(Date.now() - 2000).toISOString(),
+        },
+        {
+          entity_id: deletedEntityId,
+          entity_type: "task",
+          source_id: sourceId,
+          fields: { _deleted: true },
+          user_id: localUserId,
+          source_priority: 1000,
+          schema_version: "1.0",
+          observed_at: new Date(Date.now() - 1000).toISOString(),
+        },
+      ]);
+      expect(observationError).toBeNull();
+
+      const entities = await queryEntities({
+        userId: localUserId,
+        entityType: "task",
+        includeMerged: false,
+        limit: 1,
+        offset: 0,
+      });
+
+      expect(entities).toHaveLength(1);
+      expect(entities[0].entity_id).toBe(activeEntityId);
+
+      const withCount = await queryEntitiesWithCount({
+        userId: localUserId,
+        entityType: "task",
+        includeMerged: false,
+        limit: 20,
+        offset: 0,
+      });
+
+      expect(withCount.entities).toHaveLength(1);
+      expect(withCount.total).toBe(1);
+    });
   });
 
   describe("retrieve_entity_by_identifier variations", () => {