optimize dataset items truncation logic

Author: MQ37

src/const.ts

@@ -43,8 +43,11 @@ export const defaults = {
 
 // Actor output const
 export const ACTOR_OUTPUT_MAX_CHARS_PER_ITEM = 5_000;
-export const ACTOR_OUTPUT_TRUNCATED_MESSAGE = `Output was truncated because it will not fit into context.`
-    + `There is no reason to call this tool again! You can use ${HelperTools.DATASET_GET_ITEMS} tool to get more items from the dataset.`;
+export const ACTOR_OUTPUT_TRUNCATED_MESSAGE = `Output was truncated because it will not fit into context.
+    There is no reason to call this tool again!
+    You can use ${HelperTools.DATASET_GET_ITEMS} tool to get more items from the dataset.
+    The items were truncated from the back, so if using the ${HelperTools.DATASET_GET_ITEMS} tool,
+    you can skip the first N items that you already have.`;
 
 export const ACTOR_ADDITIONAL_INSTRUCTIONS = `Never call/execute tool/Actor unless confirmed by the user.
      Workflow: When an Actor runs, it processes data and stores results in Apify storage,

src/mcp/server.ts

@@ -19,7 +19,6 @@ import log from '@apify/log';
 
 import {
     ACTOR_OUTPUT_MAX_CHARS_PER_ITEM,
-    ACTOR_OUTPUT_TRUNCATED_MESSAGE,
     defaults,
     SERVER_NAME,
     SERVER_VERSION,
@@ -29,7 +28,7 @@ import { actorNameToToolName } from '../tools/utils.js';
 import type { ActorMcpTool, ActorTool, HelperTool, ToolEntry } from '../types.js';
 import { createMCPClient } from './client.js';
 import { EXTERNAL_TOOL_CALL_TIMEOUT_MSEC } from './const.js';
-import { processParamsGetTools } from './utils.js';
+import { processParamsGetTools, truncateDatasetItems } from './utils.js';
 
 type ActorsMcpServerOptions = {
     enableAddingActors?: boolean;
@@ -462,12 +461,17 @@ export class ActorsMcpServer {
                         { type: 'text', text: `Dataset information: ${JSON.stringify(datasetInfo)}` },
                     ];
 
-                    const itemContents = items.items.map((item: Record<string, unknown>) => {
-                        const text = JSON.stringify(item).slice(0, ACTOR_OUTPUT_MAX_CHARS_PER_ITEM);
-                        return text.length === ACTOR_OUTPUT_MAX_CHARS_PER_ITEM
-                            ? { type: 'text', text: `${text} ... ${ACTOR_OUTPUT_TRUNCATED_MESSAGE}` }
-                            : { type: 'text', text };
-                    });
+                    // Get max char length for whole dataset based on the number of items
+                    const maxDatasetLength = ACTOR_OUTPUT_MAX_CHARS_PER_ITEM * items.items.length;
+                    const itemContents = truncateDatasetItems(items, maxDatasetLength, datasetInfo?.itemCount || 0)
+                        .items.map(
+                            (item: Record<string, unknown>) => {
+                                return {
+                                    type: 'text',
+                                    text: JSON.stringify(item),
+                                };
+                            },
+                        );
                     content.push(...itemContents);
                     return { content };
                 }

src/mcp/utils.ts

@@ -1,6 +1,9 @@
 import { createHash } from 'node:crypto';
 import { parse } from 'node:querystring';
 
+import type { PaginatedList } from 'apify-client';
+
+import { ACTOR_OUTPUT_TRUNCATED_MESSAGE } from '../const.js';
 import { processInput } from '../input.js';
 import { addRemoveTools, getActorsAsTools } from '../tools/index.js';
 import type { Input, ToolEntry } from '../types.js';
@@ -58,3 +61,43 @@ export function parseInputParamsFromUrl(url: string): Input {
     const params = parse(query) as unknown as Input;
     return processInput(params);
 }
+
+/**
+ * Truncates dataset items to fit within a specified character limit.
+ *
+ * This function will remove items from the end of the dataset until the total
+ * character count of the dataset items is within the specified limit.
+ * If there is only one item (left) in the dataset, it will not be truncated.
+ */
+export function truncateDatasetItems(
+    items: PaginatedList<Record<string, unknown>>,
+    maxChars: number,
+    originalItemCount: number,
+): PaginatedList<Record<string, unknown>> {
+    // If within the limit, return as is.
+    if (JSON.stringify(items).length <= maxChars) {
+        return items;
+    }
+
+    // Do not truncate single item datasets.
+    if (items.items.length < 2) {
+        return items;
+    }
+
+    // Truncate from back and check if the total length is within the limit.
+    while (items.items.length > 1) {
+        if (JSON.stringify(items).length <= maxChars) {
+            break; // If the dataset is within the limit, stop truncating.
+        }
+        items.items.pop(); // Remove the last item if the dataset exceeds the limit.
+    }
+
+    // Add truncation message
+    items.items.push({
+        truncationInfo: ACTOR_OUTPUT_TRUNCATED_MESSAGE,
+        originalItemCount,
+        itemCountAfterTruncation: items.items.length,
+    });
+
+    return items;
+}