feat: Add Actor runs API, dataset API, KV-store (#122)

* fix: Use a new API to get Actor default build`
* feat: add get-actor and get-user-runs-list
* feat: add dataset, Actor runs, fix tests
* fix: add key-value stores

Author: jirispilka

package-lock.json

@@ -16,6 +16,22 @@ import { parseInputParamsFromUrl, processParamsGetTools } from '../mcp/utils.js'
 import { getHelpMessage, HEADER_READINESS_PROBE, Routes } from './const.js';
 import { getActorRunData } from './utils.js';
 
+/**
+ * Helper function to load tools and actors based on input parameters
+ * @param mcpServer The MCP server instance
+ * @param url The request URL to parse parameters from
+ * @param apifyToken The Apify token for authentication
+ */
+async function loadToolsAndActors(mcpServer: ActorsMcpServer, url: string, apifyToken: string): Promise<void> {
+    const input = parseInputParamsFromUrl(url);
+    if (input.actors || input.enableAddingActors) {
+        await mcpServer.loadToolsFromUrl(url, apifyToken);
+    }
+    if (!input.actors) {
+        await mcpServer.loadDefaultActors(apifyToken);
+    }
+}
+
 export function createExpressApp(
     host: string,
     mcpServer: ActorsMcpServer,
@@ -49,7 +65,7 @@ export function createExpressApp(
             // TODO: I think we should remove this logic, root should return only help message
             const tools = await processParamsGetTools(req.url, process.env.APIFY_TOKEN as string);
             if (tools) {
-                mcpServer.updateTools(tools);
+                mcpServer.upsertTools(tools);
             }
             res.setHeader('Content-Type', 'text/event-stream');
             res.setHeader('Cache-Control', 'no-cache');
@@ -67,14 +83,7 @@ export function createExpressApp(
     app.get(Routes.SSE, async (req: Request, res: Response) => {
         try {
             log.info(`Received GET message at: ${Routes.SSE}`);
-            const input = parseInputParamsFromUrl(req.url);
-            if (input.actors || input.enableAddingActors) {
-                await mcpServer.loadToolsFromUrl(req.url, process.env.APIFY_TOKEN as string);
-            }
-            // Load default tools if no actors are specified
-            if (!input.actors) {
-                await mcpServer.loadDefaultTools(process.env.APIFY_TOKEN as string);
-            }
+            await loadToolsAndActors(mcpServer, req.url, process.env.APIFY_TOKEN as string);
             transportSSE = new SSEServerTransport(Routes.MESSAGE, res);
             await mcpServer.connect(transportSSE);
         } catch (error) {
@@ -124,16 +133,7 @@ export function createExpressApp(
                     enableJsonResponse: true, // Enable JSON response mode
                 });
                 // Load MCP server tools
-                // TODO using query parameters in POST request is not standard
-                const input = parseInputParamsFromUrl(req.url);
-                if (input.actors || input.enableAddingActors) {
-                    await mcpServer.loadToolsFromUrl(req.url, process.env.APIFY_TOKEN as string);
-                }
-                // Load default tools if no actors are specified
-                if (!input.actors) {
-                    await mcpServer.loadDefaultTools(process.env.APIFY_TOKEN as string);
-                }
-
+                await loadToolsAndActors(mcpServer, req.url, process.env.APIFY_TOKEN as string);
                 // Connect the transport to the MCP server BEFORE handling the request
                 await mcpServer.connect(transport);
 

src/actor/server.ts

@@ -3,13 +3,7 @@ export const ACTOR_README_MAX_LENGTH = 5_000;
 export const ACTOR_ENUM_MAX_LENGTH = 200;
 export const ACTOR_MAX_DESCRIPTION_LENGTH = 500;
 
-// Actor output const
-export const ACTOR_OUTPUT_MAX_CHARS_PER_ITEM = 5_000;
-export const ACTOR_OUTPUT_TRUNCATED_MESSAGE = `Output was truncated because it will not fit into context.`
-    + `There is no reason to call this tool again!`;
-
-export const ACTOR_ADDITIONAL_INSTRUCTIONS = 'Never call/execute tool/Actor unless confirmed by the user. '
-    + 'Always limit the number of results in the call arguments.';
+export const ACTOR_RUN_DATASET_OUTPUT_MAX_ITEMS = 5;
 
 // Actor run const
 export const ACTOR_MAX_MEMORY_MBYTES = 4_096; // If the Actor requires 8GB of memory, free users can't run actors-mcp-server and requested Actor
@@ -22,29 +16,43 @@ export const SERVER_VERSION = '1.0.0';
 export const USER_AGENT_ORIGIN = 'Origin/mcp-server';
 
 export enum HelperTools {
-    SEARCH_ACTORS = 'search-actors',
-    ADD_ACTOR = 'add-actor',
-    REMOVE_ACTOR = 'remove-actor',
-    GET_ACTOR_DETAILS = 'get-actor-details',
-    HELP_TOOL = 'help-tool',
+    ACTOR_ADD = 'add-actor',
+    ACTOR_GET = 'get-actor',
+    ACTOR_GET_DETAILS = 'get-actor-details',
+    ACTOR_REMOVE = 'remove-actor',
+    ACTOR_RUNS_ABORT = 'abort-actor-run',
+    ACTOR_RUNS_GET = 'get-actor-run',
+    ACTOR_RUNS_LOG = 'get-actor-log',
+    ACTOR_RUN_LIST_GET = 'get-actor-run-list',
+    DATASET_GET = 'get-dataset',
+    DATASET_LIST_GET = 'get-dataset-list',
+    DATASET_GET_ITEMS = 'get-dataset-items',
+    KEY_VALUE_STORE_LIST_GET = 'get-key-value-store-list',
+    KEY_VALUE_STORE_GET = 'get-key-value-store',
+    KEY_VALUE_STORE_KEYS_GET = 'get-key-value-store-keys',
+    KEY_VALUE_STORE_RECORD_GET = 'get-key-value-store-record',
+    APIFY_MCP_HELP_TOOL = 'apify-actor-help-tool',
+    STORE_SEARCH = 'search-actors',
 }
 
 export const defaults = {
     actors: [
         'apify/rag-web-browser',
     ],
-    helperTools: [
-        HelperTools.SEARCH_ACTORS,
-        HelperTools.GET_ACTOR_DETAILS,
-        HelperTools.HELP_TOOL,
-    ],
-    actorAddingTools: [
-        HelperTools.ADD_ACTOR,
-        HelperTools.REMOVE_ACTOR,
-    ],
 };
 
-export const APIFY_USERNAME = 'apify';
+// Actor output const
+export const ACTOR_OUTPUT_MAX_CHARS_PER_ITEM = 5_000;
+export const ACTOR_OUTPUT_TRUNCATED_MESSAGE = `Output was truncated because it will not fit into context.`
+    + `There is no reason to call this tool again! You can use ${HelperTools.DATASET_GET_ITEMS} tool to get more items from the dataset.`;
+
+export const ACTOR_ADDITIONAL_INSTRUCTIONS = `Never call/execute tool/Actor unless confirmed by the user.
+     Workflow: When an Actor runs, it processes data and stores results in Apify storage,
+     Datasets (for structured/tabular data) and Key-Value Store (for various data types like JSON, images, HTML).
+     Each Actor run produces a dataset ID and key-value store ID for accessing the results.
+     By default, the number of items returned from an Actor run is limited to ${ACTOR_RUN_DATASET_OUTPUT_MAX_ITEMS}.
+     You can always use ${HelperTools.DATASET_GET_ITEMS} tool to get more items from the dataset.
+     Actor run input is always stored in the key-value store, recordKey: INPUT.`;
 
 export const TOOL_CACHE_MAX_SIZE = 500;
 export const TOOL_CACHE_TTL_SECS = 30 * 60;

src/const.ts

@@ -67,7 +67,7 @@ async function callSearchTool(client: Client): Promise<void> {
         const searchRequest: CallToolRequest = {
             method: 'tools/call',
             params: {
-                name: HelperTools.SEARCH_ACTORS,
+                name: HelperTools.STORE_SEARCH,
                 arguments: { search: 'rag web browser', limit: 1 },
             },
         };

src/examples/clientStreamableHttp.ts

@@ -44,7 +44,7 @@ if (STANDBY_MODE) {
         const { actors } = input;
         const actorsToLoad = Array.isArray(actors) ? actors : actors.split(',');
         const tools = await getActorsAsTools(actorsToLoad, process.env.APIFY_TOKEN as string);
-        mcpServer.updateTools(tools);
+        mcpServer.upsertTools(tools);
     }
     app.listen(PORT, () => {
         log.info(`The Actor web server is listening for user requests at ${HOST}`);
@@ -56,9 +56,9 @@ if (STANDBY_MODE) {
         await Actor.fail('If you need to debug a specific Actor, please provide the debugActor and debugActorInput fields in the input');
     }
     const options = { memory: input.maxActorMemoryBytes } as ActorCallOptions;
-    const items = await callActorGetDataset(input.debugActor!, input.debugActorInput!, process.env.APIFY_TOKEN, options);
+    const { datasetInfo, items } = await callActorGetDataset(input.debugActor!, input.debugActorInput!, process.env.APIFY_TOKEN, options);
 
     await Actor.pushData(items);
-    log.info(`Pushed ${items.length} items to the dataset`);
+    log.info(`Pushed ${datasetInfo?.itemCount} items to the dataset`);
     await Actor.exit();
 }

src/main.ts

@@ -1,24 +1,24 @@
 import type { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import Ajv from 'ajv';
 
-import type { ActorMCPTool, ToolWrap } from '../types.js';
+import type { ActorMcpTool, ToolEntry } from '../types.js';
 import { getMCPServerID, getProxyMCPServerToolName } from './utils.js';
 
 export async function getMCPServerTools(
     actorID: string,
     client: Client,
     // Name of the MCP server
     serverUrl: string,
-): Promise<ToolWrap[]> {
+): Promise<ToolEntry[]> {
     const res = await client.listTools();
     const { tools } = res;
 
     const ajv = new Ajv({ coerceTypes: 'array', strict: false });
 
-    const compiledTools: ToolWrap[] = [];
+    const compiledTools: ToolEntry[] = [];
     for (const tool of tools) {
-        const mcpTool: ActorMCPTool = {
-            actorID,
+        const mcpTool: ActorMcpTool = {
+            actorId: actorID,
             serverId: getMCPServerID(serverUrl),
             serverUrl,
             originToolName: tool.name,
@@ -29,7 +29,7 @@ export async function getMCPServerTools(
             ajvValidate: ajv.compile(tool.inputSchema),
         };
 
-        const wrap: ToolWrap = {
+        const wrap: ToolEntry = {
             type: 'actor-mcp',
             tool: mcpTool,
         };