feat: add dev get html skeleton tool (#273)

* feat: add dev get html skeleton tool

* rename page to chunk

Author: MQ37

package-lock.json

@@ -38,11 +38,13 @@
     "@apify/datastructures": "^2.0.3",
     "@apify/log": "^2.5.16",
     "@modelcontextprotocol/sdk": "^1.17.4",
+    "@types/cheerio": "^0.22.35",
     "@types/turndown": "^5.0.5",
     "ajv": "^8.17.1",
     "algoliasearch": "^5.31.0",
     "apify": "^3.4.2",
     "apify-client": "^2.12.6",
+    "cheerio": "^1.1.2",
     "express": "^4.21.2",
     "to-json-schema": "^0.2.5",
     "turndown": "^7.2.0",
@@ -71,6 +73,7 @@
     "start": "npm run start:dev",
     "start:prod": "node dist/main.js",
     "start:dev": "tsx src/main.ts",
+    "start:standby": "APIFY_META_ORIGIN=STANDBY npm run start",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "build": "tsc -b src",

package.json

@@ -44,11 +44,14 @@ export enum HelperTools {
     STORE_SEARCH = 'search-actors',
     DOCS_SEARCH = 'search-apify-docs',
     DOCS_FETCH = 'fetch-apify-docs',
+    GET_HTML_SKELETON = 'get-html-skeleton',
 }
 
+export const ACTOR_RAG_WEB_BROWSER = 'apify/rag-web-browser';
+
 export const defaults = {
     actors: [
-        'apify/rag-web-browser',
+        ACTOR_RAG_WEB_BROWSER,
     ],
 };
 
@@ -67,6 +70,8 @@ export const ACTOR_CACHE_MAX_SIZE = 500;
 export const ACTOR_CACHE_TTL_SECS = 30 * 60; // 30 minutes
 export const APIFY_DOCS_CACHE_MAX_SIZE = 500;
 export const APIFY_DOCS_CACHE_TTL_SECS = 60 * 60; // 1 hour
+export const GET_HTML_SKELETON_CACHE_TTL_SECS = 5 * 60; // 5 minutes
+export const GET_HTML_SKELETON_CACHE_MAX_SIZE = 200;
 
 export const ACTOR_PRICING_MODEL = {
     /** Rental Actors */

src/const.ts

@@ -1,8 +1,17 @@
-import { ACTOR_CACHE_MAX_SIZE, ACTOR_CACHE_TTL_SECS, APIFY_DOCS_CACHE_MAX_SIZE, APIFY_DOCS_CACHE_TTL_SECS } from './const.js';
+import {
+    ACTOR_CACHE_MAX_SIZE,
+    ACTOR_CACHE_TTL_SECS,
+    APIFY_DOCS_CACHE_MAX_SIZE,
+    APIFY_DOCS_CACHE_TTL_SECS,
+    GET_HTML_SKELETON_CACHE_MAX_SIZE,
+    GET_HTML_SKELETON_CACHE_TTL_SECS,
+} from './const.js';
 import type { ActorDefinitionPruned, ApifyDocsSearchResult } from './types.js';
 import { TTLLRUCache } from './utils/ttl-lru.js';
 
 export const actorDefinitionPrunedCache = new TTLLRUCache<ActorDefinitionPruned>(ACTOR_CACHE_MAX_SIZE, ACTOR_CACHE_TTL_SECS);
 export const searchApifyDocsCache = new TTLLRUCache<ApifyDocsSearchResult[]>(APIFY_DOCS_CACHE_MAX_SIZE, APIFY_DOCS_CACHE_TTL_SECS);
 /** Stores processed Markdown content */
 export const fetchApifyDocsCache = new TTLLRUCache<string>(APIFY_DOCS_CACHE_MAX_SIZE, APIFY_DOCS_CACHE_TTL_SECS);
+/** Stores HTML content per URL so we can paginate the tool output */
+export const getHtmlSkeletonCache = new TTLLRUCache<string>(GET_HTML_SKELETON_CACHE_MAX_SIZE, GET_HTML_SKELETON_CACHE_TTL_SECS);