|
| 1 | +import fs from 'node:fs/promises'; |
| 2 | +import path from 'node:path'; |
| 3 | + |
| 4 | +const BUILD_DIR = path.resolve('build'); |
| 5 | + |
| 6 | +const FILES_ROUTES = { |
| 7 | + 'llms.txt': [ |
| 8 | + 'https://docs.apify.com/api/client/js/llms.txt', |
| 9 | + 'https://docs.apify.com/api/client/python/llms.txt', |
| 10 | + 'https://docs.apify.com/sdk/js/llms.txt', |
| 11 | + 'https://docs.apify.com/sdk/python/llms.txt', |
| 12 | + 'https://docs.apify.com/cli/llms.txt', |
| 13 | + ], |
| 14 | + 'llms-full.txt': [ |
| 15 | + 'https://docs.apify.com/api/client/js/llms-full.txt', |
| 16 | + 'https://docs.apify.com/api/client/python/llms-full.txt', |
| 17 | + 'https://docs.apify.com/sdk/js/llms-full.txt', |
| 18 | + 'https://docs.apify.com/sdk/python/llms-full.txt', |
| 19 | + 'https://docs.apify.com/cli/llms-full.txt', |
| 20 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md', |
| 21 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/ACTOR_FILE.md', |
| 22 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/DATASET_SCHEMA.md', |
| 23 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/IDEAS.md', |
| 24 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/INPUT_SCHEMA.md', |
| 25 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/KEY_VALUE_STORE_SCHEMA.md', |
| 26 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/OUTPUT_SCHEMA.md', |
| 27 | + 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/REQUEST_QUEUE_SCHEMA.md', |
| 28 | + ], |
| 29 | +}; |
| 30 | + |
| 31 | +async function fetchFile(route) { |
| 32 | + try { |
| 33 | + const res = await fetch(route); |
| 34 | + if (!res.ok) throw new Error(`Failed to fetch ${route}: ${res.status}`); |
| 35 | + return await res.text(); |
| 36 | + } catch (err) { |
| 37 | + console.error(`Error fetching ${route}:`, err.message); |
| 38 | + return ''; |
| 39 | + } |
| 40 | +} |
| 41 | + |
| 42 | +async function joinFiles() { |
| 43 | + await fs.mkdir(BUILD_DIR, { recursive: true }); |
| 44 | + for (const [llmsFile, files] of Object.entries(FILES_ROUTES)) { |
| 45 | + const contents = await Promise.all( |
| 46 | + files.map((route) => fetchFile(route)), |
| 47 | + ); |
| 48 | + const joined = contents.filter(Boolean).join('\n\n'); |
| 49 | + await fs.appendFile(path.join(BUILD_DIR, llmsFile), joined, 'utf8'); |
| 50 | + console.log(`Wrote ${llmsFile} to build/`); |
| 51 | + } |
| 52 | +} |
| 53 | + |
| 54 | +async function sanitizeFile(filePath) { |
| 55 | + const content = await fs.readFile(filePath, 'utf8'); |
| 56 | + const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags |
| 57 | + await fs.writeFile(filePath, sanitizedContent, 'utf8'); |
| 58 | + console.log(`Sanitized ${filePath}`); |
| 59 | +} |
| 60 | + |
| 61 | +joinFiles().catch((err) => { |
| 62 | + console.error('Failed to join LLMs files:', err); |
| 63 | + process.exit(1); |
| 64 | +}); |
| 65 | + |
| 66 | +Object.keys(FILES_ROUTES).forEach((llmsFile) => sanitizeFile(path.join(BUILD_DIR, llmsFile))); |
0 commit comments