docs: Add llms.txt to docs build (#1316)

### Description

`@signalwire/docusaurus-plugin-llms-txt` will create .md files plus
llms-python.txt, which will be linked in JS Crawlee `llms.txt`
apify/crawlee#3085

Author: HonzaTuron

website/docusaurus.config.js

@@ -99,10 +99,6 @@ module.exports = {
                         return `https://github.com/apify/crawlee-python/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`;
                     },
                 },
-                // blog: {
-                //     blogTitle: 'Crawlee Blog - learn how to build better scrapers',
-                //     blogDescription: 'Guides and tutorials on using Crawlee, the most reliable open-source web scraping and browser automation library for JavaScript and Node.js developers.',
-                // },
                 theme: {
                     customCss: '/src/css/custom.css',
                 },
@@ -168,6 +164,17 @@ module.exports = {
                 id: 'GTM-5P7MCS7',
             },
         ],
+        [
+            '@signalwire/docusaurus-plugin-llms-txt',
+            {
+                enableDescriptions: false,
+                content: {
+                    includeVersionedDocs: false,
+                    enableLlmsFullTxt: true,
+                    relativePaths: false,
+                },
+            },
+        ],
         async function runnableCodeBlock() {
             return {
                 name: 'runnable-code-block',

website/package.json

@@ -44,6 +44,7 @@
         "@docusaurus/theme-mermaid": "3.8.1",
         "@giscus/react": "^3.0.0",
         "@mdx-js/react": "^3.0.1",
+        "@signalwire/docusaurus-plugin-llms-txt": "^1.2.1",
         "axios": "^1.5.0",
         "buffer": "^6.0.3",
         "clsx": "^2.0.0",