feat: serve pages as markdown and update llms.txt (#2249)

- Install docusaurus-markdown-source-plugin to inject "Open Markdown"
  dropdown on every /docs/ page linking to currentPath + .md
- Add nginx location block to intercept .md URLs, strip the extension,
  and proxy to developers.stellar.org with Accept: text/markdown header
- Update static/llms.txt to document the .md URL scheme and Accept header

Author: kaankacar

docusaurus.config.ts

@@ -84,6 +84,7 @@ const config: Config = {
     ],
     './src/plugins/route-export/index.ts',
     './src/plugins/analytics-module/index.ts',
+    'docusaurus-markdown-source-plugin',
   ],
   markdown: {
     mermaid: true,

nginx/nginx.conf

@@ -33,5 +33,12 @@ http {
     location /404.html {
         add_header 'Cache-Control' 'no-store' always;
     }
+    location ~* ^(.+)\.md$ {
+        resolver 1.1.1.1 valid=300s;
+        proxy_pass https://developers.stellar.org$1;
+        proxy_set_header Accept "text/markdown";
+        proxy_set_header Host "developers.stellar.org";
+        proxy_ssl_server_name on;
+    }
   }
 }

package.json

@@ -49,6 +49,7 @@
     "@stellar/open-rpc-docs-react": "^0.2.1",
     "ajv": "^8.18.0",
     "clsx": "^2.1.1",
+    "docusaurus-markdown-source-plugin": "^2.0.1",
     "docusaurus-plugin-openapi-docs": "4.5.1",
     "docusaurus-plugin-sass": "^0.2.6",
     "docusaurus-plugin-sentry": "^2.1.0",

static/llms.txt

@@ -1,6 +1,10 @@
 # Stellar Developer Documentation
 Stellar is a layer-1 open-source, decentralized, peer-to-peer blockchain network that provides a framework for developers to create applications, issue assets, write smart contracts, and connect to existing financial rails. Stellar is designed to enable creators, innovators, and developers to build projects on the network that can interoperate with each other.
 
+Each documentation page is available as clean markdown by appending `.md` to the URL
+(e.g. `https://developers.stellar.org/docs/build/smart-contracts/overview.md`).
+Alternatively, send an `Accept: text/markdown` request header to any page URL to receive markdown without modifying the URL.
+
 
 [Build](https://developers.stellar.org/docs/build)
 

yarn.lock

@@ -6526,6 +6526,13 @@ dns-packet@^5.2.2:
   dependencies:
     "@leichtgewicht/ip-codec" "^2.0.1"
 
+docusaurus-markdown-source-plugin@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/docusaurus-markdown-source-plugin/-/docusaurus-markdown-source-plugin-2.0.1.tgz#6c3fa0df4ec1f2230922ff04af3a62ef6ba8b156"
+  integrity sha512-RiKMi+SDlrQQ25VkbOvz9M7glH9lcpAsz0WzepilnC2MgXA6uLNpl7PezrlY6nvW7MnhB4Aakwwonz+0WVBQAQ==
+  dependencies:
+    fs-extra "^11.0.0"
+
 docusaurus-plugin-openapi-docs@4.5.1:
   version "4.5.1"
   resolved "https://registry.yarnpkg.com/docusaurus-plugin-openapi-docs/-/docusaurus-plugin-openapi-docs-4.5.1.tgz#b1133b0e03e284a6ade47fa2c3fcf5eab8ec6911"
@@ -7415,6 +7422,15 @@ fs-extra@^10.0.0, fs-extra@^10.1.0:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
+fs-extra@^11.0.0:
+  version "11.3.3"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.3.tgz#a27da23b72524e81ac6c3815cc0179b8c74c59ee"
+  integrity sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg==
+  dependencies:
+    graceful-fs "^4.2.0"
+    jsonfile "^6.0.1"
+    universalify "^2.0.0"
+
 fs-extra@^11.1.1, fs-extra@^11.2.0:
   version "11.3.0"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.0.tgz#0daced136bbaf65a555a326719af931adc7a314d"