Provide entire docs in a single text file for LLMs.

.gitignore

@@ -27,3 +27,8 @@ lib/
 
 src/**/*.mjs
 scripts/**/*.mjs
+
+# Generated via generate-llms script
+public/llms/manual/**/llm*.txt
+public/llms/react/**/llm*.txt
+pages/docs/**/**/llms.mdx

data/sidebar_manual_v1100.json

@@ -73,6 +73,7 @@
   "Extra": [
     "newcomer-examples",
     "project-structure",
-    "faq"
+    "faq",
+    "llms"
   ]
 }

data/sidebar_manual_v1200.json

@@ -72,6 +72,7 @@
   "Extra": [
     "newcomer-examples",
     "project-structure",
-    "faq"
+    "faq",
+    "llms"
   ]
 }

data/sidebar_react_latest.json

@@ -31,5 +31,8 @@
     "beyond-jsx",
     "forwarding-refs",
     "extensions-of-props"
+  ],
+  "Extra": [
+    "llms"
   ]
 }

data/sidebar_react_v0100.json

@@ -26,5 +26,8 @@
   "Guides": [
     "beyond-jsx",
     "forwarding-refs"
+  ],
+  "Extra": [
+    "llms"
   ]
 }

data/sidebar_react_v0110.json

@@ -27,5 +27,8 @@
     "beyond-jsx",
     "forwarding-refs",
     "extensions-of-props"
+  ],
+  "Extra": [
+    "llms"
   ]
 }

next.config.mjs

@@ -136,6 +136,11 @@ const config = {
         destination: `/docs/manual/${process.env.VERSION_NEXT}/:slug*`,
         permanent: false,
       },
+      {
+        source: "/llms/manual/latest/:file*",
+        destination: `/llms/manual/${process.env.VERSION_LATEST}/:file*`,
+        permanent: false,
+      },
     ];
   },
 };

package.json

@@ -61,7 +61,8 @@
     "build": "rescript && npm run update-index && next build",
     "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs",
     "reanalyze": "reanalyze -all-cmt .",
-    "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml"
+    "update-index": "npm run generate-llms && node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml",
+    "generate-llms": "node scripts/generate_llms.mjs"
   },
   "devDependencies": {
     "@mdx-js/react": "^2.3.0",

public/_redirects

@@ -11,3 +11,5 @@
 
 /docs/manual/latest/:slug*                            /docs/manual/v11.0.0/:slug*                            307
 /docs/manual/next/:slug*                              /docs/manual/v12.0.0/:slug*                            307
+
+/llms/manual/latest/:file*                            /llms/manual/v11.0.0/:file*                            307

public/llms/manual/template.mdx

@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/manual/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/manual/llms.txt](/llms/manual/<VERSION>/llms.txt) — a list of the available files for ReScript language.
+- [/docs/manual/llm-full.txt](/llms/manual/<VERSION>/llm-full.txt) — complete documentation for ReScript language.
+- [/docs/manual/llm-small.txt](/llms/manual/<VERSION>/llm-small.txt) — compressed version of the former, without examples.
+
+...and package-level documentation:
+
+- [/docs/react/llms](/docs/react/latest/llms) — the LLms documentation for ReScript React.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/manual/template.txt

@@ -0,0 +1,19 @@
+# ReScript Documentation for LLMs
+
+> ReScript is a robustly typed language that compiles to efficient and human-readable JavaScript. It comes with a lightning fast compiler toolchain that scales to any codebase size.
+
+## Documentation Sets
+
+- [Complete documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-full.txt): The complete ReScript documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/manual/<VERSION>/llm-small.txt): A minimal version of the ReScript documentation, with the essential content for quick reference
+
+## Individual Package Documentation
+
+- [ReScript React documentation](https://rescript-lang.org/llms/react/latest/llms.txt): This is the developer documentation for ReScript React.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/react/template.mdx

@@ -0,0 +1,23 @@
+---
+title: "LLMs"
+description: "Documentation for LLMs"
+canonical: "/docs/react/<VERSION>/llms"
+---
+
+# Documentation for LLMs
+
+We adhere to the [llms.txt convention](https://llmstxt.org/) to make documentation accessible to large language models and their applications.
+
+Currently, we have the following files...
+
+- [/docs/react/llms.txt](/llms/react/<VERSION>/llms.txt) — a list of the available files for ReScript React.
+- [/docs/react/llms-full.txt](/llms/react/<VERSION>/llm-full.txt) — complete documentation for ReScript React.
+- [/docs/react/llms-small.txt](/llms/react/<VERSION>/llm-small.txt) — compressed version of the former, without examples for ReScript React.
+
+...and the language documentation:
+
+- [/docs/manual/llms](/docs/manual/latest/llms) — the LLms documentation for ReScript.
+
+## Notes
+
+- The content is automatically generated from the same source as the official documentation for the specific version

public/llms/react/template.txt

@@ -0,0 +1,19 @@
+# ReScript React Documentation for LLMs
+
+> ReScript React is a strongly typed interface for React, designed to integrate seamlessly with modern React (>= v18.0) while compiling to idiomatic JavaScript, enabling robust and scalable React applications.
+
+## ReScript React Documentation
+
+- [Complete documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-full.txt): The complete ReScript React documentation including all examples and additional content
+- [Abridged documentation](https://rescript-lang.org/llms/react/<VERSION>/llm-small.txt): A minimal version of the ReScript React documentation, with the essential content for quick reference
+
+## Language Documentation
+
+- [ReScript documentation](https://rescript-lang.org/llms/manual/latest/llms.txt): This is the developer documentation for ReScript.
+
+## Notes
+
+- The abridged documentation excludes the detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation for the specific version

scripts/generate_llms.res

@@ -0,0 +1,137 @@
+let readMarkdownFile = (filePath: string): string => {
+  let fileContent = Node.Fs.readFileSync2(filePath, "utf8")
+  fileContent
+}
+
+let rec collectFiles = (dirPath: string): array<string> => {
+  let entries = Node.Fs.readdirSync(dirPath)
+  entries->Array.reduce([], (acc, entry) => {
+    let fullPath = Node.Path.join([dirPath, entry])
+    let stats = Node.Fs.statSync(fullPath)
+    switch stats["isDirectory"]() {
+    | true => acc->Array.concat(collectFiles(fullPath))
+    | false => {
+        acc->Array.push(fullPath)
+        acc
+      }
+    }
+  })
+}
+
+let clearFile = (filePath: string): unit => {
+  Node.Fs.writeFileSync(filePath, "")
+}
+
+let createDirectoryIfNotExists = (dirPath: string): unit => {
+  if !Node.Fs.existsSync(dirPath) {
+    Node.Fs.mkdirSync(dirPath)
+  }
+}
+
+let removeCodeTabTags = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<CodeTab.*?>[\\s\\S]*?</CodeTab>", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeCodeBlocks = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeFileTitle = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeUnnecessaryBreaks = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("^\n{2,}", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeToDos = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<!-- TODO[\\s\\S]*?-->", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let fillContentWithVersion = (content: string, version: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<VERSION>", ~flags="g")
+  Js.String.replaceByRe(regex, version, content)
+}
+
+let createFullFile = (content: string, filePath: string): unit => {
+  Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8")
+}
+
+let createSmallFile = (content: string, filePath: string): unit => {
+  let smallContent =
+    content
+    ->removeCodeTabTags
+    ->removeFileTitle
+    ->removeToDos
+    ->removeCodeBlocks
+    ->removeUnnecessaryBreaks
+  Node.Fs.appendFileSync(filePath, smallContent, "utf8")
+}
+
+let createLlmsFiles = (version: string, docsDirectory: string, llmsDirectory: string): unit => {
+  let mdxFileTemplatePath = llmsDirectory->Node.Path.join2("template.mdx")
+  let mdxFilePath = docsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.mdx")
+  let txtFileTemplatePath = llmsDirectory->Node.Path.join2("template.txt")
+  let txtFilePath = llmsDirectory->Node.Path.join2(version)->Node.Path.join2("llms.txt")
+
+  Node.Fs.writeFileSync(
+    mdxFilePath,
+    readMarkdownFile(mdxFileTemplatePath)->fillContentWithVersion(version),
+  )
+
+  Node.Fs.writeFileSync(
+    txtFilePath,
+    readMarkdownFile(txtFileTemplatePath)->fillContentWithVersion(version),
+  )
+}
+
+let processVersions = (
+  versions: array<string>,
+  docsDirectory: string,
+  llmsDirectory: string,
+): unit => {
+  let fullFileName = "llm-full.txt"
+  let smallFileName = "llm-small.txt"
+
+  versions->Array.forEach(version => {
+    let versionDir = docsDirectory->Node.Path.join2(version)
+    let llmsDir = llmsDirectory->Node.Path.join2(version)
+    let fullFilePath = llmsDir->Node.Path.join2(fullFileName)
+    let smallFilePath = llmsDir->Node.Path.join2(smallFileName)
+
+    createDirectoryIfNotExists(llmsDir)
+    clearFile(fullFilePath)
+    clearFile(smallFilePath)
+
+    createLlmsFiles(version, docsDirectory, llmsDirectory)
+
+    versionDir
+    ->collectFiles
+    ->Array.forEach(filePath => {
+      if Js.String.endsWith(".mdx", filePath) {
+        let content = readMarkdownFile(filePath)
+
+        content->createFullFile(fullFilePath)
+
+        content->createSmallFile(smallFilePath)
+      }
+    })
+  })
+}
+
+let manualVersions = ["v12.0.0", "v11.0.0"]
+let reactManualVersions = ["latest", "v0.10.0", "v0.11.0"]
+
+let manualDocsDirectory = "pages/docs/manual"
+let reactDocsDirectory = "pages/docs/react"
+
+let manualLlmsDirectory = "public/llms/manual"
+let reactLlmsDirectory = "public/llms/react"
+
+processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory)
+processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory)

src/bindings/Node.res

@@ -20,10 +20,13 @@ module Process = {
 
 module Fs = {
   @module("fs") external readFileSync: string => string = "readFileSync"
+  @module("fs") external readFileSync2: (string, string) => string = "readFileSync"
   @module("fs") external readdirSync: string => array<string> = "readdirSync"
   @module("fs") external writeFileSync: (string, string) => unit = "writeFileSync"
+  @module("fs") external appendFileSync: (string, string, string) => unit = "appendFileSync"
   @module("fs") external existsSync: string => bool = "existsSync"
   @module("fs") external mkdirSync: string => unit = "mkdirSync"
+  @module("fs") external statSync: string => {.."isDirectory": unit => bool} = "statSync"
 }
 
 module Buffer = {
@@ -34,4 +37,4 @@ module Buffer = {
 module ChildProcess = {
   @module("child_process")
   external execSync: string => Buffer.t = "execSync"
-}
+}