Merge pull request #467 from ealmloff/llms.txt

Add llms.txt generation for every page in the docsite.

Author: jkelleyrtp

Cargo.lock

@@ -95,6 +95,45 @@ debug = 0
 [profile.android-dev]
 inherits = "dev"
 
+[profile.wasm-release]
+inherits = "release"
+
+[profile.server-release]
+inherits = "release"
+
+[profile.ios-dev]
+inherits = "dev"
+
+[profile.ios-release]
+inherits = "release"
+
+[profile.android-release]
+inherits = "release"
+
+[profile.windows-dev]
+inherits = "dev"
+
+[profile.windows-release]
+inherits = "release"
+
+[profile.macos-dev]
+inherits = "dev"
+
+[profile.macos-release]
+inherits = "release"
+
+[profile.linux-dev]
+inherits = "dev"
+
+[profile.linux-release]
+inherits = "release"
+
+[profile.liveview-dev]
+inherits = "dev"
+
+[profile.liveview-release]
+inherits = "release"
+
 [profile.release.build-override]
 opt-level = 3
 codegen-units = 1

Cargo.toml

@@ -1,3 +1,5 @@
 [application]
 name = "docsite"
 
+[web.app]
+base_path = "docsite"

Dioxus.toml

@@ -71,3 +71,4 @@ server = [
 production = [
     "dioxus_docs_router/production",
 ]
+doc_test = []

packages/docsite/Cargo.toml

@@ -18,3 +18,4 @@ pub mod playground;
 pub use playground::*;
 pub mod search;
 pub use search::*;
+pub mod llms;

packages/docsite/src/components.rs

@@ -0,0 +1,46 @@
+pub fn generate_llms_txt() {
+    #[cfg(not(target_arch = "wasm32"))]
+    {
+        use crate::static_dir;
+        use crate::Route;
+        use dioxus::prelude::Routable;
+        use std::fmt::Display;
+
+        fn write_content_to_llm_txt(route: impl Display, content: &str) {
+            let route = route.to_string();
+            let route = route.trim_matches('/');
+            let (route, _) = route.split_once('#').unwrap_or((&route, ""));
+            let (route, _) = route.split_once('?').unwrap_or((&route, ""));
+            let path = static_dir().join(route).join("index").join("llms.txt");
+            _ = std::fs::create_dir_all(path.parent().unwrap());
+            std::fs::write(&path, content).unwrap_or_else(|err| {
+                panic!("Failed to write llms.txt to {}: {}", path.display(), err)
+            });
+        }
+        for route in crate::Route::static_routes() {
+            match route {
+                Route::Docs03 { child } => {
+                    let id = child.page_id();
+                    let content = dioxus_docs_router::docs::router_03::BookRoute::page_markdown(id);
+                    write_content_to_llm_txt(route, &content);
+                }
+                Route::Docs04 { child } => {
+                    let id = child.page_id();
+                    let content = dioxus_docs_router::docs::router_04::BookRoute::page_markdown(id);
+                    write_content_to_llm_txt(route, &content);
+                }
+                Route::Docs05 { child } => {
+                    let id = child.page_id();
+                    let content = dioxus_docs_router::docs::router_05::BookRoute::page_markdown(id);
+                    write_content_to_llm_txt(route, &content);
+                }
+                Route::Docs06 { child } => {
+                    let id = child.page_id();
+                    let content = dioxus_docs_router::docs::router_06::BookRoute::page_markdown(id);
+                    write_content_to_llm_txt(route, &content);
+                }
+                _ => {}
+            }
+        }
+    }
+}

packages/docsite/src/components/llms.rs

@@ -24,4 +24,4 @@ pub fn generate_search_index() {
             );
         }
     }
-}
+}

packages/docsite/src/components/search.rs

@@ -15,6 +15,7 @@ fn main() {
     // If we are just building the search index, we don't need to launch the app
     #[cfg(feature = "server")]
     if std::env::args().any(|arg| arg == "--generate-search-index") {
+        llms::generate_llms_txt();
         search::generate_search_index();
         return;
     }
@@ -215,7 +216,6 @@ impl Route {
     }
 }
 
-// todo - when we update to 0.6.0 we need to change this to return a Vec<String>
 #[cfg(feature = "fullstack")]
 #[server(endpoint = "static_routes")]
 async fn static_routes() -> Result<Vec<String>, ServerFnError> {

packages/docsite/src/main.rs

@@ -19,6 +19,21 @@ pub enum BookRoute {
     Chapter3 { section: Chapter3Section },
 }
 impl BookRoute {
+    /// Get the markdown for a page by its ID
+    pub const fn page_markdown(id: use_mdbook::mdbook_shared::PageId) -> &'static str {
+        match id.0 {
+            1usize => {
+                "# Roadmap & Feature-set\n\nThis feature set and roadmap can help you decide if what Dioxus can do today works for you.\n\nIf a feature that you need doesn't exist or you want to contribute to projects on the roadmap, feel free to get involved by [joining the discord](https://discord.gg/XgGxMSkvUM).\n\nGenerally, here's the status of each platform:\n\n* **Web**: Dioxus is a great choice for pure web-apps – especially for CRUD/complex apps. However, it does lack the ecosystem of React, so you might be missing a component library or some useful hook.\n\n* **SSR**: Dioxus is a great choice for pre-rendering, hydration, and rendering HTML on a web endpoint. Be warned – the VirtualDom is not (currently) `Send + Sync`.\n\n* **Desktop**: You can build very competent single-window desktop apps right now. However, multi-window apps require support from Dioxus core and are not ready.\n\n* **Mobile**: Mobile support is very young. You'll be figuring things out as you go and there are not many support crates for peripherals.\n\n* **LiveView**: LiveView support is very young. You'll be figuring things out as you go. Thankfully, none of it is too hard and any work can be upstreamed into Dioxus.\n\n````rust\nfn main() {\n  dioxus_rocks;\n}\n````\n\n## Features\n\n---\n\n|Feature|Status|Description|\n|-------|------|-----------|\n|Conditional Rendering|✅|if/then to hide/show component|\n|Map, Iterator|✅|map/filter/reduce to produce rsx!|\n|Keyed Components|✅|advanced diffing with keys|\n|Web|✅|renderer for web browser|\n|Desktop (webview)|✅|renderer for desktop|\n|Shared State (Context)|✅|share state through the tree|\n|Hooks|✅|memory cells in components|\n|SSR|✅|render directly to string|\n|Component Children|✅|cx.children() as a list of nodes|\n|Headless components|✅|components that don't return real elements|\n|Fragments|✅|multiple elements without a real root|\n|Manual Props|✅|Manually pass in props with spread syntax|\n|Controlled Inputs|✅|stateful wrappers around inputs|\n|CSS/Inline Styles|✅|syntax for inline styles/attribute groups|\n|Custom elements|✅|Define new element primitives|\n|Suspense|✅|schedule future render from future/promise|\n|Integrated error handling|✅|Gracefully handle errors with ? syntax|\n|NodeRef|✅|gain direct access to nodes|\n|Re-hydration|✅|Pre-render to HTML to speed up first contentful paint|\n|Jank-Free Rendering|✅|Large diffs are segmented across frames for silky-smooth transitions|\n|Effects|✅|Run effects after a component has been committed to render|\n|Portals|🛠|Render nodes outside of the traditional tree structure|\n|Cooperative Scheduling|🛠|Prioritize important events over non-important events|\n|Server Components|🛠|Hybrid components for SPA and Server|\n|Bundle Splitting|👀|Efficiently and asynchronously load the app|\n|Lazy Components|👀|Dynamically load the new components as the page is loaded|\n|1st class global state|✅|redux/recoil/mobx on top of context|\n|Runs natively|✅|runs as a portable binary w/o a runtime (Node)|\n|Subtree Memoization|✅|skip diffing static element subtrees|\n|High-efficiency templates|✅|rsx! calls are translated to templates on the DOM's side|\n|Compile-time correct|✅|Throw errors on invalid template layouts|\n|Heuristic Engine|✅|track component memory usage to minimize future allocations|\n|Fine-grained reactivity|👀|Skip diffing for fine-grain updates|\n\n* ✅ = implemented and working\n* 🛠 = actively being worked on\n* 👀 = not yet implemented or being worked on\n\n## Roadmap\n\nThese Features are planned for the future of Dioxus:\n\n### Core\n\n* [x] Release of Dioxus Core\n* [x] Upgrade documentation to include more theory and be more comprehensive\n* [x] Support for HTML-side templates for lightning-fast dom manipulation\n* [ ] Support for multiple renderers for same virtualdom (subtrees)\n* [ ] Support for ThreadSafe (Send + Sync)\n* [ ] Support for Portals\n\n### SSR\n\n* [x] SSR Support + Hydration\n* [ ] Integrated suspense support for SSR\n\n### Desktop\n\n* [ ] Declarative window management\n* [ ] Templates for building/bundling\n* [ ] Access to Canvas/WebGL context natively\n\n### Mobile\n\n* [ ] Mobile standard library\n  * [ ] GPS\n  * [ ] Camera\n  * [ ] filesystem\n  * [ ] Biometrics\n  * [ ] WiFi\n  * [ ] Bluetooth\n  * [ ] Notifications\n  * [ ] Clipboard\n* [ ] Animations\n\n### Bundling (CLI)\n\n* [x] Translation from HTML into RSX\n* [x] Dev server\n* [x] Live reload\n* [x] Translation from JSX into RSX\n* [ ] Hot module replacement\n* [ ] Code splitting\n* [ ] Asset macros\n* [ ] Css pipeline\n* [ ] Image pipeline\n\n### Essential hooks\n\n* [x] Router\n* [x] Global state management\n* [ ] Resize observer\n\n## Work in Progress\n\n### Build Tool\n\nWe are currently working on our own build tool called [Dioxus CLI](https://github.com/DioxusLabs/dioxus/tree/master/packages/cli) which will support:\n\n* an interactive TUI\n* on-the-fly reconfiguration\n* hot CSS reloading\n* two-way data binding between browser and source code\n* an interpreter for `rsx!`\n* ability to publish to github/netlify/vercel\n* bundling for iOS/Desktop/etc\n\n### Server Component Support\n\nWhile not currently fully implemented, the expectation is that LiveView apps can be a hybrid between Wasm and server-rendered where only portions of a page are \"live\" and the rest of the page is either server-rendered, statically generated, or handled by the host SPA.\n\n### Native rendering\n\nWe are currently working on a native renderer for Dioxus using WGPU called [Blitz](https://github.com/DioxusLabs/blitz/). This will allow you to build apps that are rendered natively for iOS, Android, and Desktop.\n\n## Internal Links\n\nInternal links like [this](./chapter_1.md) are typechecked and will fail to compile if the file is not found."
+            }
+            0usize => {
+                "# Liveview\n\nLiveview allows apps to *run* on the server and *render* in the browser. It uses WebSockets to communicate between the server and the browser.\n\nExamples:\n\n* [Axum Example](https://github.com/DioxusLabs/dioxus/tree/master/packages/liveview/examples/axum.rs)\n* [Salvo Example](https://github.com/DioxusLabs/dioxus/tree/master/packages/liveview/examples/salvo.rs)\n* [Warp Example](https://github.com/DioxusLabs/dioxus/tree/master/packages/liveview/examples/warp.rs)\n\n## Support\n\nLiveview is currently limited in capability when compared to the Web platform. Liveview apps run on the server in a native thread. This means that browser APIs are not available, so rendering WebGL, Canvas, etc is not as easy as the Web. However, native system APIs are accessible, so streaming, WebSockets, filesystem, etc are all viable APIs.\n\n## Setup\n\nFor this guide, we're going to show how to use Dioxus Liveview with [Axum](https://docs.rs/axum/latest/axum/).\n\nMake sure you have Rust and Cargo installed, and then create a new project:\n\n````shell\ncargo new --bin demo\ncd app\n````\n\nAdd Dioxus and the liveview renderer with the Axum feature as dependencies:\n\n````shell\ncargo add dioxus\ncargo add dioxus-liveview --features axum\n````\n\nNext, add all the Axum dependencies. This will be different if you're using a different Web Framework\n\n````\ncargo add tokio --features full\ncargo add axum\n````\n\nYour dependencies should look roughly like this:\n\n````toml\n[dependencies]\naxum = \"0.4.5\"\ndioxus = { version = \"*\" }\ndioxus-liveview = { version = \"*\", features = [\"axum\"] }\ntokio = { version = \"1.15.0\", features = [\"full\"] }\n````\n\n````rust@included_example.rs\nfn it_works() {}\n\n````\n\n````sh\n{\"timestamp\":\"   9.927s\",\"level\":\"INFO\",\"message\":\"Bundled app successfully!\",\"target\":\"dx::cli::bundle\"}\n{\"timestamp\":\"   9.927s\",\"level\":\"INFO\",\"message\":\"App produced 2 outputs:\",\"target\":\"dx::cli::bundle\"}\n{\"timestamp\":\"   9.927s\",\"level\":\"INFO\",\"message\":\"app - [target/dx/hot_dog/bundle/macos/bundle/macos/HotDog.app]\",\"target\":\"dx::cli::bundle\"}\n{\"timestamp\":\"   9.927s\",\"level\":\"INFO\",\"message\":\"dmg - [target/dx/hot_dog/bundle/macos/bundle/dmg/HotDog_0.1.0_aarch64.dmg]\",\"target\":\"dx::cli::bundle\"}\n{\"timestamp\":\"   9.927s\",\"level\":\"DEBUG\",\"json\":\"{\\\"BundleOutput\\\":{\\\"bundles\\\":[\\\"target/dx/hot_dog/bundle/macos/bundle/macos/HotDog.app\\\"]}}\"}\n````"
+            }
+            2usize => {
+                "# Assets\n\nSome assets:\n![some_external](https://avatars.githubusercontent.com/u/79236386?s=200&v=4)\n![some_local](/example-book/assets/logo.png)\n![some_local1](/example-book/assets/logo1.png)\n![some_local2](/example-book/assets/logo2.png)"
+            }
+            _ => panic!("Invalid page ID:"),
+        }
+    }
     pub fn sections(&self) -> &'static [use_mdbook::mdbook_shared::Section] {
         &self.page().sections
     }

packages/include_mdbook/packages/mdbook-gen-example/src/router.rs

@@ -14,7 +14,8 @@ serde_json = "1.0.96"
 macro_state = "0.2.0"
 convert_case = "0.6.0"
 postcard = { version = "1.0.4", features = ["use-std"] }
-pulldown-cmark = "0.9.3"
+pulldown-cmark = "0.13.0"
+pulldown-cmark-to-cmark = "21.0.0"
 syntect = { version = "5.2.0", features = ["plist-load"] }
 dioxus-rsx = { version = "0.6.0" }
 dioxus-autofmt = { version = "0.6.0" }