Pull READMEs out of full index

Aside from index.json, the bundler creates a second _full index,_ which is an implementation detail of typst.app. This index has grown to 18MB due to the READMEs included within. This makes it annoying to work with:

- Our CDN, Azure Front Door, does not automatically compress files larger than 8MB
- Downloads from JavaScript and WebKit spuriously fail due to keepalives
- The file occupies a relatively large amount of memory once loaded

Instead, we factor out the READMEs into their own folder that can be loaded on demand.

The new full index without the README field is called `index.extra.json`. To prevent disruption of running infrastructure, `index.full.json` is retained in storage but no longer updated. This exception can be removed soon. The new `readmes` folder is also an implementation detail of typst.app and should not be used by third parties.

Author: reknih

.github/workflows/deploy.yml

@@ -44,5 +44,5 @@ jobs:
             --cache-control 'public, max-age=7776000' --put-md5
 
           azcopy_v10 cp --recursive ./dist/* "https://$ACCOUNT_NAME.blob.core.windows.net/\$web" \
-            --overwrite true --include-pattern '*.json' \
+            --overwrite true --include-pattern '*.json' --exclude-pattern '*index.full.json' \
             --cache-control 'must-revalidate' --put-md5

bundler/src/main.rs

@@ -27,6 +27,7 @@ use self::timestamp::determine_timestamps;
 
 const DIST: &str = "dist";
 const THUMBS_DIR: &str = "thumbnails";
+const READMES_DIR: &str = "readmes";
 
 struct Config {
     out_dir: PathBuf,
@@ -84,6 +85,8 @@ fn main() -> anyhow::Result<()> {
         let mut package_errors = vec![];
         fs::create_dir_all(namespace_dir.join(THUMBS_DIR))
             .context("could not create output directory")?;
+        fs::create_dir(namespace_dir.join(READMES_DIR))
+            .context("could not create directory for READMEs")?;
 
         for entry in walkdir::WalkDir::new(&path).min_depth(2).max_depth(2) {
             let entry = entry.with_context(|| {
@@ -142,16 +145,42 @@ fn main() -> anyhow::Result<()> {
         index.sort_by_key(|info| (info.package.name.clone(), info.package.version));
 
         println!("Writing index.");
+
+        // This index is allowed to be used by third parties: Both the Typst CLI
+        // and the web app download this index. However, we give no stability
+        // guarantees.
         fs::write(
             namespace_dir.join("index.json"),
             serde_json::to_vec(&index.iter().map(IndexPackageInfo::from).collect::<Vec<_>>())
                 .context("serialization of compact package index failed")?,
         )?;
+
+        // The index.extra.json is an implementation detail of typst.app. This
+        // file should not be used by third parties: They should instead use
+        // index.json.
         fs::write(
-            namespace_dir.join("index.full.json"),
-            serde_json::to_vec(&index).context("serialization of full package index failed")?,
+            namespace_dir.join("index.extra.json"),
+            serde_json::to_vec(
+                &index
+                    .iter()
+                    .map(ExtraIndexPackageInfo::from)
+                    .collect::<Vec<_>>(),
+            )
+            .context("serialization of extra package index failed")?,
         )?;
 
+        println!("Writing READMEs.");
+
+        let readme_dir = namespace_dir.join(READMES_DIR);
+
+        // The README files are an implementation detail of typst.app. They
+        // should not be used by third parties.
+        for item in &index {
+            let path =
+                readme_dir.join(format!("{}-{}.md", item.package.name, item.package.version));
+            fs::write(path, &item.readme)?;
+        }
+
         if !package_errors.is_empty() {
             namespace_errors.push((namespace.to_string(), package_errors));
         }

bundler/src/model.rs

@@ -28,7 +28,7 @@ impl From<&FullIndexPackageInfo> for IndexPackageInfo {
 /// A parsed package manifest + extra metadata.
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
-pub struct FullIndexPackageInfo {
+pub struct ExtraIndexPackageInfo {
     /// The package information from the package manifest.
     #[serde(flatten)]
     pub package: PackageInfo,
@@ -37,6 +37,32 @@ pub struct FullIndexPackageInfo {
     pub template: Option<TemplateInfo>,
     /// The compressed archive size in bytes.
     pub size: usize,
+    /// Release time of this version of the package.
+    pub updated_at: u64,
+    /// Release time of the first version of this package.
+    pub released_at: u64,
+}
+
+impl From<&FullIndexPackageInfo> for ExtraIndexPackageInfo {
+    fn from(info: &FullIndexPackageInfo) -> Self {
+        ExtraIndexPackageInfo {
+            package: info.package.clone(),
+            template: info.template.clone(),
+            size: info.size,
+            updated_at: info.updated_at,
+            released_at: info.released_at,
+        }
+    }
+}
+
+/// A parsed package manifest + extra metadata and the Readme.
+pub struct FullIndexPackageInfo {
+    /// The package information from the package manifest.
+    pub package: PackageInfo,
+    /// The template information from the package manifest.
+    pub template: Option<TemplateInfo>,
+    /// The compressed archive size in bytes.
+    pub size: usize,
     /// The unsanitized README markdown.
     pub readme: String,
     /// Release time of this version of the package.