don't compress rustdoc json twice

Author: syphar

src/docbuilder/rustwide_builder.rs

@@ -13,8 +13,8 @@ use crate::docbuilder::Limits;
 use crate::error::Result;
 use crate::repositories::RepositoryStatsUpdater;
 use crate::storage::{
-    CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, rustdoc_archive_path,
-    rustdoc_json_path, source_archive_path,
+    RustdocJsonFormatVersion, get_file_list, rustdoc_archive_path, rustdoc_json_path,
+    source_archive_path,
 };
 use crate::utils::{
     CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error,
@@ -909,21 +909,11 @@ impl RustwideBuilder {
                 .context("couldn't parse rustdoc json to find format version")?
         };
 
-        let compressed_json: Vec<u8> = {
-            let _span =
-                info_span!("compress_json", file_size = json_filename.metadata()?.len()).entered();
-
-            compress(
-                BufReader::new(File::open(&json_filename)?),
-                CompressionAlgorithm::Zstd,
-            )?
-        };
-
         for format_version in [format_version, RustdocJsonFormatVersion::Latest] {
             let _span = info_span!("store_json", %format_version).entered();
             let path = rustdoc_json_path(name, version, target, format_version);
 
-            self.storage.store_one(&path, compressed_json.clone())?;
+            self.storage.store_path(&path, &json_filename)?;
             self.storage.set_public_access(&path, true)?;
         }
 
@@ -1495,10 +1485,10 @@ mod tests {
                         .collect();
                     json_files.sort();
                     assert!(json_files[0].starts_with(&format!("empty-library_1.0.0_{target}_")));
-                    assert!(json_files[0].ends_with(".json.zst"));
+                    assert!(json_files[0].ends_with(".json"));
                     assert_eq!(
                         json_files[1],
-                        format!("empty-library_1.0.0_{target}_latest.json.zst")
+                        format!("empty-library_1.0.0_{target}_latest.json")
                     );
 
                     if target == &default_target {

src/storage/mod.rs

@@ -24,7 +24,8 @@ use mime::Mime;
 use path_slash::PathExt;
 use serde_with::{DeserializeFromStr, SerializeDisplay};
 use std::{
-    fmt, fs,
+    fmt,
+    fs::{self, File},
     io::{self, BufReader},
     num::ParseIntError,
     ops::RangeInclusive,
@@ -569,6 +570,33 @@ impl AsyncStorage {
         Ok(alg)
     }
 
+    #[instrument(skip(self))]
+    pub(crate) async fn store_path(
+        &self,
+        target_path: impl Into<String> + std::fmt::Debug,
+        source_path: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<CompressionAlgorithm> {
+        let target_path = target_path.into();
+        let source_path = source_path.as_ref();
+
+        let alg = CompressionAlgorithm::default();
+        let content = compress(BufReader::new(File::open(source_path)?), alg)?;
+
+        let mime = detect_mime(&target_path).to_owned();
+
+        self.store_inner(vec![Blob {
+            path: target_path,
+            mime,
+            content,
+            compression: Some(alg),
+            // this field is ignored by the backend
+            date_updated: Utc::now(),
+        }])
+        .await?;
+
+        Ok(alg)
+    }
+
     async fn store_inner(&self, batch: Vec<Blob>) -> Result<()> {
         match &self.backend {
             StorageBackend::Database(db) => db.store_batch(batch).await,
@@ -777,6 +805,18 @@ impl Storage {
         self.runtime.block_on(self.inner.store_one(path, content))
     }
 
+    // Store file into the backend at the given path (also used to detect mime type), returns the
+    // chosen compression algorithm
+    #[instrument(skip(self))]
+    pub(crate) fn store_path(
+        &self,
+        target_path: impl Into<String> + std::fmt::Debug,
+        source_path: impl AsRef<Path> + std::fmt::Debug,
+    ) -> Result<CompressionAlgorithm> {
+        self.runtime
+            .block_on(self.inner.store_path(target_path, source_path))
+    }
+
     /// sync wrapper for the list_prefix function
     /// purely for testing purposes since it collects all files into a Vec.
     #[cfg(test)]
@@ -843,7 +883,7 @@ pub(crate) fn rustdoc_json_path(
     format_version: RustdocJsonFormatVersion,
 ) -> String {
     format!(
-        "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json.zst"
+        "rustdoc-json/{name}/{version}/{target}/{name}_{version}_{target}_{format_version}.json"
     )
 }
 

src/web/rustdoc.rs

@@ -3155,7 +3155,7 @@ mod test {
             web.assert_redirect_cached_unchecked(
                 &format!("/crate/dummy/{request_path_suffix}"),
                 &format!("https://static.docs.rs/rustdoc-json/dummy/{redirect_version}/{redirect_target}/\
-                    dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json.zst"),
+                    dummy_{redirect_version}_{redirect_target}_{redirect_format_version}.json"),
                 CachePolicy::ForeverInCdn,
                 &env.config(),
             )