fix archive index caching for source archives, don't ignore source fetch errors

Author: syphar

.sqlx/query-73bf03e72a4795b54f878d30c30d2895a2acfa491b5925b1e576de5e6b686b8b.json

@@ -201,18 +201,37 @@ impl AsyncStorage {
         })
     }
 
+    pub(crate) async fn source_file_exists(
+        &self,
+        name: &str,
+        version: &str,
+        latest_build_id: i32,
+        path: &str,
+        archive_storage: bool,
+    ) -> Result<bool> {
+        Ok(if archive_storage {
+            self.exists_in_archive(&source_archive_path(name, version), latest_build_id, path)
+                .await?
+        } else {
+            // Add sources prefix, name and version to the path for accessing the file stored in the database
+            let remote_path = format!("sources/{name}/{version}/{path}");
+            self.exists(&remote_path).await?
+        })
+    }
+
     #[context("fetching {path} from {name} {version} (archive: {archive_storage})")]
     pub(crate) async fn fetch_source_file(
         &self,
         name: &str,
         version: &str,
+        latest_build_id: i32,
         path: &str,
         archive_storage: bool,
     ) -> Result<Blob> {
         Ok(if archive_storage {
             self.get_from_archive(
                 &source_archive_path(name, version),
-                0, // we assume the source archive can't ever change for a release
+                latest_build_id,
                 path,
                 self.max_file_size_for(path),
                 None,
@@ -633,13 +652,34 @@ impl Storage {
         &self,
         name: &str,
         version: &str,
+        latest_build_id: i32,
         path: &str,
         archive_storage: bool,
     ) -> Result<Blob> {
-        self.runtime.block_on(
-            self.inner
-                .fetch_source_file(name, version, path, archive_storage),
-        )
+        self.runtime.block_on(self.inner.fetch_source_file(
+            name,
+            version,
+            latest_build_id,
+            path,
+            archive_storage,
+        ))
+    }
+
+    pub(crate) fn source_file_exists(
+        &self,
+        name: &str,
+        version: &str,
+        latest_build_id: i32,
+        path: &str,
+        archive_storage: bool,
+    ) -> Result<bool> {
+        self.runtime.block_on(self.inner.source_file_exists(
+            name,
+            version,
+            latest_build_id,
+            path,
+            archive_storage,
+        ))
     }
 
     pub(crate) fn rustdoc_file_exists(

.sqlx/query-8fdb5774a13ea9d9a70552395a4b275e859e91e439ddcd3f6be9e4cfa845f94d.json

@@ -257,6 +257,7 @@ impl CrateDetails {
             .fetch_source_file(
                 &self.name,
                 &self.version,
+                self.latest_build_id.unwrap_or(0),
                 "Cargo.toml",
                 self.archive_storage,
             )
@@ -282,7 +283,13 @@ impl CrateDetails {
         };
         for path in &paths {
             match storage
-                .fetch_source_file(&self.name, &self.version, path, self.archive_storage)
+                .fetch_source_file(
+                    &self.name,
+                    &self.version,
+                    self.latest_build_id.unwrap_or(0),
+                    path,
+                    self.archive_storage,
+                )
                 .await
             {
                 Ok(readme) => {

src/storage/mod.rs

@@ -9,7 +9,7 @@ use crate::{
     },
     AsyncStorage,
 };
-use anyhow::Result;
+use anyhow::{Context as _, Result};
 use axum::{extract::Path, headers::HeaderMapExt, response::IntoResponse, Extension};
 use serde::{Deserialize, Serialize};
 use std::{cmp::Ordering, sync::Arc};
@@ -225,8 +225,18 @@ pub(crate) async fn source_browser_handler(
         }
     };
 
-    let archive_storage = sqlx::query_scalar!(
-        "SELECT archive_storage
+    let row = sqlx::query!(
+        "SELECT
+            releases.archive_storage,
+            (
+                SELECT id
+                FROM builds
+                WHERE
+                    builds.rid = releases.id AND
+                    builds.build_status = TRUE
+                ORDER BY build_time DESC
+                LIMIT 1
+            ) AS latest_build_id
          FROM releases
          INNER JOIN crates ON releases.crate_id = crates.id
          WHERE
@@ -240,11 +250,30 @@ pub(crate) async fn source_browser_handler(
 
     // try to get actual file first
     // skip if request is a directory
-    let blob = if !path.ends_with('/') {
-        storage
-            .fetch_source_file(&name, &version, &path, archive_storage)
+    let blob = if !path.ends_with('/')
+        && storage
+            .source_file_exists(
+                &name,
+                &version,
+                row.latest_build_id.unwrap_or(0),
+                &path,
+                row.archive_storage,
+            )
             .await
-            .ok()
+            .context("error checking source file existence")?
+    {
+        Some(
+            storage
+                .fetch_source_file(
+                    &name,
+                    &version,
+                    row.latest_build_id.unwrap_or(0),
+                    &path,
+                    row.archive_storage,
+                )
+                .await
+                .context("error fetching source file")?,
+        )
     } else {
         None
     };