Deal with forever increasing number of packs when pulling crates.io-index (#258)

Author: paolobarbolini

Cargo.lock

@@ -18,12 +18,13 @@ actix-web = "4"
 actix-web-lab = "0.24"
 anyhow = "1"
 crates-index = { version = "3", default-features = false, features = ["git", "git-https-reqwest"] }
+# to be kept in sync with the version used by `crates-index`
+gix = { version = "0.71", default-features = false }
 derive_more = { version = "2", features = ["display", "error", "from"] }
 dotenvy = "0.15"
 either = "1.12"
 font-awesome-as-a-crate = "0.3"
 futures-util = { version = "0.3", default-features = false, features = ["std"] }
-error_reporter = "1"
 indexmap = { version = "2", features = ["serde"] }
 lru_time_cache = "0.11"
 maud = "0.27"

Cargo.toml

@@ -1,6 +1,6 @@
-use std::{sync::Arc, time::Duration};
+use std::{fs, sync::Arc, time::Duration};
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use crates_index::{Crate, GitIndex};
 use parking_lot::Mutex;
 use tokio::{
@@ -12,19 +12,23 @@ use crate::models::crates::CrateName;
 
 #[derive(Clone)]
 pub struct ManagedIndex {
-    index: Arc<Mutex<GitIndex>>,
+    index: Arc<Mutex<Option<GitIndex>>>,
 }
 
 impl ManagedIndex {
     pub fn new() -> Self {
         // the index path is configurable through the `CARGO_HOME` env variable
-        let index = Arc::new(Mutex::new(GitIndex::new_cargo_default().unwrap()));
+        let index = Arc::new(Mutex::new(Some(GitIndex::new_cargo_default().unwrap())));
 
         Self { index }
     }
 
     pub fn crate_(&self, crate_name: CrateName) -> Option<Crate> {
-        self.index.lock().crate_(crate_name.as_ref())
+        self.index
+            .lock()
+            .as_ref()
+            .expect("ManagedIndex is poisoned")
+            .crate_(crate_name.as_ref())
     }
 
     pub async fn refresh_at_interval(&self, update_interval: Duration) {
@@ -34,21 +38,101 @@ impl ManagedIndex {
         loop {
             if let Err(err) = self.refresh().await {
                 tracing::error!(
-                    "failed refreshing the crates.io-index, the operation will be retried: {}",
-                    error_reporter::Report::new(err),
+                    "failed refreshing the crates.io-index, the operation will be retried: {err:#}"
                 );
             }
             update_interval.tick().await;
         }
     }
 
-    async fn refresh(&self) -> Result<(), crates_index::Error> {
-        let index = Arc::clone(&self.index);
+    async fn refresh(&self) -> Result<()> {
+        let this_index = Arc::clone(&self.index);
 
-        spawn_blocking(move || index.lock().update())
-            .await
-            .expect("blocking index update task should never panic")?;
+        spawn_blocking(move || {
+            let mut index = this_index.lock();
+            let git_index = index.as_mut().context("ManagedIndex is poisoned")?;
+
+            match git_index.update() {
+                Ok(()) => Ok(()),
+                Err(err) => match current_entries(&err) {
+                    Some(..4096) => {
+                        tracing::info!(
+                            "Reopening crates.io-index to make gix expand the internal slotmap"
+                        );
+                        *git_index = GitIndex::with_path(git_index.path(), git_index.url())
+                            .context("could not reopen git index")?;
+                        git_index
+                            .update()
+                            .context("failed to update crates.io-index after `git gc`")
+                    }
+                    Some(4096..) => {
+                        tracing::info!(
+                            "Cloning a new crates.io-index and replacing it with the current one"
+                        );
+                        let path = git_index.path().to_owned();
+                        let url = git_index.url().to_owned();
+
+                        // Avoid keeping the index locked for too long
+                        drop(index);
+
+                        // Clone the new index
+                        let mut tmp_path = path.clone();
+                        tmp_path.as_mut_os_string().push(".new");
+                        if tmp_path.try_exists()? {
+                            fs::remove_dir_all(&tmp_path)?;
+                        }
+                        let new_index = GitIndex::with_path(&tmp_path, &url)
+                            .context("could not clone new git index")?;
+
+                        // Swap the old index with the new one
+                        drop(new_index);
+
+                        let mut index = this_index.lock();
+                        *index = None;
+                        // NOTE: if any of the following operations fail,
+                        // the index is poisoned
+                        fs::remove_dir_all(&path)?;
+                        fs::rename(tmp_path, &path)?;
+
+                        *index = Some(
+                            GitIndex::with_path(path, url).context("could not reopen git index")?,
+                        );
+                        Ok(())
+                    }
+                    None => {
+                        Err(anyhow::Error::from(err).context("failed to update crates.io-index"))
+                    }
+                },
+            }
+        })
+        .await
+        .expect("blocking index update task should never panic")?;
 
         Ok(())
     }
 }
+
+fn current_entries(err: &crates_index::Error) -> Option<usize> {
+    let crates_index::Error::Git(err) = err else {
+        return None;
+    };
+    let crates_index::error::GixError::Fetch(err) = err else {
+        return None;
+    };
+    let gix::remote::fetch::Error::UpdateRefs(err) = err else {
+        return None;
+    };
+    let gix::remote::fetch::refs::update::Error::FindObject(gix::object::find::Error(err)) = err
+    else {
+        return None;
+    };
+    let err = err.downcast_ref::<gix::odb::store::find::Error>()?;
+    let gix::odb::store::find::Error::LoadIndex(err) = err else {
+        return None;
+    };
+    let gix::odb::store::load_index::Error::InsufficientSlots { current, needed } = err else {
+        return None;
+    };
+
+    Some(*current + *needed)
+}