Adds new Indexed repository type, related RepositoryIndex traits, and flabuffers index and configuration

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

Author: dcookspi

Cargo.lock

@@ -77,6 +77,7 @@ indicatif = "0.17.8"
 is_default_derive_macro = { path = "crates/is_default_derive_macro" }
 itertools = "0.14"
 libc = "0.2.172"
+memmap2 = "0.9.10"
 miette = "7.0"
 nix = { version = "0.29", features = ["mount", "sched", "user"] }
 nom = "7.1"

Cargo.toml

@@ -40,7 +40,9 @@ async fn test_archive_create_parents(#[case] solver: SolverImpl) {
     let filename = rt.tmpdir.path().join("deep/nested/path/archive.spk");
     let repo = match &*rt.tmprepo {
         spk_solve::RepositoryHandle::SPFS(repo) => repo,
-        spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => {
+        spk_solve::RepositoryHandle::Mem(_)
+        | spk_solve::RepositoryHandle::Runtime(_)
+        | spk_solve::RepositoryHandle::Indexed(_) => {
             panic!("only spfs repositories are supported")
         }
     };

crates/spk-build/src/archive_test.rs

@@ -52,7 +52,9 @@ impl Run for Export {
             .iter()
             .map(|repo| match &**repo {
                 storage::RepositoryHandle::SPFS(repo) => Ok(repo),
-                storage::RepositoryHandle::Mem(_) | storage::RepositoryHandle::Runtime(_) => {
+                storage::RepositoryHandle::Mem(_)
+                | storage::RepositoryHandle::Runtime(_)
+                | storage::RepositoryHandle::Indexed(_) => {
                     bail!("Only spfs repositories are supported")
                 }
             })

crates/spk-cli/group3/src/cmd_export.rs

@@ -42,7 +42,9 @@ async fn test_archive_io(#[case] solver: SolverImpl) {
     filename.ensure();
     let repo = match &*rt.tmprepo {
         spk_solve::RepositoryHandle::SPFS(repo) => repo,
-        spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => {
+        spk_solve::RepositoryHandle::Mem(_)
+        | spk_solve::RepositoryHandle::Runtime(_)
+        | spk_solve::RepositoryHandle::Indexed(_) => {
             panic!("only spfs repositories are supported")
         }
     };

crates/spk-cli/group3/src/cmd_import_test.rs

@@ -87,6 +87,36 @@ pub struct Solver {
 
     /// Name of the solver whose output to show  when multiple solvers are being run.
     pub solver_to_show: String,
+
+    /// Whether to get the solver to use repository indexes instead of
+    /// the repository directly.
+    pub use_indexes: bool,
+
+    /// What kinds of index to use. Only applies if there is more than
+    /// one kinds of index available for the repositories. The default
+    /// is 'flatb', a flatbuffers file based index.
+    pub index_kind: String,
+
+    /// Whether to validate flatbuffers index data before using it.
+    /// Validating is safer but adds some overhead at the start of a
+    /// solve when using an index.
+    pub verify_flatbuffers_index_before_use: bool,
+}
+
+// /// A map of repositories, name to settings
+// #[derive(Clone, Default, Debug, Deserialize, Serialize)]
+// #[serde(default)]
+// pub struct Repositories {
+//     pub repositories: HashMap<String, Repository>,
+// }
+
+/// The settings for a single repository
+#[derive(Clone, Default, Debug, Deserialize, Serialize)]
+#[serde(default)]
+pub struct Repository {
+    /// Whether to use an index with this repository, if one is
+    /// available.
+    pub use_index: bool,
 }
 
 #[derive(Clone, Default, Debug, Deserialize, Serialize)]
@@ -142,6 +172,7 @@ pub struct Config {
     // with environment variables.
     pub sentry: Sentry,
     pub solver: Solver,
+    pub repositories: HashMap<String, Repository>,
     pub statsd: Statsd,
     pub metadata: Metadata,
     pub cli: Cli,