Adds 'spk repo index' subcommand for index generation and updates.

Adds --use-indexes and --no-indexes flags to repository.
Updates resolvo solver to get global variables data from an indexed repository.

Signed-off-by: David Gilligan-Cook <dcook@imageworks.com>

Author: dcookspi

Cargo.lock

@@ -17,5 +17,6 @@ miette = { workspace = true, features = ["fancy"] }
 async-trait = { workspace = true }
 clap = { workspace = true }
 spk-cli-common = { workspace = true }
+spk-schema = { workspace = true }
 spk-storage = { workspace = true }
 tracing = { workspace = true }

crates/spk-cli/cmd-repo/Cargo.toml

@@ -2,10 +2,14 @@
 // SPDX-License-Identifier: Apache-2.0
 // https://github.com/spkenv/spk
 
+use std::str::FromStr;
+use std::time::Instant;
+
 use clap::{Args, Subcommand};
 use miette::{Context, Result};
-use spk_cli_common::{CommandArgs, Run};
-use spk_storage as storage;
+use spk_cli_common::{CommandArgs, Run, flags};
+use spk_schema::VersionIdent;
+use spk_storage::{self as storage, FlatBufferRepoIndex, RepositoryIndexMut};
 use storage::Repository;
 
 /// Perform repository-level actions and maintenance
@@ -45,19 +49,104 @@ pub enum RepoCommand {
         #[clap(name = "REPO")]
         repo: String,
     },
+    /// Generate an index for a repository
+    Index {
+        #[clap(flatten)]
+        repos: flags::Repositories,
+
+        /// Package/version of a published package to update in an
+        /// existing index.
+        ///
+        /// Other packages in the index will remain unchanged. Without
+        /// this the full index will be constructed from scratch. If
+        /// the repo does not have an index, a full index will be
+        /// constructed from scratch if possible.
+        ///
+        /// This option is only supported for flatbuffer indexes.
+        #[clap(long, name = "PACKAGE/VERSION")]
+        update: Option<String>,
+    },
 }
 
 impl RepoCommand {
     pub async fn run(&mut self) -> Result<i32> {
-        let repo = match &self {
-            Self::Upgrade { repo } => repo,
-        };
-        let repo = match repo.as_str() {
-            "local" => storage::local_repository().await?,
-            _ => storage::remote_repository(repo).await?,
-        };
-        let status = repo.upgrade().await.wrap_err("Upgrade failed")?;
-        tracing::info!("{}", status);
-        Ok(1)
+        match &self {
+            // spk repo upgrade ...
+            Self::Upgrade { repo: repo_name } => {
+                let repo = match repo_name.as_str() {
+                    "local" => storage::local_repository().await?,
+                    _ => storage::remote_repository(repo_name).await?,
+                };
+
+                let status = repo.upgrade().await.wrap_err("Upgrade failed")?;
+                tracing::info!("{}", status);
+                Ok(1)
+            }
+
+            // spk repo index ...
+            Self::Index { repos, update } => {
+                // Generate or update an index a repo. The repo must
+                // be the underlying repo and not an indexed repo. So
+                // this disables index use for this command regardless
+                // of config or command line flags.
+                flags::disable_index_use();
+                let repos = repos.get_repos_for_non_destructive_operation().await?;
+                if repos.len() != 1 {
+                    tracing::error!(
+                        "{} repos specified, Can only index one repo at a time. Please specify a single repo.",
+                        repos.len()
+                    );
+                    return Ok(2);
+                }
+
+                if let Some(package_version) = update {
+                    // Update the existing index for the given package/version
+                    let start = Instant::now();
+                    let version_ident = VersionIdent::from_str(package_version)?;
+                    let mut was_full_index = String::from("");
+
+                    // Load the current index for this repo now
+                    let repo = &repos[0].1;
+                    match FlatBufferRepoIndex::from_repo_file(repo).await {
+                        Ok(current_index) => {
+                            let repo = &repos[0].1;
+                            current_index
+                                .update_repo_with_package_version(repo, &version_ident)
+                                .await?
+                        }
+                        Err(err) => {
+                            // There isn't an existing index, so generate one from scratch that
+                            // will also include the update package version.
+                            // TODO: could also just error out and say run a full index first,
+                            // or ask the user "are you sure?" before continuing.
+                            tracing::warn!("Failed to load flatbuffer index: {err}");
+                            tracing::warn!("No current index to update. Creating a full index ...");
+                            FlatBufferRepoIndex::index_repo(&repos).await?;
+                            was_full_index =
+                                " [no previous index, so a full index was created]".to_string()
+                        }
+                    };
+
+                    tracing::info!(
+                        "Index update for '{package_version}' in '{}' repo completed in: {} secs{was_full_index}",
+                        repo.name(),
+                        start.elapsed().as_secs_f64()
+                    );
+                } else {
+                    // Generate a full index from scratch
+                    let start = Instant::now();
+                    FlatBufferRepoIndex::index_repo(&repos).await?;
+
+                    let repo = &repos[0].1;
+                    tracing::info!(
+                        "Index generation for '{}' repo completed in: {} secs",
+                        repo.name(),
+                        start.elapsed().as_secs_f64()
+                    );
+                }
+
+                Ok(0)
+            }
+        }
     }
 }

crates/spk-cli/cmd-repo/src/cmd_repo.rs

@@ -7,9 +7,11 @@ mod variant;
 use std::collections::HashSet;
 use std::convert::From;
 use std::sync::Arc;
+use std::sync::atomic::AtomicBool;
 
 use clap::{Args, ValueEnum, ValueHint};
 use miette::{Context, IntoDiagnostic, Result, bail, miette};
+use once_cell::sync::Lazy;
 use solve::{
     DEFAULT_SOLVER_RUN_FILE_PREFIX,
     DecisionFormatter,
@@ -45,6 +47,7 @@ use spk_solve as solve;
 #[cfg(feature = "statsd")]
 use spk_solve::{SPK_RUN_TIME_METRIC, get_metrics_client};
 use spk_storage as storage;
+use spk_storage::IndexedRepository;
 use spk_workspace::{FindOrLoadPackageTemplateError, FindPackageTemplateError};
 pub use variant::{Variant, VariantBuildStatus, VariantLocation};
 
@@ -61,6 +64,12 @@ static SPK_SOLVER_OUTPUT_TO_DIR: &str = "SPK_SOLVER_OUTPUT_TO_DIR";
 static SPK_SOLVER_OUTPUT_TO_DIR_MIN_VERBOSITY: &str = "SPK_SOLVER_OUTPUT_TO_DIR_MIN_VERBOSITY";
 static SPK_SOLVER_OUTPUT_FILE_PREFIX: &str = "SPK_SOLVER_OUTPUT_FILE_PREFIX";
 
+static DISABLE_INDEX_USE: Lazy<AtomicBool> = Lazy::new(|| AtomicBool::new(false));
+
+pub fn disable_index_use() {
+    DISABLE_INDEX_USE.store(true, std::sync::atomic::Ordering::Release);
+}
+
 #[derive(Args, Clone)]
 pub struct Runtime {
     /// Reconfigure the current spfs runtime (useful for speed and debugging)
@@ -1052,6 +1061,20 @@ pub struct Repositories {
     /// per-job or per-show repos.
     #[clap(long)]
     pub wrap_origin: Option<std::path::PathBuf>,
+
+    /// Get the package data from the repo indexes instead of the
+    /// repos. This only applies to non-destructive repo operations.
+    /// This can be configured as the default in spk's config file, or
+    /// on a per-repo basis in spk's config file.
+    #[clap(long)]
+    pub use_indexes: bool,
+
+    /// Do not get the package data from the repo index, always use
+    /// the repo instead. This only applies to non-destructive repo
+    /// operations. This option can be configured as the default in
+    /// spk's config file.
+    #[clap(long, conflicts_with = "use_indexes")]
+    pub no_indexes: bool,
 }
 
 impl Repositories {
@@ -1152,8 +1175,39 @@ impl Repositories {
             repos.push(("local".into(), repo.into()));
         }
         if self.local_repo_only {
+            // Local repo only case does not use indexes because they
+            // are typically small and not indexed. If local repos
+            // became large and were indexed, this might change.
             return Ok(repos);
         }
+
+        // Check whether using the indexes for the repos is enabled or not
+        let disable_all_index_use = DISABLE_INDEX_USE.load(std::sync::atomic::Ordering::Relaxed);
+
+        // Get the overrides from the command line flags
+        let use_index_cli_override = if self.use_indexes || self.no_indexes {
+            if self.no_indexes {
+                // Don't use any indexes
+                Some(false)
+            } else {
+                // Use indexes, if the command line flag is given to use them
+                Some(self.use_indexes)
+            }
+        } else {
+            // No command line index flags given, so there's no index override
+            None
+        };
+
+        // Get the overall config setting, which includes env var
+        // setting for it, but not the per repo settings. Those are
+        // checked below as repos are created.
+        let config = spk_config::get_config()?;
+        let all_use_index_from_config = config.solver.use_indexes;
+        tracing::debug!(
+            "Disable all indexes: {disable_all_index_use}, Cli use all indexes: {use_index_cli_override:?}, Config use all indexes: {all_use_index_from_config}"
+        );
+
+        // Add the enabled repos
         for (name, ts, is_default_origin) in enabled
             .into_iter()
             .map(|(name, ts)| (name, ts, false))
@@ -1190,7 +1244,57 @@ impl Repositories {
             if let Some(ts) = ts.as_ref().or(self.when.as_ref()) {
                 repo.pin_at_time(ts);
             }
-            repos.push((name.into(), repo.into()));
+
+            // Decide whether to use an index for this repository based on
+            // the various settings: all repos, per repo, and overrides.
+            let use_index = if disable_all_index_use {
+                // Disable all take precedence over everything
+                tracing::debug!("All index use disabled");
+                false
+            } else if let Some(use_indexes) = use_index_cli_override {
+                // Otherwise the all indexes command line flag takes
+                // precedence over what was in the config file
+                tracing::debug!("A cli-based use index override was given as: {use_indexes}");
+                use_indexes
+            } else {
+                // Otherwise use this specific repository's config setting, if any
+                match config.repositories.get(name) {
+                    Some(r_config) => {
+                        tracing::debug!("Using index for '{name}' repo");
+                        r_config.use_index
+                    }
+                    // And last use the all indexes env var and config
+                    // file setting.
+                    None => {
+                        tracing::debug!("Using config settings for all indexes");
+                        all_use_index_from_config
+                    }
+                }
+            };
+            tracing::debug!("Using index for '{name}': {use_index}");
+
+            if use_index {
+                // Use the index for this repo, if there is one,
+                // otherwise use the repo itself.
+                tracing::debug!("Using a repo index for '{name}'");
+                let indexed_repo = match IndexedRepository::load_from_repo(Arc::new(
+                    repo.clone().into(),
+                ))
+                .await
+                {
+                    Ok(ir) => ir.into(),
+                    Err(_err) => {
+                        tracing::warn!(
+                            "Failed to load index for '{name}' repo, falling back to the repo itself"
+                        );
+                        repo.into()
+                    }
+                };
+                repos.push((name.to_string(), indexed_repo));
+            } else {
+                tracing::debug!("Not using a repo index for '{name}' repo");
+                repos.push((name.to_string(), repo.into()));
+            }
         }
         Ok(repos)
     }

crates/spk-cli/common/src/flags.rs

@@ -67,6 +67,8 @@ async fn test_get_solver_with_host_options(
             disable_repo: Default::default(),
             when: None,
             wrap_origin: None,
+            use_indexes: false,
+            no_indexes: false,
         },
         decision_formatter_settings: DecisionFormatterSettings {
             time: Default::default(),

crates/spk-cli/common/src/flags_test.rs

@@ -781,6 +781,11 @@ impl View {
 
     /// Display the contents of a package spec
     fn print_build_spec(&self, package_spec: Arc<Spec>) -> Result<i32> {
+        // TODO: does not handle packages from indexes. This will
+        // crash if --use-indexes was used. Those packages would need
+        // to be converted to something that could be serialized, or
+        // pieces extracted individually from the index packages, for
+        // this to work.
         match &self.format.clone().unwrap_or_default() {
             OutputFormat::Yaml => serde_yaml::to_writer(std::io::stdout(), &*package_spec)
                 .into_diagnostic()

crates/spk-cli/group4/src/cmd_view.rs

@@ -21,7 +21,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::sync::Arc;
 use std::time::Instant;
 
-use pkg_request_version_set::{SpkSolvable, SyntheticComponent};
+use pkg_request_version_set::{SpkSolvable, SyntheticComponent, VarValue};
 use spk_provider::SpkProvider;
 use spk_schema::ident::{
     InclusionPolicy,
@@ -36,7 +36,7 @@ use spk_schema::ident::{
     VarRequest,
 };
 use spk_schema::ident_component::Component;
-use spk_schema::name::PkgNameBuf;
+use spk_schema::name::{OptNameBuf, PkgNameBuf};
 use spk_schema::prelude::{HasVersion, Named, Versioned};
 use spk_schema::version_range::VersionFilter;
 use spk_schema::{OptionMap, Package, Spec};
@@ -146,7 +146,28 @@ impl Solver {
         self.build_from_source_trail = trail;
     }
 
-    pub async fn solve(&self) -> Result<Solution> {
+    pub async fn solve(&mut self) -> Result<Solution> {
+        let mut known_global_vars: HashMap<OptNameBuf, HashSet<VarValue>> = Default::default();
+
+        // Gather the global vars from any indexed repos and use them
+        // to prime the known global vars cache.
+        for repo in self.repos.iter() {
+            if let RepositoryHandle::Indexed(indexed_repo) = &**repo {
+                let start_gv = Instant::now();
+                for (name, values) in indexed_repo.get_global_var_values().into_iter() {
+                    let entry = known_global_vars.entry(name).or_default();
+                    for v in values {
+                        entry.insert(VarValue::Owned(v));
+                    }
+                }
+                tracing::debug!(
+                    "Resolvo: gathered global var from '{}' in: {} secs",
+                    repo.name(),
+                    start_gv.elapsed().as_secs_f64()
+                );
+            }
+        }
+
         let repos = self.repos.clone();
         let requests = self.requests.clone();
         let binary_only = self.binary_only;
@@ -155,6 +176,7 @@ impl Solver {
         let solvables = tokio::task::spawn_blocking(move || {
             let mut provider = Some(SpkProvider::new(
                 repos.clone(),
+                known_global_vars.clone(),
                 binary_only,
                 build_from_source_trail,
             ));