From 3af31316308f7aafbfd8d78622e41e276c159d55 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 18:46:31 +0200 Subject: [PATCH 01/11] add crates-io-env-vars --- crates/crates_io_env_vars/Cargo.toml | 15 ++ crates/crates_io_env_vars/README.md | 14 ++ crates/crates_io_env_vars/src/lib.rs | 244 +++++++++++++++++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 crates/crates_io_env_vars/Cargo.toml create mode 100644 crates/crates_io_env_vars/README.md create mode 100644 crates/crates_io_env_vars/src/lib.rs diff --git a/crates/crates_io_env_vars/Cargo.toml b/crates/crates_io_env_vars/Cargo.toml new file mode 100644 index 000000000..a3059a41f --- /dev/null +++ b/crates/crates_io_env_vars/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "crates_io_env_vars" +version = "0.0.0" +license = "MIT OR Apache-2.0" +edition = "2024" + +[lints] +workspace = true + +[dependencies] +anyhow = "=1.0.98" +dotenvy = "=0.15.7" + +[dev-dependencies] +claims = "=0.8.0" diff --git a/crates/crates_io_env_vars/README.md b/crates/crates_io_env_vars/README.md new file mode 100644 index 000000000..d30947dd9 --- /dev/null +++ b/crates/crates_io_env_vars/README.md @@ -0,0 +1,14 @@ +# crates_io_env_vars + +This package contains convenient wrappers for the `std::env::var()` function. + +These functions use the `dotenvy` crate to automatically load environment +variables from a `.env` file, if it exists. This is useful for development +environments, where you don't want to set all environment variables manually. + +There are also variants of the functions that make use of the `FromStr` trait to +automatically parse the environment variables into the desired types or fail +with corresponding error messages. + +Finally, there are `list()` functions that allow parsing of comma-separated +lists of values. diff --git a/crates/crates_io_env_vars/src/lib.rs b/crates/crates_io_env_vars/src/lib.rs new file mode 100644 index 000000000..0d349b4d7 --- /dev/null +++ b/crates/crates_io_env_vars/src/lib.rs @@ -0,0 +1,244 @@ +#![doc = include_str!("../README.md")] + +use anyhow::{Context, anyhow}; +use std::error::Error; +use std::str::FromStr; + +/// Reads an environment variable for the current process. +/// +/// Compared to [std::env::var] there are a couple of differences: +/// +/// - [var] uses [dotenvy] which loads the `.env` file from the current or +/// parent directories before returning the value. +/// +/// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable +/// wasn't set. +#[track_caller] +pub fn var(key: &str) -> anyhow::Result> { + match dotenvy::var(key) { + Ok(content) => Ok(Some(content)), + Err(dotenvy::Error::EnvVar(std::env::VarError::NotPresent)) => Ok(None), + Err(error) => Err(error.into()), + } +} + +/// Reads an environment variable for the current process, and fails if it was +/// not found. +/// +/// Compared to [std::env::var] there are a couple of differences: +/// +/// - [var] uses [dotenvy] which loads the `.env` file from the current or +/// parent directories before returning the value. +#[track_caller] +pub fn required_var(key: &str) -> anyhow::Result { + required(var(key), key) +} + +/// Reads an environment variable for the current process, and parses it if +/// it is set. +/// +/// Compared to [std::env::var] there are a couple of differences: +/// +/// - [var] uses [dotenvy] which loads the `.env` file from the current or +/// parent directories before returning the value. +/// +/// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable +/// wasn't set. +#[track_caller] +pub fn var_parsed(key: &str) -> anyhow::Result> +where + R: FromStr, + R::Err: Error + Send + Sync + 'static, +{ + match var(key) { + Ok(Some(content)) => { + Ok(Some(content.parse().with_context(|| { + format!("Failed to parse {key} environment variable") + })?)) + } + Ok(None) => Ok(None), + Err(error) => Err(error), + } +} + +/// Reads an environment variable for the current process, and parses it if +/// it is set or fails otherwise. +/// +/// Compared to [std::env::var] there are a couple of differences: +/// +/// - [var] uses [dotenvy] which loads the `.env` file from the current or +/// parent directories before returning the value. +#[track_caller] +pub fn required_var_parsed(key: &str) -> anyhow::Result +where + R: FromStr, + R::Err: Error + Send + Sync + 'static, +{ + required(var_parsed(key), key) +} + +fn required(res: anyhow::Result>, key: &str) -> anyhow::Result { + match res { + Ok(opt) => opt.ok_or_else(|| anyhow!("Failed to find required {key} environment variable")), + Err(error) => Err(error), + } +} + +/// Reads an environment variable and parses it as a comma-separated list, or +/// returns an empty list if the variable is not set. +#[track_caller] +pub fn list(key: &str) -> anyhow::Result> { + let values = match var(key)? { + None => vec![], + Some(s) if s.is_empty() => vec![], + Some(s) => s.split(',').map(str::trim).map(String::from).collect(), + }; + + Ok(values) +} + +/// Reads an environment variable and parses it as a comma-separated list, or +/// returns an empty list if the variable is not set. Each individual value is +/// parsed using [FromStr]. +#[track_caller] +pub fn list_parsed(key: &str, f: F) -> anyhow::Result> +where + F: Fn(&str) -> C, + C: Context, +{ + let values = match var(key)? { + None => vec![], + Some(s) if s.is_empty() => vec![], + Some(s) => s + .split(',') + .map(str::trim) + .map(|s| { + f(s).with_context(|| { + format!("Failed to parse value \"{s}\" of {key} environment variable") + }) + }) + .collect::>()?, + }; + + Ok(values) +} + +#[cfg(test)] +mod tests { + use super::*; + use claims::*; + use std::sync::{LazyLock, Mutex}; + + const TEST_VAR: &str = "CRATES_IO_ENV_VARS_TEST_VAR"; + + /// A mutex to ensure that the tests don't run in parallel, since they all + /// modify the shared environment variable. + static MUTEX: LazyLock> = LazyLock::new(|| Mutex::new(())); + + #[test] + fn test_var() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "test") }; + assert_some_eq!(assert_ok!(var(TEST_VAR)), "test"); + + unsafe { std::env::remove_var(TEST_VAR) }; + assert_none!(assert_ok!(var(TEST_VAR))); + } + + #[test] + fn test_required_var() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "test") }; + assert_ok_eq!(required_var(TEST_VAR), "test"); + + unsafe { std::env::remove_var(TEST_VAR) }; + let error = assert_err!(required_var(TEST_VAR)); + assert_eq!( + error.to_string(), + "Failed to find required CRATES_IO_ENV_VARS_TEST_VAR environment variable" + ); + } + + #[test] + fn test_var_parsed() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "42") }; + assert_some_eq!(assert_ok!(var_parsed::(TEST_VAR)), 42); + + unsafe { std::env::set_var(TEST_VAR, "test") }; + let error = assert_err!(var_parsed::(TEST_VAR)); + assert_eq!( + error.to_string(), + "Failed to parse CRATES_IO_ENV_VARS_TEST_VAR environment variable" + ); + + unsafe { std::env::remove_var(TEST_VAR) }; + assert_none!(assert_ok!(var_parsed::(TEST_VAR))); + } + + #[test] + fn test_required_var_parsed() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "42") }; + assert_ok_eq!(required_var_parsed::(TEST_VAR), 42); + + unsafe { std::env::set_var(TEST_VAR, "test") }; + let error = assert_err!(required_var_parsed::(TEST_VAR)); + assert_eq!( + error.to_string(), + "Failed to parse CRATES_IO_ENV_VARS_TEST_VAR environment variable" + ); + + unsafe { std::env::remove_var(TEST_VAR) }; + let error = assert_err!(required_var_parsed::(TEST_VAR)); + assert_eq!( + error.to_string(), + "Failed to find required CRATES_IO_ENV_VARS_TEST_VAR environment variable" + ); + } + + #[test] + fn test_list() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "test") }; + assert_ok_eq!(list(TEST_VAR), vec!["test"]); + + unsafe { std::env::set_var(TEST_VAR, "test, foo, bar ") }; + assert_ok_eq!(list(TEST_VAR), vec!["test", "foo", "bar"]); + + unsafe { std::env::set_var(TEST_VAR, "") }; + assert_ok_eq!(list(TEST_VAR), Vec::::new()); + + unsafe { std::env::remove_var(TEST_VAR) }; + assert_ok_eq!(list(TEST_VAR), Vec::::new()); + } + + #[test] + fn test_list_parsed() { + let _guard = MUTEX.lock().unwrap(); + + unsafe { std::env::set_var(TEST_VAR, "42") }; + assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), vec![42]); + + unsafe { std::env::set_var(TEST_VAR, "42, 1, -53 ") }; + assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), vec![42, 1, -53]); + + unsafe { std::env::set_var(TEST_VAR, "42, what") }; + let error = assert_err!(list_parsed(TEST_VAR, i32::from_str)); + assert_eq!( + error.to_string(), + "Failed to parse value \"what\" of CRATES_IO_ENV_VARS_TEST_VAR environment variable" + ); + + unsafe { std::env::set_var(TEST_VAR, "") }; + assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), Vec::::new()); + + unsafe { std::env::remove_var(TEST_VAR) }; + assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), Vec::::new()); + } +} From 92dca7ce8142287d269d91085bfba6eae35839c1 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 19:32:17 +0200 Subject: [PATCH 02/11] WIP --- Cargo.lock | 69 ++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/build_queue.rs | 4 +- src/context.rs | 35 +++++++-------- src/docbuilder/rustwide_builder.rs | 42 ++++++++---------- src/utils/consistency/mod.rs | 46 +++++++++++--------- src/utils/daemon.rs | 49 ++++++++++----------- src/utils/queue_builder.rs | 4 +- src/web/mod.rs | 53 ++++++++++------------- 9 files changed, 176 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d5cfa7a9..cb2c821a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1130,6 +1130,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537c317ddf588aab15c695bf92cf55dec159b93221c074180ca3e0e5a94da415" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5abbf2d4a4c6896197c9de13d6d7cb7eff438c63dacde1dde980569cb00248" +dependencies = [ + "darling 0.21.3", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", +] + [[package]] name = "borsh" version = "1.5.7" @@ -1770,8 +1795,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -1788,13 +1823,38 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", "quote", "syn 2.0.106", ] @@ -1960,6 +2020,7 @@ dependencies = [ "axum-extra", "backtrace", "base64 0.22.1", + "bon", "bzip2", "chrono", "clap", @@ -6665,7 +6726,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", "syn 2.0.106", diff --git a/Cargo.toml b/Cargo.toml index 4d431018f..c522d0e46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ sqlx = { version = "0.8", features = [ "runtime-tokio", "postgres", "sqlite", "c url = { version = "2.1.1", features = ["serde"] } docsrs-metadata = { path = "crates/metadata" } anyhow = { version = "1.0.42", features = ["backtrace"]} +bon = "3.7.1" backtrace = "0.3.61" thiserror = "2.0.3" comrak = { version = "0.41.0", default-features = false } diff --git a/src/build_queue.rs b/src/build_queue.rs index 2c4b9e7dd..8782b1848 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -609,9 +609,9 @@ impl BuildQueue { /// Builds the top package from the queue. Returns whether there was a package in the queue. /// /// Note that this will return `Ok(true)` even if the package failed to build. - pub(crate) fn build_next_queue_package( + pub(crate) fn build_next_queue_package( &self, - context: &C, + context: &Context, builder: &mut RustwideBuilder, ) -> Result { let mut processed = false; diff --git a/src/context.rs b/src/context.rs index 9629790ed..69e57ecb7 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,27 +1,28 @@ use crate::cdn::CdnBackend; use crate::db::Pool; -use crate::error::Result; use crate::repositories::RepositoryStatsUpdater; use crate::{ AsyncBuildQueue, AsyncStorage, BuildQueue, Config, Index, InstanceMetrics, RegistryApi, ServiceMetrics, Storage, }; -use std::{future::Future, sync::Arc}; +use bon::Builder; +use std::sync::Arc; use tokio::runtime::Runtime; -pub trait Context { - fn config(&self) -> Result>; - fn async_build_queue(&self) -> impl Future>> + Send; - fn build_queue(&self) -> Result>; - fn storage(&self) -> Result>; - fn async_storage(&self) -> impl Future>> + Send; - fn cdn(&self) -> impl Future>> + Send; - fn pool(&self) -> Result; - fn async_pool(&self) -> impl Future> + Send; - fn service_metrics(&self) -> Result>; - fn instance_metrics(&self) -> Result>; - fn index(&self) -> Result>; - fn registry_api(&self) -> Result>; - fn repository_stats_updater(&self) -> Result>; - fn runtime(&self) -> Result>; +#[derive(Builder)] +pub struct Context { + pub config: Arc, + pub async_build_queue: Arc, + pub build_queue: Arc, + pub storage: Arc, + pub async_storage: Arc, + pub cdn: Arc, + pub pool: Pool, + pub async_pool: Pool, + pub service_metrics: Arc, + pub instance_metrics: Arc, + pub index: Arc, + pub registry_api: Arc, + pub repository_stats_updater: Arc, + pub runtime: Arc, } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 0b215ee34..99a1f42a6 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -83,12 +83,10 @@ async fn get_configured_toolchain(conn: &mut sqlx::PgConnection) -> Result(context: &C) -> Result { - let config = context.config()?; - - let mut builder = WorkspaceBuilder::new(&config.rustwide_workspace, USER_AGENT) - .running_inside_docker(config.inside_docker); - if let Some(custom_image) = &config.docker_image { +fn build_workspace(context: &Context) -> Result { + let mut builder = WorkspaceBuilder::new(&context.config.rustwide_workspace, USER_AGENT) + .running_inside_docker(context.config.inside_docker); + if let Some(custom_image) = &context.config.docker_image { let image = match SandboxImage::local(custom_image) { Ok(i) => i, Err(CommandError::SandboxImageMissing(_)) => SandboxImage::remote(custom_image)?, @@ -127,35 +125,29 @@ pub struct RustwideBuilder { } impl RustwideBuilder { - pub fn init(context: &C) -> Result { - let config = context.config()?; - let pool = context.pool()?; - let runtime = context.runtime()?; - let toolchain = runtime.block_on(async { - let mut conn = pool.get_async().await?; + pub fn init(context: &Context) -> Result { + let toolchain = context.runtime.block_on(async { + let mut conn = context.pool.get_async().await?; get_configured_toolchain(&mut conn).await })?; Ok(RustwideBuilder { workspace: build_workspace(context)?, toolchain, - config, - db: pool, - runtime: runtime.clone(), - storage: context.storage()?, - async_storage: runtime.block_on(context.async_storage())?, - metrics: context.instance_metrics()?, - registry_api: context.registry_api()?, - repository_stats_updater: context.repository_stats_updater()?, + config: context.config.clone(), + db: context.pool.clone(), + runtime: context.runtime.clone(), + storage: context.storage.clone(), + async_storage: context.async_storage.clone(), + metrics: context.instance_metrics.clone(), + registry_api: context.registry_api.clone(), + repository_stats_updater: context.repository_stats_updater.clone(), workspace_initialize_time: Instant::now(), }) } - pub fn reinitialize_workspace_if_interval_passed( - &mut self, - context: &C, - ) -> Result<()> { - let interval = context.config()?.build_workspace_reinitialization_interval; + pub fn reinitialize_workspace_if_interval_passed(&mut self, context: &Context) -> Result<()> { + let interval = context.config.build_workspace_reinitialization_interval; if self.workspace_initialize_time.elapsed() >= interval { info!("start reinitialize workspace again"); self.workspace = build_workspace(context)?; diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index 8043c0975..152802bf8 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -24,18 +24,16 @@ const BUILD_PRIORITY: i32 = 15; /// /// Even when activities fail, the command can just be re-run. While the diff calculation will /// be repeated, we won't re-execute fixing activities. -pub async fn run_check(ctx: &C, dry_run: bool) -> Result<()> { - let index = ctx.index()?; - +pub async fn run_check(ctx: &Context, dry_run: bool) -> Result<()> { info!("Loading data from database..."); - let mut conn = ctx.async_pool().await?.get_async().await?; - let db_data = db::load(&mut conn, &*ctx.config()?) + let mut conn = ctx.pool.get_async().await?; + let db_data = db::load(&mut conn, &*ctx.config) .await .context("Loading crate data from database for consistency check")?; tracing::info!("Loading data from index..."); let index_data = spawn_blocking({ - let index = index.clone(); + let index = ctx.index.clone(); move || index::load(&index) }) .await @@ -80,19 +78,13 @@ struct HandleResult { yanks_corrected: u32, } -async fn handle_diff<'a, I, C>(ctx: &C, iter: I, dry_run: bool) -> Result +async fn handle_diff<'a, I>(ctx: &Context, iter: I, dry_run: bool) -> Result where I: Iterator, - C: Context, { let mut result = HandleResult::default(); - let config = ctx.config()?; - - let storage = ctx.async_storage().await?; - let build_queue = ctx.async_build_queue().await?; - - let mut conn = ctx.async_pool().await?.get_async().await?; + let mut conn = ctx.async_pool.get_async().await?; for difference in iter { println!("{difference}"); @@ -100,7 +92,8 @@ where match difference { diff::Difference::CrateNotInIndex(name) => { if !dry_run - && let Err(err) = delete::delete_crate(&mut conn, &storage, &config, name).await + && let Err(err) = + delete::delete_crate(&mut conn, &ctx.async_storage, &ctx.config, name).await { warn!("{:?}", err); } @@ -109,7 +102,8 @@ where diff::Difference::CrateNotInDb(name, versions) => { for version in versions { if !dry_run - && let Err(err) = build_queue + && let Err(err) = ctx + .async_build_queue .add_crate(name, version, BUILD_PRIORITY, None) .await { @@ -120,8 +114,14 @@ where } diff::Difference::ReleaseNotInIndex(name, version) => { if !dry_run - && let Err(err) = - delete::delete_version(&mut conn, &storage, &config, name, version).await + && let Err(err) = delete::delete_version( + &mut conn, + &ctx.async_storage, + &ctx.config, + name, + version, + ) + .await { warn!("{:?}", err); } @@ -129,7 +129,8 @@ where } diff::Difference::ReleaseNotInDb(name, version) => { if !dry_run - && let Err(err) = build_queue + && let Err(err) = ctx + .async_build_queue .add_crate(name, version, BUILD_PRIORITY, None) .await { @@ -138,7 +139,12 @@ where result.builds_queued += 1; } diff::Difference::ReleaseYank(name, version, yanked) => { - if !dry_run && let Err(err) = build_queue.set_yanked(name, version, *yanked).await { + if !dry_run + && let Err(err) = ctx + .async_build_queue + .set_yanked(name, version, *yanked) + .await + { warn!("{:?}", err); } result.yanks_corrected += 1; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 26ecb50bc..ee59c05ca 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -52,11 +52,11 @@ pub async fn watch_registry( } } -fn start_registry_watcher(context: &C) -> Result<(), Error> { - let build_queue = context.runtime()?.block_on(context.async_build_queue())?; - let config = context.config()?; - let index = context.index()?; - let runtime = context.runtime()?; +fn start_registry_watcher(context: &Context) -> Result<(), Error> { + let build_queue = context.async_build_queue.clone(); + let config = context.config.clone(); + let index = context.index.clone(); + let runtime = context.runtime.clone(); runtime.spawn(async { // space this out to prevent it from clashing against the queue-builder thread on launch @@ -68,13 +68,13 @@ fn start_registry_watcher(context: &C) -> Result<(), Error> { Ok(()) } -pub fn start_background_repository_stats_updater(context: &C) -> Result<(), Error> { +pub fn start_background_repository_stats_updater(context: &Context) -> Result<(), Error> { // This call will still skip github repositories updates and continue if no token is provided // (gitlab doesn't require to have a token). The only time this can return an error is when // creating a pool or if config fails, which shouldn't happen here because this is run right at // startup. - let updater = context.repository_stats_updater()?; - let runtime = context.runtime()?; + let updater = context.repository_stats_updater.clone(); + let runtime = context.runtime.clone(); async_cron( &runtime, "repository stats updater", @@ -90,11 +90,11 @@ pub fn start_background_repository_stats_updater(context: &C) -> Res Ok(()) } -pub fn start_background_queue_rebuild(context: &C) -> Result<(), Error> { - let runtime = context.runtime()?; - let pool = context.pool()?; - let config = context.config()?; - let build_queue = runtime.block_on(context.async_build_queue())?; +pub fn start_background_queue_rebuild(context: &Context) -> Result<(), Error> { + let runtime = context.runtime.clone(); + let pool = context.pool.clone(); + let config = context.config.clone(); + let build_queue = context.async_build_queue.clone(); if config.max_queued_rebuilds.is_none() { info!("rebuild config incomplete, skipping rebuild queueing"); @@ -119,12 +119,12 @@ pub fn start_background_queue_rebuild(context: &C) -> Result<(), Err Ok(()) } -pub fn start_background_cdn_invalidator(context: &C) -> Result<(), Error> { - let metrics = context.instance_metrics()?; - let config = context.config()?; - let pool = context.pool()?; - let runtime = context.runtime()?; - let cdn = runtime.block_on(context.cdn())?; +pub fn start_background_cdn_invalidator(context: &Context) -> Result<(), Error> { + let metrics = context.instance_metrics.clone(); + let config = context.config.clone(); + let pool = context.pool.clone(); + let runtime = context.runtime.clone(); + let cdn = context.cdn.clone(); if config.cloudfront_distribution_id_web.is_none() && config.cloudfront_distribution_id_static.is_none() @@ -178,12 +178,7 @@ pub fn start_background_cdn_invalidator(context: &C) -> Result<(), E Ok(()) } -pub fn start_daemon( - context: C, - enable_registry_watcher: bool, -) -> Result<(), Error> { - let context = Arc::new(context); - +pub fn start_daemon(context: Arc, enable_registry_watcher: bool) -> Result<(), Error> { // Start the web server before doing anything more expensive // Please check with an administrator before changing this (see #1172 for context). info!("Starting web server"); @@ -198,8 +193,8 @@ pub fn start_daemon( } // build new crates every minute - let build_queue = context.build_queue()?; - let config = context.config()?; + let build_queue = context.build_queue.clone(); + let config = context.config.clone(); let rustwide_builder = RustwideBuilder::init(&*context)?; thread::Builder::new() .name("build queue reader".to_string()) diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 3430f62d4..25e371dea 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -7,8 +7,8 @@ use std::time::Duration; use std::{fs, io, path::Path, thread}; use tracing::{debug, error, warn}; -pub fn queue_builder( - context: &C, +pub fn queue_builder( + context: &Context, mut builder: RustwideBuilder, build_queue: Arc, config: Arc, diff --git a/src/web/mod.rs b/src/web/mod.rs index 605d58e35..4a5709b6b 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -400,17 +400,13 @@ async fn set_sentry_transaction_name_from_axum_route( next.run(request).await } -async fn apply_middleware( +async fn apply_middleware( router: AxumRouter, - context: &C, + context: &Context, template_data: Option>, ) -> Result { - let config = context.config()?; let has_templates = template_data.is_some(); - let async_storage = context.async_storage().await?; - let build_queue = context.async_build_queue().await?; - Ok(router.layer( ServiceBuilder::new() .layer(TraceLayer::new_for_http()) @@ -421,18 +417,21 @@ async fn apply_middleware( )) .layer(CatchPanicLayer::new()) .layer(option_layer( - config + context + .config .report_request_timeouts .then_some(middleware::from_fn(log_timeouts_to_sentry)), )) - .layer(option_layer(config.request_timeout.map(TimeoutLayer::new))) - .layer(Extension(context.async_pool().await?)) - .layer(Extension(build_queue)) - .layer(Extension(context.service_metrics()?)) - .layer(Extension(context.instance_metrics()?)) - .layer(Extension(context.config()?)) - .layer(Extension(context.registry_api()?)) - .layer(Extension(async_storage)) + .layer(option_layer( + context.config.request_timeout.map(TimeoutLayer::new), + )) + .layer(Extension(context.pool.clone())) + .layer(Extension(context.async_build_queue.clone())) + .layer(Extension(context.service_metrics.clone())) + .layer(Extension(context.instance_metrics.clone())) + .layer(Extension(context.config.clone())) + .layer(Extension(context.registry_api.clone())) + .layer(Extension(context.async_storage.clone())) .layer(option_layer(template_data.map(Extension))) .layer(middleware::from_fn(csp::csp_middleware)) .layer(option_layer(has_templates.then_some(middleware::from_fn( @@ -442,20 +441,20 @@ async fn apply_middleware( )) } -pub(crate) async fn build_axum_app( - context: &C, +pub(crate) async fn build_axum_app( + context: &Context, template_data: Arc, ) -> Result { apply_middleware(routes::build_axum_routes(), context, Some(template_data)).await } -pub(crate) async fn build_metrics_axum_app(context: &C) -> Result { +pub(crate) async fn build_metrics_axum_app(context: &Context) -> Result { apply_middleware(routes::build_metric_routes(), context, None).await } -pub fn start_background_metrics_webserver( +pub fn start_background_metrics_webserver( addr: Option, - context: &C, + context: &Context, ) -> Result<(), Error> { let axum_addr: SocketAddr = addr.unwrap_or(DEFAULT_BIND); @@ -465,7 +464,7 @@ pub fn start_background_metrics_webserver( axum_addr.port() ); - let runtime = context.runtime()?; + let runtime = context.runtime.clone(); let metrics_axum_app = runtime .block_on(build_metrics_axum_app(context))? .into_make_service(); @@ -493,8 +492,8 @@ pub fn start_background_metrics_webserver( } #[instrument(skip_all)] -pub fn start_web_server(addr: Option, context: &C) -> Result<(), Error> { - let template_data = Arc::new(TemplateData::new(context.config()?.render_threads)?); +pub fn start_web_server(addr: Option, context: &Context) -> Result<(), Error> { + let template_data = Arc::new(TemplateData::new(context.config.render_threads)?); let axum_addr = addr.unwrap_or(DEFAULT_BIND); @@ -504,13 +503,7 @@ pub fn start_web_server(addr: Option, context: &C) -> Re axum_addr.port() ); - // initialize the storage and the repo-updater in sync context - // so it can stay sync for now and doesn't fail when they would - // be initialized while starting the server below. - context.storage()?; - context.repository_stats_updater()?; - - context.runtime()?.block_on(async { + context.runtime.block_on(async { let app = build_axum_app(context, template_data) .await? .into_make_service(); From cb0771ffa704f990d68e76b8bfdb86c2dc215edf Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 20:03:30 +0200 Subject: [PATCH 03/11] config::from_env --- src/context.rs | 63 +++++++++++++++++++++++++++++++++--- src/utils/consistency/mod.rs | 2 +- src/web/metrics.rs | 2 +- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/src/context.rs b/src/context.rs index 69e57ecb7..398035407 100644 --- a/src/context.rs +++ b/src/context.rs @@ -5,11 +5,10 @@ use crate::{ AsyncBuildQueue, AsyncStorage, BuildQueue, Config, Index, InstanceMetrics, RegistryApi, ServiceMetrics, Storage, }; -use bon::Builder; +use anyhow::Result; use std::sync::Arc; -use tokio::runtime::Runtime; +use tokio::runtime; -#[derive(Builder)] pub struct Context { pub config: Arc, pub async_build_queue: Arc, @@ -18,11 +17,65 @@ pub struct Context { pub async_storage: Arc, pub cdn: Arc, pub pool: Pool, - pub async_pool: Pool, pub service_metrics: Arc, pub instance_metrics: Arc, pub index: Arc, pub registry_api: Arc, pub repository_stats_updater: Arc, - pub runtime: Arc, + pub runtime: Arc, +} + +impl Context { + pub async fn from_config(config: Config) -> Result { + let config = Arc::new(config); + let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?); + + let instance_metrics = Arc::new(InstanceMetrics::new()?); + + let pool = Pool::new(&config, runtime.clone(), instance_metrics.clone())?; + let async_storage = Arc::new( + AsyncStorage::new(pool.clone(), instance_metrics.clone(), config.clone()).await?, + ); + + let async_build_queue = Arc::new(AsyncBuildQueue::new( + pool.clone(), + instance_metrics.clone(), + config.clone(), + async_storage.clone(), + )); + + let build_queue = Arc::new(BuildQueue::new(runtime.clone(), async_build_queue.clone())); + + let storage = Arc::new(Storage::new(async_storage.clone(), runtime.clone())); + + let cdn = Arc::new(CdnBackend::new(&config).await); + + let index = Arc::new({ + let path = config.registry_index_path.clone(); + if let Some(registry_url) = config.registry_url.clone() { + Index::from_url(path, registry_url) + } else { + Index::new(path) + }? + }); + + Ok(Self { + async_build_queue, + build_queue, + storage, + async_storage, + cdn, + pool: pool.clone(), + service_metrics: Arc::new(ServiceMetrics::new()?), + instance_metrics, + index, + registry_api: Arc::new(RegistryApi::new( + config.registry_api_host.clone(), + config.crates_io_api_call_retries, + )?), + repository_stats_updater: Arc::new(RepositoryStatsUpdater::new(&config, pool)), + runtime, + config, + }) + } } diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index 152802bf8..3995a5ab3 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -84,7 +84,7 @@ where { let mut result = HandleResult::default(); - let mut conn = ctx.async_pool.get_async().await?; + let mut conn = ctx.pool.get_async().await?; for difference in iter { println!("{difference}"); diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 33585adf4..467f81a79 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -187,7 +187,7 @@ mod tests { } // this shows what the routes were *actually* recorded as, making it easier to update ROUTES if the name changes. - let metrics_serialized = metrics.gather(&env.async_pool().await?)?; + let metrics_serialized = metrics.gather(&env.pool().await?)?; let all_routes_visited = metrics_serialized .iter() .find(|x| x.name() == "docsrs_routes_visited") From e9ee9fa3182f8241f3d92aca5efcdde4354bf72d Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 20:06:02 +0200 Subject: [PATCH 04/11] drop bincontext --- src/bin/cratesfyi.rs | 142 +------------------------------------------ src/context.rs | 12 ++-- 2 files changed, 9 insertions(+), 145 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 5e674b912..8c927a326 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -187,7 +187,8 @@ enum CommandLine { impl CommandLine { fn handle_args(self) -> Result<()> { - let ctx = BinContext::new(); + let config = Config::from_env()?; + let ctx = Context::from_config(config)?; match self { Self::Build { subcommand } => subcommand.handle_args(ctx)?, @@ -832,142 +833,3 @@ enum DeleteSubcommand { version: String, }, } - -struct BinContext { - build_queue: OnceCell>, - async_build_queue: tokio::sync::OnceCell>, - storage: OnceCell>, - cdn: tokio::sync::OnceCell>, - config: OnceCell>, - pool: OnceCell, - service_metrics: OnceCell>, - instance_metrics: OnceCell>, - index: OnceCell>, - registry_api: OnceCell>, - repository_stats_updater: OnceCell>, - runtime: OnceCell>, -} - -impl BinContext { - fn new() -> Self { - Self { - build_queue: OnceCell::new(), - async_build_queue: tokio::sync::OnceCell::new(), - storage: OnceCell::new(), - cdn: tokio::sync::OnceCell::new(), - config: OnceCell::new(), - pool: OnceCell::new(), - service_metrics: OnceCell::new(), - instance_metrics: OnceCell::new(), - index: OnceCell::new(), - registry_api: OnceCell::new(), - repository_stats_updater: OnceCell::new(), - runtime: OnceCell::new(), - } - } -} - -macro_rules! lazy { - ( $(fn $name:ident($self:ident) -> $type:ty = $init:expr);+ $(;)? ) => { - $(fn $name(&$self) -> Result> { - Ok($self - .$name - .get_or_try_init::<_, Error>(|| Ok(Arc::new($init)))? - .clone()) - })* - } -} - -impl Context for BinContext { - lazy! { - fn build_queue(self) -> BuildQueue = { - let runtime = self.runtime()?; - BuildQueue::new( - runtime.clone(), - runtime.block_on(self.async_build_queue())? - ) - }; - fn storage(self) -> Storage = { - let runtime = self.runtime()?; - Storage::new( - runtime.block_on(self.async_storage())?, - runtime - ) - }; - fn config(self) -> Config = Config::from_env()?; - fn service_metrics(self) -> ServiceMetrics = { - ServiceMetrics::new()? - }; - fn instance_metrics(self) -> InstanceMetrics = InstanceMetrics::new()?; - fn runtime(self) -> Runtime = { - Builder::new_multi_thread() - .enable_all() - .build()? - }; - fn index(self) -> Index = { - let config = self.config()?; - let path = config.registry_index_path.clone(); - if let Some(registry_url) = config.registry_url.clone() { - Index::from_url(path, registry_url) - } else { - Index::new(path) - }? - }; - fn registry_api(self) -> RegistryApi = { - let config = self.config()?; - RegistryApi::new(config.registry_api_host.clone(), config.crates_io_api_call_retries)? - }; - fn repository_stats_updater(self) -> RepositoryStatsUpdater = { - let config = self.config()?; - let pool = self.pool()?; - RepositoryStatsUpdater::new(&config, pool) - }; - } - - async fn async_pool(&self) -> Result { - self.pool() - } - - fn pool(&self) -> Result { - Ok(self - .pool - .get_or_try_init::<_, Error>(|| { - Ok(Pool::new( - &*self.config()?, - self.runtime()?, - self.instance_metrics()?, - )?) - })? - .clone()) - } - - async fn async_storage(&self) -> Result> { - Ok(Arc::new( - AsyncStorage::new(self.pool()?, self.instance_metrics()?, self.config()?).await?, - )) - } - - async fn async_build_queue(&self) -> Result> { - Ok(self - .async_build_queue - .get_or_try_init(|| async { - Ok::<_, Error>(Arc::new(AsyncBuildQueue::new( - self.pool()?, - self.instance_metrics()?, - self.config()?, - self.async_storage().await?, - ))) - }) - .await? - .clone()) - } - - async fn cdn(&self) -> Result> { - let config = self.config()?; - Ok(self - .cdn - .get_or_init(|| async { Arc::new(CdnBackend::new(&config).await) }) - .await - .clone()) - } -} diff --git a/src/context.rs b/src/context.rs index 398035407..d146107ae 100644 --- a/src/context.rs +++ b/src/context.rs @@ -26,16 +26,18 @@ pub struct Context { } impl Context { - pub async fn from_config(config: Config) -> Result { + pub fn from_config(config: Config) -> Result { let config = Arc::new(config); let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?); let instance_metrics = Arc::new(InstanceMetrics::new()?); let pool = Pool::new(&config, runtime.clone(), instance_metrics.clone())?; - let async_storage = Arc::new( - AsyncStorage::new(pool.clone(), instance_metrics.clone(), config.clone()).await?, - ); + let async_storage = Arc::new(runtime.block_on(AsyncStorage::new( + pool.clone(), + instance_metrics.clone(), + config.clone(), + ))?); let async_build_queue = Arc::new(AsyncBuildQueue::new( pool.clone(), @@ -48,7 +50,7 @@ impl Context { let storage = Arc::new(Storage::new(async_storage.clone(), runtime.clone())); - let cdn = Arc::new(CdnBackend::new(&config).await); + let cdn = Arc::new(runtime.block_on(CdnBackend::new(&config))); let index = Arc::new({ let path = config.registry_index_path.clone(); From 3677870ef5b30a677ff2075b8b4105ad48b7261e Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 20:13:13 +0200 Subject: [PATCH 05/11] bin --- src/bin/cratesfyi.rs | 117 ++++++++++++++++++++----------------------- src/utils/daemon.rs | 4 +- 2 files changed, 56 insertions(+), 65 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 8c927a326..d3e43c781 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -5,27 +5,22 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use anyhow::{Context as _, Error, Result, anyhow}; +use anyhow::{Context as _, Result, anyhow}; use clap::{Parser, Subcommand, ValueEnum}; -use docs_rs::cdn::CdnBackend; -use docs_rs::db::{self, CrateId, Overrides, Pool, add_path_into_database}; -use docs_rs::repositories::RepositoryStatsUpdater; +use docs_rs::db::{self, CrateId, Overrides, add_path_into_database}; use docs_rs::utils::{ ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities, queue_builder, remove_crate_priority, set_config, set_crate_priority, }; use docs_rs::{ - AsyncBuildQueue, AsyncStorage, BuildQueue, Config, Context, Index, InstanceMetrics, - PackageKind, RegistryApi, RustwideBuilder, ServiceMetrics, Storage, - start_background_metrics_webserver, start_web_server, + Config, Context, PackageKind, RustwideBuilder, start_background_metrics_webserver, + start_web_server, }; use futures_util::StreamExt; -use once_cell::sync::OnceCell; use sentry::{ TransactionContext, integrations::panic as sentry_panic, integrations::tracing as sentry_tracing, }; -use tokio::runtime::{Builder, Runtime}; use tracing_log::LogTracer; use tracing_subscriber::{EnvFilter, filter::Directive, prelude::*}; @@ -210,11 +205,11 @@ impl CommandLine { start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; - ctx.runtime()?.block_on(async { + ctx.runtime.block_on(async { docs_rs::utils::watch_registry( - ctx.async_build_queue().await?, - ctx.config()?, - ctx.index()?, + ctx.async_build_queue.clone(), + ctx.config.clone(), + ctx.index.clone(), ) .await })?; @@ -224,8 +219,8 @@ impl CommandLine { } => { start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; - let build_queue = ctx.build_queue()?; - let config = ctx.config()?; + let build_queue = ctx.build_queue.clone(); + let config = ctx.config.clone(); let rustwide_builder = RustwideBuilder::init(&ctx)?; queue_builder(&ctx, rustwide_builder, build_queue, config)?; } @@ -288,8 +283,8 @@ enum QueueSubcommand { } impl QueueSubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { - let build_queue = ctx.build_queue()?; + fn handle_args(self, ctx: Context) -> Result<()> { + let build_queue = ctx.build_queue.clone(); match self { Self::Add { crate_name, @@ -299,7 +294,7 @@ impl QueueSubcommand { &crate_name, &crate_version, build_priority, - ctx.config()?.registry_url.as_deref(), + ctx.config.registry_url.as_deref(), )?, Self::GetLastSeenReference => { @@ -315,7 +310,7 @@ impl QueueSubcommand { (Some(reference), false) => reference, (None, true) => { println!("Fetching changes to set reference to HEAD"); - let (_, oid) = ctx.index()?.diff()?.peek_changes()?; + let (_, oid) = ctx.index.diff()?.peek_changes()?; oid } (_, _) => unreachable!(), @@ -360,9 +355,9 @@ enum PrioritySubcommand { } impl PrioritySubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { - ctx.runtime()?.block_on(async move { - let mut conn = ctx.pool()?.get_async().await?; + fn handle_args(self, ctx: Context) -> Result<()> { + ctx.runtime.block_on(async move { + let mut conn = ctx.pool.get_async().await?; match self { Self::List => { for (pattern, priority) in list_crate_priorities(&mut conn).await? { @@ -442,8 +437,8 @@ enum BuildSubcommand { } impl BuildSubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { - let build_queue = ctx.build_queue()?; + fn handle_args(self, ctx: Context) -> Result<()> { + let build_queue = ctx.build_queue.clone(); let rustwide_builder = || -> Result { RustwideBuilder::init(&ctx) }; match self { @@ -459,7 +454,7 @@ impl BuildSubcommand { .build_local_package(&path) .context("Building documentation failed")?; } else { - let registry_url = ctx.config()?.registry_url.clone(); + let registry_url = ctx.config.registry_url.clone(); builder .build_package( &crate_name @@ -477,8 +472,8 @@ impl BuildSubcommand { } Self::UpdateToolchain { only_first_time } => { - let rustc_version = ctx.runtime()?.block_on({ - let pool = ctx.pool()?; + let rustc_version = ctx.runtime.block_on({ + let pool = ctx.pool.clone(); async move { let mut conn = pool .get_async() @@ -513,9 +508,9 @@ impl BuildSubcommand { } Self::SetToolchain { toolchain_name } => { - ctx.runtime()?.block_on(async move { + ctx.runtime.block_on(async move { let mut conn = ctx - .pool()? + .pool .get_async() .await .context("failed to get a database connection")?; @@ -590,11 +585,11 @@ enum DatabaseSubcommand { } impl DatabaseSubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { + fn handle_args(self, ctx: Context) -> Result<()> { match self { Self::Migrate { version } => { - let pool = ctx.pool()?; - ctx.runtime()? + let pool = ctx.pool.clone(); + ctx.runtime .block_on(async { let mut conn = pool.get_async().await?; db::migrate(&mut conn, version).await @@ -603,8 +598,8 @@ impl DatabaseSubcommand { } Self::UpdateLatestVersionId => { - let pool = ctx.pool()?; - ctx.runtime()? + let pool = ctx.pool.clone(); + ctx.runtime .block_on(async { let mut list_conn = pool.get_async().await?; let mut update_conn = pool.get_async().await?; @@ -628,27 +623,27 @@ impl DatabaseSubcommand { } Self::UpdateRepositoryFields => { - ctx.runtime()? - .block_on(ctx.repository_stats_updater()?.update_all_crates())?; + ctx.runtime + .block_on(ctx.repository_stats_updater.update_all_crates())?; } Self::BackfillRepositoryStats => { - ctx.runtime()? - .block_on(ctx.repository_stats_updater()?.backfill_repositories())?; + ctx.runtime + .block_on(ctx.repository_stats_updater.backfill_repositories())?; } - Self::UpdateCrateRegistryFields { name } => ctx.runtime()?.block_on(async move { - let mut conn = ctx.pool()?.get_async().await?; - let registry_data = ctx.registry_api()?.get_crate_data(&name).await?; + Self::UpdateCrateRegistryFields { name } => ctx.runtime.block_on(async move { + let mut conn = ctx.pool.get_async().await?; + let registry_data = ctx.registry_api.get_crate_data(&name).await?; db::update_crate_data_in_database(&mut conn, &name, ®istry_data).await })?, Self::AddDirectory { directory } => { - ctx.runtime()? + ctx.runtime .block_on(async { - let storage = ctx.async_storage().await?; + let storage = ctx.async_storage.clone(); - add_path_into_database(&storage, &ctx.config()?.prefix, directory).await + add_path_into_database(&storage, &ctx.config.prefix, directory).await }) .context("Failed to add directory into database")?; } @@ -656,13 +651,13 @@ impl DatabaseSubcommand { Self::Delete { command: DeleteSubcommand::Version { name, version }, } => ctx - .runtime()? + .runtime .block_on(async move { - let mut conn = ctx.pool()?.get_async().await?; + let mut conn = ctx.pool.get_async().await?; db::delete_version( &mut conn, - &*ctx.async_storage().await?, - &*ctx.config()?, + &*ctx.async_storage, + &*ctx.config, &name, &version, ) @@ -672,16 +667,10 @@ impl DatabaseSubcommand { Self::Delete { command: DeleteSubcommand::Crate { name }, } => ctx - .runtime()? + .runtime .block_on(async move { - let mut conn = ctx.pool()?.get_async().await?; - db::delete_crate( - &mut conn, - &*ctx.async_storage().await?, - &*ctx.config()?, - &name, - ) - .await + let mut conn = ctx.pool.get_async().await?; + db::delete_crate(&mut conn, &*ctx.async_storage, &*ctx.config, &name).await }) .context("failed to delete the crate")?, Self::Blacklist { command } => command.handle_args(ctx)?, @@ -689,7 +678,7 @@ impl DatabaseSubcommand { Self::Limits { command } => command.handle_args(ctx)?, Self::Synchronize { dry_run } => { - ctx.runtime()? + ctx.runtime .block_on(docs_rs::utils::consistency::run_check(&ctx, dry_run))?; } } @@ -721,9 +710,9 @@ enum LimitsSubcommand { } impl LimitsSubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { - let pool = ctx.pool()?; - ctx.runtime()?.block_on(async move { + fn handle_args(self, ctx: Context) -> Result<()> { + let pool = ctx.pool.clone(); + ctx.runtime.block_on(async move { let mut conn = pool.get_async().await?; match self { @@ -789,9 +778,9 @@ enum BlacklistSubcommand { } impl BlacklistSubcommand { - fn handle_args(self, ctx: BinContext) -> Result<()> { - ctx.runtime()?.block_on(async { - let conn = &mut *ctx.pool()?.get_async().await?; + fn handle_args(self, ctx: Context) -> Result<()> { + ctx.runtime.block_on(async { + let conn = &mut *ctx.pool.get_async().await?; match self { Self::List => { let crates = db::blacklist::list_crates(conn) diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index ee59c05ca..2cbd014e9 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -178,7 +178,9 @@ pub fn start_background_cdn_invalidator(context: &Context) -> Result<(), Error> Ok(()) } -pub fn start_daemon(context: Arc, enable_registry_watcher: bool) -> Result<(), Error> { +pub fn start_daemon(context: Context, enable_registry_watcher: bool) -> Result<(), Error> { + let context = Arc::new(context); + // Start the web server before doing anything more expensive // Please check with an administrator before changing this (see #1172 for context). info!("Starting web server"); From 0e7ac765554237ec0908152b006568ad1f6963bc Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 21:19:01 +0200 Subject: [PATCH 06/11] fixes --- Cargo.lock | 1 + Cargo.toml | 1 + src/bin/cratesfyi.rs | 6 +- src/build_queue.rs | 34 ++-- src/cdn.rs | 38 ++-- src/db/add_package.rs | 30 +-- src/db/blacklist.rs | 8 +- src/db/delete.rs | 46 ++--- src/db/overrides.rs | 2 +- src/docbuilder/limits.rs | 8 +- src/docbuilder/rustwide_builder.rs | 52 ++--- src/repositories/github.rs | 17 +- src/test/mod.rs | 314 ++++++++++------------------- src/utils/consistency/db.rs | 3 +- src/utils/consistency/mod.rs | 30 +-- src/utils/daemon.rs | 14 +- src/utils/mod.rs | 4 +- src/utils/queue.rs | 8 +- src/web/build_details.rs | 4 +- src/web/builds.rs | 6 +- src/web/crate_details.rs | 40 ++-- src/web/file.rs | 4 +- src/web/metrics.rs | 3 +- src/web/mod.rs | 12 +- src/web/releases.rs | 16 +- src/web/routes.rs | 2 +- src/web/rustdoc.rs | 10 +- src/web/source.rs | 2 +- 28 files changed, 296 insertions(+), 419 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb2c821a0..cb302404e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2005,6 +2005,7 @@ name = "docs-rs" version = "0.6.0" dependencies = [ "anyhow", + "arc-swap", "askama", "async-compression", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index c522d0e46..ff7931158 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ exclude = [ ] [dependencies] +arc-swap = "1.7.1" sentry = { version = "0.42.0", features = ["panic", "tracing", "tower-http", "anyhow", "backtrace"] } log = "0.4" tracing = "0.1.37" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index d3e43c781..37af7d487 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -656,8 +656,8 @@ impl DatabaseSubcommand { let mut conn = ctx.pool.get_async().await?; db::delete_version( &mut conn, - &*ctx.async_storage, - &*ctx.config, + &ctx.async_storage, + &ctx.config, &name, &version, ) @@ -670,7 +670,7 @@ impl DatabaseSubcommand { .runtime .block_on(async move { let mut conn = ctx.pool.get_async().await?; - db::delete_crate(&mut conn, &*ctx.async_storage, &*ctx.config, &name).await + db::delete_crate(&mut conn, &ctx.async_storage, &ctx.config, &name).await }) .context("failed to delete the crate")?, Self::Blacklist { command } => command.handle_args(ctx)?, diff --git a/src/build_queue.rs b/src/build_queue.rs index 8782b1848..eab51e78e 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -754,10 +754,10 @@ mod tests { .create() .await?; - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); assert!(build_queue.queued_crates().await?.is_empty()); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; let queue = build_queue.queued_crates().await?; @@ -777,7 +777,7 @@ mod tests { config.max_queued_rebuilds = Some(1); }); - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); build_queue .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) .await?; @@ -785,7 +785,7 @@ mod tests { .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!("UPDATE queue SET attempt = 99") .execute(&mut *conn) .await?; @@ -803,7 +803,7 @@ mod tests { .create() .await?; - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; assert_eq!(build_queue.queued_crates().await?.len(), 1); @@ -819,7 +819,7 @@ mod tests { config.max_queued_rebuilds = Some(1); }); - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); build_queue .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) .await?; @@ -838,10 +838,10 @@ mod tests { .create() .await?; - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); assert_eq!(build_queue.queued_crates().await?.len(), 2); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; assert_eq!(build_queue.queued_crates().await?.len(), 2); @@ -853,7 +853,7 @@ mod tests { #[test] fn test_add_duplicate_doesnt_fail_last_priority_wins() { crate::test::async_wrapper(|env| async move { - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); queue.add_crate("some_crate", "0.1.1", 0, None).await?; queue.add_crate("some_crate", "0.1.1", 9, None).await?; @@ -873,9 +873,9 @@ mod tests { config.build_attempts = 5; }); - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( " INSERT INTO queue (name, version, priority, attempt, last_attempt ) @@ -910,11 +910,11 @@ mod tests { #[test] fn test_has_build_queued() { crate::test::async_wrapper(|env| async move { - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); queue.add_crate("dummy", "0.1.1", 0, None).await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!(queue.has_build_queued("dummy", "0.1.1").await.unwrap()); sqlx::query!("UPDATE queue SET attempt = 6") @@ -955,7 +955,7 @@ mod tests { runtime.block_on(async { // fake the build-attempt timestamp so it's older - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( "UPDATE queue SET last_attempt = $1", Utc::now() - chrono::Duration::try_seconds(60).unwrap() @@ -1060,7 +1060,7 @@ mod tests { env.runtime() .block_on(async { cdn::queued_or_active_crate_invalidations( - &mut *env.async_db().await.async_conn().await, + &mut *env.async_db().async_conn().await, ) .await })? @@ -1087,7 +1087,7 @@ mod tests { let fetch_invalidations = || { env.runtime() .block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; cdn::queued_or_active_crate_invalidations(&mut conn).await }) .unwrap() @@ -1323,7 +1323,7 @@ mod tests { fn test_broken_db_reference_breaks() { crate::test::wrapper(|env| { env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; set_config(&mut conn, ConfigName::LastSeenIndexReference, "invalid") .await .unwrap(); diff --git a/src/cdn.rs b/src/cdn.rs index 5dae03b92..17750615b 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -699,7 +699,7 @@ mod tests { config.cdn_backend = CdnKind::CloudFront; }); - assert!(matches!(*env.cdn().await, CdnBackend::CloudFront { .. })); + assert!(matches!(*env.cdn(), CdnBackend::CloudFront { .. })); assert!(matches!( CdnBackend::new(&env.config()).await, CdnBackend::CloudFront { .. } @@ -712,7 +712,7 @@ mod tests { #[test] fn create_dummy() { async_wrapper(|env| async move { - assert!(matches!(*env.cdn().await, CdnBackend::Dummy { .. })); + assert!(matches!(*env.cdn(), CdnBackend::Dummy { .. })); assert!(matches!( CdnBackend::new(&env.config()).await, CdnBackend::Dummy { .. } @@ -731,7 +731,7 @@ mod tests { }); let config = env.config(); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!( queued_or_active_crate_invalidations(&mut conn) .await? @@ -757,9 +757,9 @@ mod tests { config.cdn_max_queued_age = Duration::from_secs(0); }); - let cdn = env.cdn().await; + let cdn = env.cdn(); let config = env.config(); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!( queued_or_active_crate_invalidations(&mut conn) .await? @@ -813,7 +813,7 @@ mod tests { assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); - let cdn = env.cdn().await; + let cdn = env.cdn(); let config = env.config(); // now handle the queued invalidations @@ -865,9 +865,9 @@ mod tests { config.cloudfront_distribution_id_static = Some("distribution_id_static".into()); }); - let cdn = env.cdn().await; + let cdn = env.cdn(); let config = env.config(); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!( queued_or_active_crate_invalidations(&mut conn) .await? @@ -921,7 +921,7 @@ mod tests { assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); - let cdn = env.cdn().await; + let cdn = env.cdn(); let config = env.config(); // now handle the queued invalidations @@ -1001,7 +1001,7 @@ mod tests { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); }); - let cdn = env.cdn().await; + let cdn = env.cdn(); // create an invalidation with 15 paths, so we're over the limit let already_running_invalidation = cdn @@ -1013,7 +1013,7 @@ mod tests { ) .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!( queued_or_active_crate_invalidations(&mut conn) .await? @@ -1049,7 +1049,7 @@ mod tests { // handle the queued invalidations handle_queued_invalidation_requests( &env.config(), - &*env.cdn().await, + &*env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1083,7 +1083,7 @@ mod tests { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); }); - let cdn = env.cdn().await; + let cdn = env.cdn(); // create an invalidation with 15 paths, so we're over the limit let already_running_invalidation = cdn @@ -1093,7 +1093,7 @@ mod tests { ) .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!( queued_or_active_crate_invalidations(&mut conn) .await? @@ -1112,7 +1112,7 @@ mod tests { // handle the queued invalidations handle_queued_invalidation_requests( &env.config(), - &*env.cdn().await, + &*env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1143,7 +1143,7 @@ mod tests { // now handle again handle_queued_invalidation_requests( &env.config(), - &*env.cdn().await, + &*env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1174,9 +1174,9 @@ mod tests { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); }); - let cdn = env.cdn().await; + let cdn = env.cdn(); - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; // no invalidation is queued assert!( queued_or_active_crate_invalidations(&mut conn) @@ -1187,7 +1187,7 @@ mod tests { // run the handler handle_queued_invalidation_requests( &env.config(), - &*env.cdn().await, + &*env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", diff --git a/src/db/add_package.rs b/src/db/add_package.rs index dcff5b1e3..8b5c0b4c7 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -655,7 +655,7 @@ mod test { #[test] fn test_set_build_to_error() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -689,7 +689,7 @@ mod test { #[test] fn test_finish_build_success_valid_rustc_date() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -738,7 +738,7 @@ mod test { #[test] fn test_finish_build_success_invalid_rustc_date() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -783,7 +783,7 @@ mod test { #[test] fn test_finish_build_error() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -826,7 +826,7 @@ mod test { #[test] fn new_keywords() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let release_id = env .fake_release() @@ -913,7 +913,7 @@ mod test { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let kw_r = sqlx::query!( r#"SELECT kw.name as "name!", @@ -957,7 +957,7 @@ mod test { #[test] fn new_owner_long_avatar() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let owner1 = CrateOwner { @@ -997,7 +997,7 @@ mod test { #[test] fn new_owners() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let owner1 = CrateOwner { @@ -1037,7 +1037,7 @@ mod test { #[test] fn update_owner_details() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; // set initial owner details @@ -1086,7 +1086,7 @@ mod test { #[test] fn add_new_owners_and_delete_old() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; // set initial owner details @@ -1203,7 +1203,7 @@ mod test { #[test] fn test_initialize_crate() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let name = "krate"; let crate_id = initialize_crate(&mut conn, name).await?; @@ -1227,7 +1227,7 @@ mod test { #[test] fn test_initialize_release() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let name = "krate"; let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; @@ -1254,7 +1254,7 @@ mod test { #[test] fn test_initialize_build() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let name = "krate"; let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; @@ -1281,7 +1281,7 @@ mod test { #[test] fn test_long_crate_name() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let name: String = "krate".repeat(100); let crate_id = initialize_crate(&mut conn, &name).await?; @@ -1299,7 +1299,7 @@ mod test { #[test] fn test_long_release_version() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; let version: String = "version".repeat(100); diff --git a/src/db/blacklist.rs b/src/db/blacklist.rs index f4e104521..c4200c893 100644 --- a/src/db/blacklist.rs +++ b/src/db/blacklist.rs @@ -71,7 +71,7 @@ mod tests { #[test] fn test_list_blacklist() { crate::test::async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; // crates are added out of order to verify sorting add_crate(&mut conn, "crate A").await?; @@ -86,7 +86,7 @@ mod tests { #[test] fn test_add_to_and_remove_from_blacklist() { crate::test::async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!(!is_blacklisted(&mut conn, "crate foo").await?); add_crate(&mut conn, "crate foo").await?; @@ -100,7 +100,7 @@ mod tests { #[test] fn test_add_twice_to_blacklist() { crate::test::async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; add_crate(&mut conn, "crate foo").await?; assert!(add_crate(&mut conn, "crate foo").await.is_err()); @@ -113,7 +113,7 @@ mod tests { #[test] fn test_remove_non_existing_crate() { crate::test::async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!(remove_crate(&mut conn, "crate foo").await.is_err()); diff --git a/src/db/delete.rs b/src/db/delete.rs index 6a7135823..4ed445894 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -250,7 +250,7 @@ mod tests { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert!(get_id(&mut conn, "some-package").await.is_ok()); Ok(()) @@ -261,7 +261,7 @@ mod tests { #[test_case(false)] fn test_delete_crate(archive_storage: bool) { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; // Create fake packages in the database let pkg1_v1_id = env @@ -300,7 +300,6 @@ mod tests { ] { assert!( env.async_storage() - .await .rustdoc_file_exists( pkg, version, @@ -312,13 +311,7 @@ mod tests { ); } - delete_crate( - &mut conn, - &*env.async_storage().await, - &env.config(), - "package-1", - ) - .await?; + delete_crate(&mut conn, &*env.async_storage(), &env.config(), "package-1").await?; assert!(!crate_exists(&mut conn, "package-1").await?); assert!(crate_exists(&mut conn, "package-2").await?); @@ -329,7 +322,6 @@ mod tests { // files for package 2 still exists assert!( env.async_storage() - .await .rustdoc_file_exists( "package-2", "1.0.0", @@ -344,20 +336,17 @@ mod tests { if archive_storage { assert!( !env.async_storage() - .await .exists(&rustdoc_archive_path("package-1", "1.0.0")) .await? ); assert!( !env.async_storage() - .await .exists(&rustdoc_archive_path("package-1", "2.0.0")) .await? ); } else { assert!( !env.async_storage() - .await .rustdoc_file_exists( "package-1", "1.0.0", @@ -369,7 +358,6 @@ mod tests { ); assert!( !env.async_storage() - .await .rustdoc_file_exists( "package-1", "2.0.0", @@ -418,7 +406,7 @@ mod tests { .await } - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let v1 = env .fake_release() .await @@ -435,11 +423,10 @@ mod tests { assert!(release_exists(&mut conn, v1).await?); assert!( env.async_storage() - .await .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage().await, "1.0.0").await?); + assert!(json_exists(&*env.async_storage(), "1.0.0").await?); let crate_id = sqlx::query_scalar!( r#"SELECT crate_id as "crate_id: CrateId" FROM releases WHERE id = $1"#, v1.0 @@ -467,11 +454,10 @@ mod tests { assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .await .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage().await, "2.0.0").await?); + assert!(json_exists(&*env.async_storage(), "2.0.0").await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] @@ -479,7 +465,7 @@ mod tests { delete_version( &mut conn, - &*env.async_storage().await, + &*env.async_storage(), &env.config(), "a", "1.0.0", @@ -490,11 +476,11 @@ mod tests { // for archive storage the archive and index files // need to be cleaned up. let rustdoc_archive = rustdoc_archive_path("a", "1.0.0"); - assert!(!env.async_storage().await.exists(&rustdoc_archive).await?); + assert!(!env.async_storage().exists(&rustdoc_archive).await?); // local and remote index are gone too let archive_index = format!("{rustdoc_archive}.index"); - assert!(!env.async_storage().await.exists(&archive_index).await?); + assert!(!env.async_storage().exists(&archive_index).await?); assert!( !env.config() .local_archive_cache_path @@ -504,21 +490,19 @@ mod tests { } else { assert!( !env.async_storage() - .await .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) .await? ); } - assert!(!json_exists(&*env.async_storage().await, "1.0.0").await?); + assert!(!json_exists(&*env.async_storage(), "1.0.0").await?); assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .await .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage().await, "2.0.0").await?); + assert!(json_exists(&*env.async_storage(), "2.0.0").await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] @@ -536,7 +520,7 @@ mod tests { #[test] fn test_delete_incomplete_version() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let (release_id, _) = @@ -545,7 +529,7 @@ mod tests { delete_version( &mut conn, - &*env.async_storage().await, + &*env.async_storage(), &env.config(), "a", "1.0.0", @@ -561,14 +545,14 @@ mod tests { #[test] fn test_delete_incomplete_crate() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let (release_id, _) = fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") .await?; - delete_crate(&mut conn, &*env.async_storage().await, &env.config(), "a").await?; + delete_crate(&mut conn, &*env.async_storage(), &env.config(), "a").await?; assert!(!crate_exists(&mut conn, "a").await?); assert!(!release_exists(&mut conn, release_id).await?); diff --git a/src/db/overrides.rs b/src/db/overrides.rs index a11e0f3e8..4329c1c0e 100644 --- a/src/db/overrides.rs +++ b/src/db/overrides.rs @@ -91,7 +91,7 @@ mod test { #[test] fn retrieve_overrides() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let krate = "hexponent"; diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs index 815169272..ac8ef8af1 100644 --- a/src/docbuilder/limits.rs +++ b/src/docbuilder/limits.rs @@ -76,7 +76,7 @@ mod test { #[test] fn retrieve_limits() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let defaults = Limits::new(&env.config()); @@ -134,7 +134,7 @@ mod test { #[test] fn targets_default_to_one_with_timeout() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let krate = "hexponent"; Overrides::save( @@ -160,7 +160,7 @@ mod test { config.build_default_memory_limit = Some(6 * GB); }); - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let limits = Limits::for_crate(&env.config(), &mut conn, "krate").await?; @@ -173,7 +173,7 @@ mod test { #[test] fn overrides_dont_lower_memory_limit() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let defaults = Limits::new(&env.config()); diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 99a1f42a6..1b0edd9c1 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1296,7 +1296,7 @@ mod tests { version: &str, ) -> Result>, sqlx::Error> { env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query_scalar!( r#"SELECT releases.features "features?: Vec" @@ -1356,7 +1356,7 @@ mod tests { storage.store_one(&old_rustdoc_file, Vec::new())?; storage.store_one(&old_source_file, Vec::new())?; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1366,7 +1366,7 @@ mod tests { // check release record in the db (default and other targets) let row = env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( r#"SELECT r.rustdoc_status, @@ -1548,7 +1548,7 @@ mod tests { let crate_ = DUMMY_CRATE_NAME; let version = DUMMY_CRATE_VERSION; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1583,7 +1583,7 @@ mod tests { storage.store_one(&old_rustdoc_file, Vec::new())?; storage.store_one(&old_source_file, Vec::new())?; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( !builder @@ -1593,7 +1593,7 @@ mod tests { // check release record in the db (default and other targets) let row = env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( "SELECT r.rustdoc_status, @@ -1642,7 +1642,7 @@ mod tests { // create a successful release & build in the database let release_id = env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, crate_).await?; let release_id = initialize_release(&mut conn, crate_id, version).await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -1687,7 +1687,7 @@ mod tests { fn check_rustdoc_status(env: &TestEnvironment, rid: ReleaseId) -> Result<()> { assert_eq!( env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query_scalar!( "SELECT rustdoc_status FROM releases WHERE id = $1", rid.0 @@ -1702,7 +1702,7 @@ mod tests { check_rustdoc_status(env, release_id)?; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( // not successful build @@ -1722,7 +1722,7 @@ mod tests { #[ignore] fn test_proc_macro(crate_: &str, version: &str) { wrapper(|env| { - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1750,7 +1750,7 @@ mod tests { wrapper(|env| { let crate_ = "windows-win"; let version = "2.4.1"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; if builder.toolchain.as_ci().is_some() { return Ok(()); @@ -1809,7 +1809,7 @@ mod tests { // * there is a newer version of the dependency available that correctly builds let crate_ = "docs_rs_test_incorrect_lockfile"; let version = "0.1.2"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1836,7 +1836,7 @@ mod tests { // would not have had its details pulled down from the sparse-index. let crate_ = "docs_rs_test_incorrect_lockfile"; let version = "0.2.0"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1854,7 +1854,7 @@ mod tests { wrapper(|env| { let crate_ = "proc-macro2"; let version = "1.0.95"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1875,7 +1875,7 @@ mod tests { // added. Will fail when we try to build. let crate_ = "simconnect-sys"; let version = "0.23.1"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` @@ -1904,7 +1904,7 @@ mod tests { // package without Cargo.toml, so fails directly in the fetch stage. let crate_ = "emheap"; let version = "0.1.0"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` @@ -1914,7 +1914,7 @@ mod tests { assert!(summary.should_reattempt); let row = env.runtime().block_on(async { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( r#"SELECT rustc_version, @@ -1948,7 +1948,7 @@ mod tests { wrapper(|env| { let crate_ = "serde"; let version = "1.0.152"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1973,7 +1973,7 @@ mod tests { wrapper(|env| { let crate_ = "stylish-core"; let version = "0.1.1"; - let mut builder = RustwideBuilder::init(env).unwrap(); + let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder @@ -1996,7 +1996,7 @@ mod tests { #[ignore] fn test_build_std() { wrapper(|env| { - let mut builder = RustwideBuilder::init(env)?; + let mut builder = RustwideBuilder::init(&env.context)?; builder.update_toolchain()?; assert!( builder @@ -2011,9 +2011,9 @@ mod tests { #[ignore] fn test_workspace_reinitialize_at_once() { wrapper(|env| { - let mut builder = RustwideBuilder::init(env)?; + let mut builder = RustwideBuilder::init(&env.context)?; builder.update_toolchain()?; - builder.reinitialize_workspace_if_interval_passed(env)?; + builder.reinitialize_workspace_if_interval_passed(&env.context)?; assert!( builder .build_local_package(Path::new("tests/crates/build-std"))? @@ -2032,7 +2032,7 @@ mod tests { env.override_config(|cfg: &mut Config| { cfg.build_workspace_reinitialization_interval = Duration::from_secs(1) }); - let mut builder = RustwideBuilder::init(env)?; + let mut builder = RustwideBuilder::init(&env.context)?; builder.update_toolchain()?; assert!( builder @@ -2040,7 +2040,7 @@ mod tests { .successful ); sleep(Duration::from_secs(1)); - builder.reinitialize_workspace_if_interval_passed(env)?; + builder.reinitialize_workspace_if_interval_passed(&env.context)?; assert!( builder .build_local_package(Path::new("tests/crates/build-std"))? @@ -2054,14 +2054,14 @@ mod tests { #[ignore] fn test_new_builder_detects_existing_rustc() { wrapper(|env: &TestEnvironment| { - let mut builder = RustwideBuilder::init(env)?; + let mut builder = RustwideBuilder::init(&env.context)?; builder.update_toolchain()?; let old_version = builder.rustc_version()?; drop(builder); // new builder should detect the existing rustc version from the previous builder // (simulating running `update-toolchain` and `build crate` in separate invocations) - let mut builder = RustwideBuilder::init(env)?; + let mut builder = RustwideBuilder::init(&env.context)?; assert!( builder .build_package( diff --git a/src/repositories/github.rs b/src/repositories/github.rs index 912551547..1c3603722 100644 --- a/src/repositories/github.rs +++ b/src/repositories/github.rs @@ -269,6 +269,7 @@ mod tests { use super::{Config, GitHub}; use crate::repositories::RateLimitReached; use crate::repositories::updater::{RepositoryForge, repository_name}; + use crate::test::TestEnvironment; async fn mock_server_and_github(config: &Config) -> (mockito::ServerGuard, GitHub) { let server = mockito::Server::new_async().await; @@ -281,8 +282,8 @@ mod tests { #[test] fn test_rate_limit_fail() { - crate::test::async_wrapper(|env| async move { - let mut config = env.base_config(); + crate::test::async_wrapper(|_env| async move { + let mut config = TestEnvironment::base_config(); config.github_accesstoken = Some("qsjdnfqdq".to_owned()); let (mut server, updater) = mock_server_and_github(&config).await; @@ -304,8 +305,8 @@ mod tests { #[test] fn test_rate_limit_manual() { - crate::test::async_wrapper(|env| async move { - let mut config = env.base_config(); + crate::test::async_wrapper(|_env| async move { + let mut config = TestEnvironment::base_config(); config.github_accesstoken = Some("qsjdnfqdq".to_owned()); let (mut server, updater) = mock_server_and_github(&config).await; @@ -325,8 +326,8 @@ mod tests { #[test] fn not_found() { - crate::test::async_wrapper(|env| async move { - let mut config = env.base_config(); + crate::test::async_wrapper(|_env| async move { + let mut config = TestEnvironment::base_config(); config.github_accesstoken = Some("qsjdnfqdq".to_owned()); let (mut server, updater) = mock_server_and_github(&config).await; @@ -352,8 +353,8 @@ mod tests { #[test] fn get_repository_info() { - crate::test::async_wrapper(|env| async move { - let mut config = env.base_config(); + crate::test::async_wrapper(|_env| async move { + let mut config = TestEnvironment::base_config(); config.github_accesstoken = Some("qsjdnfqdq".to_owned()); let (mut server, updater) = mock_server_and_github(&config).await; diff --git a/src/test/mod.rs b/src/test/mod.rs index e137943d5..06ea41918 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -17,7 +17,6 @@ use axum::{Router, body::Body, http::Request, response::Response as AxumResponse use fn_error_context::context; use futures_util::{FutureExt, stream::TryStreamExt}; use http_body_util::BodyExt; // for `collect` -use once_cell::sync::OnceCell; use serde::de::DeserializeOwned; use sqlx::Connection as _; use std::{fs, future::Future, panic, rc::Rc, str::FromStr, sync::Arc}; @@ -351,19 +350,8 @@ impl AxumRouterTestExt for axum::Router { } pub(crate) struct TestEnvironment { - build_queue: OnceCell>, - async_build_queue: tokio::sync::OnceCell>, - config: OnceCell>, - db: tokio::sync::OnceCell, - storage: OnceCell>, - async_storage: tokio::sync::OnceCell>, - cdn: tokio::sync::OnceCell>, - index: OnceCell>, - registry_api: OnceCell>, - runtime: OnceCell>, - instance_metrics: OnceCell>, - service_metrics: OnceCell>, - repository_stats_updater: OnceCell>, + pub context: Context, + db: TestDatabase, } pub(crate) fn init_logger() { @@ -385,38 +373,95 @@ pub(crate) fn init_logger() { impl TestEnvironment { fn new() -> Self { init_logger(); + + let config = Arc::new(TestEnvironment::base_config()); + let runtime = Arc::new( + Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to initialize runtime"), + ); + + let instance_metrics = + Arc::new(InstanceMetrics::new().expect("failed to initialize the instance metrics")); + + let test_db = TestDatabase::new(&config, runtime.clone(), instance_metrics.clone()) + .expect("can't initialize test database"); + let pool = test_db.pool(); + + let async_storage = Arc::new( + runtime + .block_on(AsyncStorage::new( + pool.clone(), + instance_metrics.clone(), + config.clone(), + )) + .expect("can't create async storage"), + ); + + let async_build_queue = Arc::new(AsyncBuildQueue::new( + pool.clone(), + instance_metrics.clone(), + config.clone(), + async_storage.clone(), + )); + + let build_queue = Arc::new(BuildQueue::new(runtime.clone(), async_build_queue.clone())); + + let storage = Arc::new(Storage::new(async_storage.clone(), runtime.clone())); + + let cdn = Arc::new(runtime.block_on(CdnBackend::new(&config))); + + let index = Arc::new({ + let path = config.registry_index_path.clone(); + if let Some(registry_url) = config.registry_url.clone() { + Index::from_url(path, registry_url) + } else { + Index::new(path) + } + .expect("can't create Index") + }); + Self { - build_queue: OnceCell::new(), - async_build_queue: tokio::sync::OnceCell::new(), - config: OnceCell::new(), - db: tokio::sync::OnceCell::new(), - storage: OnceCell::new(), - async_storage: tokio::sync::OnceCell::new(), - cdn: tokio::sync::OnceCell::new(), - index: OnceCell::new(), - registry_api: OnceCell::new(), - instance_metrics: OnceCell::new(), - service_metrics: OnceCell::new(), - runtime: OnceCell::new(), - repository_stats_updater: OnceCell::new(), + context: Context { + config: config.clone(), + async_build_queue, + build_queue, + storage, + async_storage, + cdn, + pool: pool.clone(), + service_metrics: Arc::new( + ServiceMetrics::new().expect("can't initialize service metrics"), + ), + instance_metrics, + index, + registry_api: Arc::new( + RegistryApi::new( + config.registry_api_host.clone(), + config.crates_io_api_call_retries, + ) + .expect("can't create registry api"), + ), + repository_stats_updater: Arc::new(RepositoryStatsUpdater::new(&config, pool)), + runtime, + }, + db: test_db, } } fn cleanup(self) { - if let Some(storage) = self.storage.get() { - storage - .cleanup_after_test() - .expect("failed to cleanup after tests"); - } + self.context + .storage + .cleanup_after_test() + .expect("failed to cleanup after tests"); - if let Some(config) = self.config.get() - && config.local_archive_cache_path.exists() - { - fs::remove_dir_all(&config.local_archive_cache_path).unwrap(); + if self.context.config.local_archive_cache_path.exists() { + fs::remove_dir_all(&self.context.config.local_archive_cache_path).unwrap(); } } - pub(crate) fn base_config(&self) -> Config { + pub(crate) fn base_config() -> Config { let mut config = Config::from_env().expect("failed to get base config"); // create index directory @@ -446,230 +491,79 @@ impl TestEnvironment { } pub(crate) fn override_config(&self, f: impl FnOnce(&mut Config)) { - let mut config = self.base_config(); + let mut config = Self::base_config(); f(&mut config); - if self.config.set(Arc::new(config)).is_err() { + if self.context.config.set(Arc::new(config)).is_err() { panic!("can't call override_config after the configuration is accessed!"); } } - pub(crate) async fn async_build_queue(&self) -> Arc { - self.async_build_queue - .get_or_init(|| async { - Arc::new(AsyncBuildQueue::new( - self.async_db().await.pool(), - self.instance_metrics(), - self.config(), - self.async_storage().await, - )) - }) - .await - .clone() + pub(crate) fn async_build_queue(&self) -> Arc { + self.context.async_build_queue.clone() } pub(crate) fn build_queue(&self) -> Arc { - let runtime = self.runtime(); - self.build_queue - .get_or_init(|| { - Arc::new(BuildQueue::new( - runtime.clone(), - runtime.block_on(self.async_build_queue()), - )) - }) - .clone() + self.context.build_queue.clone() } - pub(crate) async fn cdn(&self) -> Arc { - self.cdn - .get_or_init(|| async { Arc::new(CdnBackend::new(&self.config()).await) }) - .await - .clone() + pub(crate) fn cdn(&self) -> Arc { + self.context.cdn.clone() } pub(crate) fn config(&self) -> Arc { - self.config - .get_or_init(|| Arc::new(self.base_config())) - .clone() + self.context.config.clone() } - pub(crate) async fn async_storage(&self) -> Arc { - self.async_storage - .get_or_init(|| async { - let db = self.async_db().await; - Arc::new( - AsyncStorage::new(db.pool(), self.instance_metrics(), self.config()) - .await - .expect("failed to initialize the async storage"), - ) - }) - .await - .clone() + pub(crate) fn async_storage(&self) -> Arc { + self.context.async_storage.clone() } pub(crate) fn storage(&self) -> Arc { - let runtime = self.runtime(); - self.storage - .get_or_init(|| { - Arc::new(Storage::new( - runtime.block_on(self.async_storage()), - runtime, - )) - }) - .clone() + self.context.storage.clone() } pub(crate) fn instance_metrics(&self) -> Arc { - self.instance_metrics - .get_or_init(|| { - Arc::new(InstanceMetrics::new().expect("failed to initialize the instance metrics")) - }) - .clone() + self.context.instance_metrics.clone() } pub(crate) fn service_metrics(&self) -> Arc { - self.service_metrics - .get_or_init(|| { - Arc::new(ServiceMetrics::new().expect("failed to initialize the service metrics")) - }) - .clone() + self.context.service_metrics.clone() } pub(crate) fn runtime(&self) -> Arc { - self.runtime - .get_or_init(|| { - Arc::new( - Builder::new_current_thread() - .enable_all() - .build() - .expect("failed to initialize runtime"), - ) - }) - .clone() + self.context.runtime.clone() } pub(crate) fn index(&self) -> Arc { - self.index - .get_or_init(|| { - Arc::new( - Index::new(self.config().registry_index_path.clone()) - .expect("failed to initialize the index"), - ) - }) - .clone() + self.context.index.clone() } pub(crate) fn registry_api(&self) -> Arc { - self.registry_api - .get_or_init(|| { - Arc::new( - RegistryApi::new( - self.config().registry_api_host.clone(), - self.config().crates_io_api_call_retries, - ) - .expect("failed to initialize the registry api"), - ) - }) - .clone() + self.context.registry_api.clone() } pub(crate) fn repository_stats_updater(&self) -> Arc { - self.repository_stats_updater - .get_or_init(|| { - Arc::new(RepositoryStatsUpdater::new( - &self.config(), - self.pool().expect("failed to get the pool"), - )) - }) - .clone() + self.context.repository_stats_updater.clone() } pub(crate) fn db(&self) -> &TestDatabase { - self.runtime().block_on(self.async_db()) + &self.db } - pub(crate) async fn async_db(&self) -> &TestDatabase { - self.db - .get_or_init(|| async { - let config = self.config(); - let runtime = self.runtime(); - let instance_metrics = self.instance_metrics(); - self.runtime() - .spawn_blocking(move || TestDatabase::new(&config, runtime, instance_metrics)) - .await - .unwrap() - .expect("failed to initialize the db") - }) - .await + pub(crate) fn async_db(&self) -> &TestDatabase { + &self.db } pub(crate) async fn web_app(&self) -> Router { let template_data = Arc::new(TemplateData::new(1).unwrap()); - build_axum_app(self, template_data) + build_axum_app(&self.context, template_data) .await .expect("could not build axum app") } pub(crate) async fn fake_release(&self) -> fakes::FakeRelease<'_> { - fakes::FakeRelease::new(self.async_db().await, self.async_storage().await) - } -} - -impl Context for TestEnvironment { - fn config(&self) -> Result> { - Ok(TestEnvironment::config(self)) - } - - async fn async_build_queue(&self) -> Result> { - Ok(TestEnvironment::async_build_queue(self).await) - } - - fn build_queue(&self) -> Result> { - Ok(TestEnvironment::build_queue(self)) - } - - fn storage(&self) -> Result> { - Ok(TestEnvironment::storage(self)) - } - - async fn async_storage(&self) -> Result> { - Ok(TestEnvironment::async_storage(self).await) - } - - async fn cdn(&self) -> Result> { - Ok(TestEnvironment::cdn(self).await) - } - - async fn async_pool(&self) -> Result { - Ok(self.async_db().await.pool()) - } - - fn pool(&self) -> Result { - Ok(self.db().pool()) - } - - fn instance_metrics(&self) -> Result> { - Ok(self.instance_metrics()) - } - - fn service_metrics(&self) -> Result> { - Ok(self.service_metrics()) - } - - fn index(&self) -> Result> { - Ok(self.index()) - } - - fn registry_api(&self) -> Result> { - Ok(self.registry_api()) - } - - fn repository_stats_updater(&self) -> Result> { - Ok(self.repository_stats_updater()) - } - - fn runtime(&self) -> Result> { - Ok(self.runtime()) + fakes::FakeRelease::new(self.async_db(), self.async_storage()) } } diff --git a/src/utils/consistency/db.rs b/src/utils/consistency/db.rs index 673561a08..8076a3ff8 100644 --- a/src/utils/consistency/db.rs +++ b/src/utils/consistency/db.rs @@ -68,7 +68,6 @@ mod tests { fn test_load() { async_wrapper(|env| async move { env.async_build_queue() - .await .add_crate("queued", "0.0.1", 0, None) .await?; env.fake_release() @@ -85,7 +84,7 @@ mod tests { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let result = load(&mut conn, &env.config()).await?; assert_eq!( diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index 3995a5ab3..acb90588d 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -27,7 +27,7 @@ const BUILD_PRIORITY: i32 = 15; pub async fn run_check(ctx: &Context, dry_run: bool) -> Result<()> { info!("Loading data from database..."); let mut conn = ctx.pool.get_async().await?; - let db_data = db::load(&mut conn, &*ctx.config) + let db_data = db::load(&mut conn, &ctx.config) .await .context("Loading crate data from database for consistency check")?; @@ -163,7 +163,7 @@ mod tests { use sqlx::Row as _; async fn count(env: &TestEnvironment, sql: &str) -> Result { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; Ok(sqlx::query_scalar(sql).fetch_one(&mut *conn).await?) } @@ -171,7 +171,7 @@ mod tests { where O: Send + Unpin + for<'r> sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type, { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; Ok::<_, anyhow::Error>( sqlx::query(sql) .fetch_all(&mut *conn) @@ -196,7 +196,7 @@ mod tests { let diff = [Difference::CrateNotInIndex("krate".into())]; // calling with dry-run leads to no change - handle_diff(&*env, diff.iter(), true).await?; + handle_diff(&env.context, diff.iter(), true).await?; assert_eq!( count(&env, "SELECT count(*) FROM crates WHERE name = 'krate'").await?, @@ -204,7 +204,7 @@ mod tests { ); // without dry-run the crate will be deleted - handle_diff(&*env, diff.iter(), false).await?; + handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( count(&env, "SELECT count(*) FROM crates WHERE name = 'krate'").await?, @@ -238,11 +238,11 @@ mod tests { assert_eq!(count(&env, "SELECT count(*) FROM releases").await?, 2); - handle_diff(&*env, diff.iter(), true).await?; + handle_diff(&env.context, diff.iter(), true).await?; assert_eq!(count(&env, "SELECT count(*) FROM releases").await?, 2); - handle_diff(&*env, diff.iter(), false).await?; + handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( single_row::(&env, "SELECT version FROM releases").await?, @@ -270,14 +270,14 @@ mod tests { false, )]; - handle_diff(&*env, diff.iter(), true).await?; + handle_diff(&env.context, diff.iter(), true).await?; assert_eq!( single_row::(&env, "SELECT yanked FROM releases").await?, vec![true] ); - handle_diff(&*env, diff.iter(), false).await?; + handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( single_row::(&env, "SELECT yanked FROM releases").await?, @@ -293,13 +293,13 @@ mod tests { async_wrapper(|env| async move { let diff = [Difference::ReleaseNotInDb("krate".into(), "0.1.1".into())]; - handle_diff(&*env, diff.iter(), true).await?; + handle_diff(&env.context, diff.iter(), true).await?; - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); assert!(build_queue.queued_crates().await?.is_empty()); - handle_diff(&*env, diff.iter(), false).await?; + handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( build_queue @@ -322,13 +322,13 @@ mod tests { vec!["0.1.1".into(), "0.1.2".into()], )]; - handle_diff(&*env, diff.iter(), true).await?; + handle_diff(&env.context, diff.iter(), true).await?; - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); assert!(build_queue.queued_crates().await?.is_empty()); - handle_diff(&*env, diff.iter(), false).await?; + handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( build_queue diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 2cbd014e9..f5308b9aa 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -186,29 +186,29 @@ pub fn start_daemon(context: Context, enable_registry_watcher: bool) -> Result<( info!("Starting web server"); let webserver_thread = thread::spawn({ let context = context.clone(); - move || start_web_server(None, &*context) + move || start_web_server(None, &context) }); if enable_registry_watcher { // check new crates every minute - start_registry_watcher(&*context)?; + start_registry_watcher(&context)?; } // build new crates every minute let build_queue = context.build_queue.clone(); let config = context.config.clone(); - let rustwide_builder = RustwideBuilder::init(&*context)?; + let rustwide_builder = RustwideBuilder::init(&context)?; thread::Builder::new() .name("build queue reader".to_string()) .spawn({ let context = context.clone(); - move || queue_builder(&*context, rustwide_builder, build_queue, config).unwrap() + move || queue_builder(&context, rustwide_builder, build_queue, config).unwrap() }) .unwrap(); - start_background_repository_stats_updater(&*context)?; - start_background_cdn_invalidator(&*context)?; - start_background_queue_rebuild(&*context)?; + start_background_repository_stats_updater(&context)?; + start_background_cdn_invalidator(&context)?; + start_background_queue_rebuild(&context)?; // NOTE: if a error occurred earlier in `start_daemon`, the server will _not_ be joined - // instead it will get killed when the process exits. diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 836fad2d3..bbde45481 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -191,7 +191,7 @@ mod tests { #[test] fn test_get_config_empty() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!("DELETE FROM config") .execute(&mut *conn) .await?; @@ -208,7 +208,7 @@ mod tests { #[test] fn test_set_and_get_config_() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!("DELETE FROM config") .execute(&mut *conn) .await?; diff --git a/src/utils/queue.rs b/src/utils/queue.rs index fee83b2a7..7df91672a 100644 --- a/src/utils/queue.rs +++ b/src/utils/queue.rs @@ -80,7 +80,7 @@ mod tests { #[test] fn set_priority() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; set_crate_priority(&mut conn, "docsrs-%", -100).await?; @@ -121,7 +121,7 @@ mod tests { #[test] fn remove_priority() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; set_crate_priority(&mut conn, "docsrs-%", -100).await?; @@ -143,7 +143,7 @@ mod tests { #[test] fn get_priority() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; set_crate_priority(&mut conn, "docsrs-%", -100).await?; @@ -170,7 +170,7 @@ mod tests { #[test] fn get_default_priority() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; assert_eq!( diff --git a/src/web/build_details.rs b/src/web/build_details.rs index 25aadd3c5..35c787156 100644 --- a/src/web/build_details.rs +++ b/src/web/build_details.rs @@ -183,7 +183,7 @@ mod tests { #[test] fn test_partial_build_result() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let (_, build_id) = fake_release_that_failed_before_build( &mut conn, "foo", @@ -214,7 +214,7 @@ mod tests { #[test] fn test_partial_build_result_plus_default_target_from_previous_build() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let (release_id, build_id) = fake_release_that_failed_before_build( &mut conn, "foo", diff --git a/src/web/builds.rs b/src/web/builds.rs index a8c1261dd..a5766da86 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -218,7 +218,7 @@ mod tests { #[test] fn build_list_empty_build() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; let response = env @@ -386,7 +386,7 @@ mod tests { ); } - let build_queue = env.async_build_queue().await; + let build_queue = env.async_build_queue(); assert_eq!(build_queue.pending_count().await?, 0); assert!(!build_queue.has_build_queued("foo", "0.1.0").await?); @@ -488,7 +488,7 @@ mod tests { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let limits = Overrides { memory: Some(6 * 1024 * 1024 * 1024), targets: Some(1), diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 7f4d9ef43..5f59f3ae1 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -923,7 +923,7 @@ mod tests { #[test] fn test_crate_details_documentation_url_is_none_when_url_is_docs_rs() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; env.fake_release() @@ -966,7 +966,7 @@ mod tests { #[test] fn test_last_successful_build_when_last_releases_failed_or_yanked() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1015,7 +1015,7 @@ mod tests { #[test] fn test_last_successful_build_when_all_releases_failed_or_yanked() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1049,7 +1049,7 @@ mod tests { #[test] fn test_last_successful_build_with_intermittent_releases_failed_or_yanked() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1089,7 +1089,7 @@ mod tests { #[test] fn test_releases_should_be_sorted() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); // Add new releases of 'foo' out-of-order since CrateDetails should sort them descending env.fake_release() @@ -1275,7 +1275,7 @@ mod tests { #[test] fn test_latest_version() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1312,7 +1312,7 @@ mod tests { #[test] fn test_latest_version_ignores_prerelease() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1349,7 +1349,7 @@ mod tests { #[test] fn test_latest_version_ignores_yanked() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1387,7 +1387,7 @@ mod tests { #[test] fn test_latest_version_only_yanked() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1427,7 +1427,7 @@ mod tests { #[test] fn test_latest_version_in_progress() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1534,7 +1534,7 @@ mod tests { #[test] fn test_updating_owners() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -1820,7 +1820,7 @@ mod tests { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!("UPDATE releases SET features = NULL WHERE id = $1", id.0) .execute(&mut *conn) .await?; @@ -1841,7 +1841,7 @@ mod tests { #[test] fn test_minimal_failed_release_doesnt_error_features() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; let text_content = env @@ -1865,7 +1865,7 @@ mod tests { #[test] fn test_minimal_failed_release_doesnt_error() { async_wrapper(|env| async move { - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; let text_content = env @@ -2200,10 +2200,10 @@ mod tests { check_readme("/crate/dummy/0.3.0".into(), "storage readme".into()).await; check_readme("/crate/dummy/0.4.0".into(), "storage meread".into()).await; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; assert!(matches!( - details.fetch_readme(&*env.async_storage().await).await, + details.fetch_readme(&*env.async_storage()).await, Ok(None) )); Ok(()) @@ -2295,7 +2295,7 @@ path = "src/lib.rs" .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!("DELETE FROM builds") .execute(&mut *conn) .await?; @@ -2326,7 +2326,7 @@ path = "src/lib.rs" .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert_eq!( release_build_status(&mut conn, "dummy", "0.1.0").await, @@ -2351,7 +2351,7 @@ path = "src/lib.rs" .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert_eq!( release_build_status(&mut conn, "dummy", "0.1.0").await, @@ -2375,7 +2375,7 @@ path = "src/lib.rs" .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; assert_eq!( release_build_status(&mut conn, "dummy", "0.1.0").await, diff --git a/src/web/file.rs b/src/web/file.rs index cd04bb4df..e371f87f7 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -101,7 +101,7 @@ mod tests { env.fake_release().await.create().await?; let mut file = File::from_path( - &*env.async_storage().await, + &*env.async_storage(), "rustdoc/fake-package/1.0.0/fake-package/index.html", &env.config(), ) @@ -153,7 +153,7 @@ mod tests { let env = env.clone(); async move { File::from_path( - &*env.async_storage().await, + &*env.async_storage(), &format!("rustdoc/dummy/0.1.0/{path}"), &env.config(), ) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 467f81a79..7b7ec5cd8 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -110,7 +110,6 @@ pub(crate) async fn request_recorder( #[cfg(test)] mod tests { - use crate::Context; use crate::test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}; use std::collections::HashMap; @@ -187,7 +186,7 @@ mod tests { } // this shows what the routes were *actually* recorded as, making it easier to update ROUTES if the name changes. - let metrics_serialized = metrics.gather(&env.pool().await?)?; + let metrics_serialized = metrics.gather(&env.context.pool)?; let all_routes_visited = metrics_serialized .iter() .find(|x| x.name() == "docsrs_routes_visited") diff --git a/src/web/mod.rs b/src/web/mod.rs index 4a5709b6b..89dabb2f3 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -1019,7 +1019,7 @@ mod test { // https://github.com/rust-lang/docs.rs/issues/223 fn prereleases_are_not_considered_for_semver() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let version = |v| version(v, db); let release = |v| release(v, &env); @@ -1079,7 +1079,7 @@ mod test { // https://github.com/rust-lang/docs.rs/issues/221 fn yanked_crates_are_not_considered() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let release_id = release("0.3.0", &env).await; @@ -1104,7 +1104,7 @@ mod test { #[test] fn in_progress_releases_are_ignored_when_others_match() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); // normal release release("1.0.0", &env).await; @@ -1134,7 +1134,7 @@ mod test { // https://github.com/rust-lang/docs.rs/issues/1682 fn prereleases_are_considered_when_others_dont_match() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); // normal release release("1.0.0", &env).await; @@ -1159,7 +1159,7 @@ mod test { // vaguely related to https://github.com/rust-lang/docs.rs/issues/395 fn metadata_has_no_effect() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); release("0.1.0+4.1", &env).await; release("0.1.1", &env).await; @@ -1255,7 +1255,7 @@ mod test { fn metadata_from_crate() { async_wrapper(|env| async move { release("0.1.0", &env).await; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; let metadata = MetaData::from_crate( &mut conn, "foo", diff --git a/src/web/releases.rs b/src/web/releases.rs index 9ba6856c5..0f8146f15 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -828,7 +828,7 @@ mod tests { #[test] fn test_release_list_with_incomplete_release_and_successful_build() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); let mut conn = db.async_conn().await; let crate_id = initialize_crate(&mut conn, "foo").await?; @@ -863,7 +863,7 @@ mod tests { #[test] fn get_releases_by_stars() { async_wrapper(|env| async move { - let db = env.async_db().await; + let db = env.async_db(); env.fake_release() .await @@ -986,7 +986,7 @@ mod tests { // crate in the db breaks this test. // That's why we reset the id-sequence to zero for this test. - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!(r#"ALTER SEQUENCE crates_id_seq RESTART WITH 1"#) .execute(&mut *conn) .await?; @@ -1443,7 +1443,7 @@ mod tests { .await?; // release that failed in the fetch-step, will miss some details - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; fake_release_that_failed_before_build( &mut conn, "failed_hard", @@ -1802,7 +1802,7 @@ mod tests { let web = env.web_app().await; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; cdn::queue_crate_invalidation(&mut conn, &env.config(), "krate_2").await?; let content = @@ -1849,7 +1849,7 @@ mod tests { .any(|el| el.text_contents().contains("active CDN deployments")) ); - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); queue.add_crate("foo", "1.0.0", 0, None).await?; queue.add_crate("bar", "0.1.0", -10, None).await?; queue.add_crate("baz", "0.0.1", 10, None).await?; @@ -1889,7 +1889,7 @@ mod tests { let web = env.web_app().await; // we have two queued releases, where the build for one is already in progress - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); queue.add_crate("foo", "1.0.0", 0, None).await?; queue.add_crate("bar", "0.1.0", 0, None).await?; @@ -1964,7 +1964,7 @@ mod tests { fn test_releases_rebuild_queue_with_crates() { async_wrapper(|env| async move { let web = env.web_app().await; - let queue = env.async_build_queue().await; + let queue = env.async_build_queue(); queue .add_crate("foo", "1.0.0", REBUILD_PRIORITY, None) .await?; diff --git a/src/web/routes.rs b/src/web/routes.rs index c6782b4cd..0f744e700 100644 --- a/src/web/routes.rs +++ b/src/web/routes.rs @@ -459,7 +459,7 @@ mod tests { fn serve_rustdoc_content() { async_wrapper(|env| async move { let web = env.web_app().await; - let storage = env.async_storage().await; + let storage = env.async_storage(); storage .store_one("/rustdoc-static/style.css", "content".as_bytes()) .await?; diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 9959b1210..010609437 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -2398,7 +2398,7 @@ mod test { .create() .await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; // https://stackoverflow.com/questions/18209625/how-do-i-modify-fields-inside-the-new-postgresql-json-datatype sqlx::query!( r#"UPDATE releases SET dependencies = dependencies::jsonb #- '{0,2}' WHERE id = $1"#, id.0 @@ -3057,7 +3057,6 @@ mod test { .await?; assert!( env.async_storage() - .await .get_public_access("rustdoc/dummy/0.1.0.zip") .await? ); @@ -3080,7 +3079,7 @@ mod test { .await?; let web = env.web_app().await; - let storage = env.async_storage().await; + let storage = env.async_storage(); // disable public access to be sure that the handler will enable it storage @@ -3132,7 +3131,6 @@ mod test { .await?; assert!( env.async_storage() - .await .get_public_access("rustdoc/dummy/0.2.0.zip") .await? ); @@ -3175,7 +3173,7 @@ mod test { .create() .await?; - let storage = env.async_storage().await; + let storage = env.async_storage(); storage.store_one("asset.js", *b"content").await?; storage.store_one(path, *b"more_content").await?; @@ -3439,7 +3437,7 @@ mod test { .create() .await?; - let storage = env.async_storage().await; + let storage = env.async_storage(); let zstd_blob = storage .get( diff --git a/src/web/source.rs b/src/web/source.rs index 730f43725..ddf9fece2 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -506,7 +506,7 @@ mod tests { let web = env.web_app().await; web.assert_success(path).await?; - let mut conn = env.async_db().await.async_conn().await; + let mut conn = env.async_db().async_conn().await; sqlx::query!( "UPDATE releases SET files = NULL From b2020a30c3c827c32b85e5dc5c4ed5b6363f4bc4 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 22 Aug 2025 21:27:49 +0200 Subject: [PATCH 07/11] no swap --- Cargo.lock | 1 - Cargo.toml | 1 - src/test/mod.rs | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb302404e..cb2c821a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2005,7 +2005,6 @@ name = "docs-rs" version = "0.6.0" dependencies = [ "anyhow", - "arc-swap", "askama", "async-compression", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index ff7931158..c522d0e46 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,6 @@ exclude = [ ] [dependencies] -arc-swap = "1.7.1" sentry = { version = "0.42.0", features = ["panic", "tracing", "tower-http", "anyhow", "backtrace"] } log = "0.4" tracing = "0.1.37" diff --git a/src/test/mod.rs b/src/test/mod.rs index 06ea41918..3e5a8efb0 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -12,6 +12,7 @@ use crate::{ ServiceMetrics, }; use anyhow::Context as _; +use arc_swap::ArcSwap; use axum::body::Bytes; use axum::{Router, body::Body, http::Request, response::Response as AxumResponse}; use fn_error_context::context; From f76d5627c9513c617e310965ce7abab888595384 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 28 Aug 2025 22:04:49 +0200 Subject: [PATCH 08/11] fixes --- src/cdn.rs | 8 +-- src/db/delete.rs | 30 +++-------- src/test/mod.rs | 105 ++++++++++----------------------------- src/web/crate_details.rs | 2 +- src/web/file.rs | 4 +- 5 files changed, 40 insertions(+), 109 deletions(-) diff --git a/src/cdn.rs b/src/cdn.rs index 17750615b..326c81225 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -1049,7 +1049,7 @@ mod tests { // handle the queued invalidations handle_queued_invalidation_requests( &env.config(), - &*env.cdn(), + &env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1112,7 +1112,7 @@ mod tests { // handle the queued invalidations handle_queued_invalidation_requests( &env.config(), - &*env.cdn(), + &env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1143,7 +1143,7 @@ mod tests { // now handle again handle_queued_invalidation_requests( &env.config(), - &*env.cdn(), + &env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", @@ -1187,7 +1187,7 @@ mod tests { // run the handler handle_queued_invalidation_requests( &env.config(), - &*env.cdn(), + &env.cdn(), &env.instance_metrics(), &mut conn, "distribution_id_web", diff --git a/src/db/delete.rs b/src/db/delete.rs index 4ed445894..503262820 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -311,7 +311,7 @@ mod tests { ); } - delete_crate(&mut conn, &*env.async_storage(), &env.config(), "package-1").await?; + delete_crate(&mut conn, &env.async_storage(), &env.config(), "package-1").await?; assert!(!crate_exists(&mut conn, "package-1").await?); assert!(crate_exists(&mut conn, "package-2").await?); @@ -426,7 +426,7 @@ mod tests { .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage(), "1.0.0").await?); + assert!(json_exists(&env.async_storage(), "1.0.0").await?); let crate_id = sqlx::query_scalar!( r#"SELECT crate_id as "crate_id: CrateId" FROM releases WHERE id = $1"#, v1.0 @@ -457,20 +457,13 @@ mod tests { .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage(), "2.0.0").await?); + assert!(json_exists(&env.async_storage(), "2.0.0").await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] ); - delete_version( - &mut conn, - &*env.async_storage(), - &env.config(), - "a", - "1.0.0", - ) - .await?; + delete_version(&mut conn, &env.async_storage(), &env.config(), "a", "1.0.0").await?; assert!(!release_exists(&mut conn, v1).await?); if archive_storage { // for archive storage the archive and index files @@ -494,7 +487,7 @@ mod tests { .await? ); } - assert!(!json_exists(&*env.async_storage(), "1.0.0").await?); + assert!(!json_exists(&env.async_storage(), "1.0.0").await?); assert!(release_exists(&mut conn, v2).await?); assert!( @@ -502,7 +495,7 @@ mod tests { .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(&*env.async_storage(), "2.0.0").await?); + assert!(json_exists(&env.async_storage(), "2.0.0").await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] @@ -527,14 +520,7 @@ mod tests { fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") .await?; - delete_version( - &mut conn, - &*env.async_storage(), - &env.config(), - "a", - "1.0.0", - ) - .await?; + delete_version(&mut conn, &env.async_storage(), &env.config(), "a", "1.0.0").await?; assert!(!release_exists(&mut conn, release_id).await?); @@ -552,7 +538,7 @@ mod tests { fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") .await?; - delete_crate(&mut conn, &*env.async_storage(), &env.config(), "a").await?; + delete_crate(&mut conn, &env.async_storage(), &env.config(), "a").await?; assert!(!crate_exists(&mut conn, "a").await?); assert!(!release_exists(&mut conn, release_id).await?); diff --git a/src/test/mod.rs b/src/test/mod.rs index 3e5a8efb0..0fb36c932 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -12,11 +12,10 @@ use crate::{ ServiceMetrics, }; use anyhow::Context as _; -use arc_swap::ArcSwap; use axum::body::Bytes; use axum::{Router, body::Body, http::Request, response::Response as AxumResponse}; use fn_error_context::context; -use futures_util::{FutureExt, stream::TryStreamExt}; +use futures_util::stream::TryStreamExt; use http_body_util::BodyExt; // for `collect` use serde::de::DeserializeOwned; use sqlx::Connection as _; @@ -28,24 +27,7 @@ use tracing::error; #[track_caller] pub(crate) fn wrapper(f: impl FnOnce(&TestEnvironment) -> Result<()>) { let env = TestEnvironment::new(); - // if we didn't catch the panic, the server would hang forever - let maybe_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| f(&env))); - env.cleanup(); - let result = match maybe_panic { - Ok(r) => r, - Err(payload) => panic::resume_unwind(payload), - }; - - if let Err(err) = result { - eprintln!("the test failed: {err}"); - for cause in err.chain() { - eprintln!(" caused by: {cause}"); - } - - eprintln!("{}", err.backtrace()); - - panic!("the test failed"); - } + f(&env).expect("test failed"); } pub(crate) fn async_wrapper(f: F) @@ -55,31 +37,7 @@ where { let env = Rc::new(TestEnvironment::new()); - let fut = f(env.clone()); - - let runtime = env.runtime(); - - // if we didn't catch the panic, the server would hang forever - let maybe_panic = runtime.block_on(panic::AssertUnwindSafe(fut).catch_unwind()); - - let env = Rc::into_inner(env).unwrap(); - env.cleanup(); - - let result = match maybe_panic { - Ok(r) => r, - Err(payload) => panic::resume_unwind(payload), - }; - - if let Err(err) = result { - eprintln!("the test failed: {err}"); - for cause in err.chain() { - eprintln!(" caused by: {cause}"); - } - - eprintln!("{}", err.backtrace()); - - panic!("the test failed"); - } + env.runtime().block_on(f(env.clone())).expect("test failed"); } pub(crate) trait AxumResponseTestExt { @@ -373,9 +331,13 @@ pub(crate) fn init_logger() { impl TestEnvironment { fn new() -> Self { + Self::with_config(Self::base_config()) + } + + fn with_config(config: Config) -> Self { init_logger(); - let config = Arc::new(TestEnvironment::base_config()); + let config = Arc::new(config); let runtime = Arc::new( Builder::new_current_thread() .enable_all() @@ -451,17 +413,6 @@ impl TestEnvironment { } } - fn cleanup(self) { - self.context - .storage - .cleanup_after_test() - .expect("failed to cleanup after tests"); - - if self.context.config.local_archive_cache_path.exists() { - fs::remove_dir_all(&self.context.config.local_archive_cache_path).unwrap(); - } - } - pub(crate) fn base_config() -> Config { let mut config = Config::from_env().expect("failed to get base config"); @@ -495,9 +446,10 @@ impl TestEnvironment { let mut config = Self::base_config(); f(&mut config); - if self.context.config.set(Arc::new(config)).is_err() { - panic!("can't call override_config after the configuration is accessed!"); - } + // FIXME: fix + // if self.context.config.set(Arc::new(config)).is_err() { + // panic!("can't call override_config after the configuration is accessed!"); + // } } pub(crate) fn async_build_queue(&self) -> Arc { @@ -528,30 +480,10 @@ impl TestEnvironment { self.context.instance_metrics.clone() } - pub(crate) fn service_metrics(&self) -> Arc { - self.context.service_metrics.clone() - } - pub(crate) fn runtime(&self) -> Arc { self.context.runtime.clone() } - pub(crate) fn index(&self) -> Arc { - self.context.index.clone() - } - - pub(crate) fn registry_api(&self) -> Arc { - self.context.registry_api.clone() - } - - pub(crate) fn repository_stats_updater(&self) -> Arc { - self.context.repository_stats_updater.clone() - } - - pub(crate) fn db(&self) -> &TestDatabase { - &self.db - } - pub(crate) fn async_db(&self) -> &TestDatabase { &self.db } @@ -568,6 +500,19 @@ impl TestEnvironment { } } +impl Drop for TestEnvironment { + fn drop(&mut self) { + self.context + .storage + .cleanup_after_test() + .expect("failed to cleanup after tests"); + + if self.context.config.local_archive_cache_path.exists() { + fs::remove_dir_all(&self.context.config.local_archive_cache_path).unwrap(); + } + } +} + #[derive(Debug)] pub(crate) struct TestDatabase { pool: Pool, diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 5f59f3ae1..49bff2fa2 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -2203,7 +2203,7 @@ mod tests { let mut conn = env.async_db().async_conn().await; let details = crate_details(&mut conn, "dummy", "0.5.0", None).await; assert!(matches!( - details.fetch_readme(&*env.async_storage()).await, + details.fetch_readme(&env.async_storage()).await, Ok(None) )); Ok(()) diff --git a/src/web/file.rs b/src/web/file.rs index e371f87f7..edb531cc2 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -101,7 +101,7 @@ mod tests { env.fake_release().await.create().await?; let mut file = File::from_path( - &*env.async_storage(), + &env.async_storage(), "rustdoc/fake-package/1.0.0/fake-package/index.html", &env.config(), ) @@ -153,7 +153,7 @@ mod tests { let env = env.clone(); async move { File::from_path( - &*env.async_storage(), + &env.async_storage(), &format!("rustdoc/dummy/0.1.0/{path}"), &env.config(), ) From 24b5bbef21399afa6ebc17a1f3d55523152ce2f0 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 20 Sep 2025 16:32:37 +0200 Subject: [PATCH 09/11] clean --- crates/crates_io_env_vars/Cargo.toml | 15 -- crates/crates_io_env_vars/README.md | 14 -- crates/crates_io_env_vars/src/lib.rs | 244 --------------------------- 3 files changed, 273 deletions(-) delete mode 100644 crates/crates_io_env_vars/Cargo.toml delete mode 100644 crates/crates_io_env_vars/README.md delete mode 100644 crates/crates_io_env_vars/src/lib.rs diff --git a/crates/crates_io_env_vars/Cargo.toml b/crates/crates_io_env_vars/Cargo.toml deleted file mode 100644 index a3059a41f..000000000 --- a/crates/crates_io_env_vars/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "crates_io_env_vars" -version = "0.0.0" -license = "MIT OR Apache-2.0" -edition = "2024" - -[lints] -workspace = true - -[dependencies] -anyhow = "=1.0.98" -dotenvy = "=0.15.7" - -[dev-dependencies] -claims = "=0.8.0" diff --git a/crates/crates_io_env_vars/README.md b/crates/crates_io_env_vars/README.md deleted file mode 100644 index d30947dd9..000000000 --- a/crates/crates_io_env_vars/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# crates_io_env_vars - -This package contains convenient wrappers for the `std::env::var()` function. - -These functions use the `dotenvy` crate to automatically load environment -variables from a `.env` file, if it exists. This is useful for development -environments, where you don't want to set all environment variables manually. - -There are also variants of the functions that make use of the `FromStr` trait to -automatically parse the environment variables into the desired types or fail -with corresponding error messages. - -Finally, there are `list()` functions that allow parsing of comma-separated -lists of values. diff --git a/crates/crates_io_env_vars/src/lib.rs b/crates/crates_io_env_vars/src/lib.rs deleted file mode 100644 index 0d349b4d7..000000000 --- a/crates/crates_io_env_vars/src/lib.rs +++ /dev/null @@ -1,244 +0,0 @@ -#![doc = include_str!("../README.md")] - -use anyhow::{Context, anyhow}; -use std::error::Error; -use std::str::FromStr; - -/// Reads an environment variable for the current process. -/// -/// Compared to [std::env::var] there are a couple of differences: -/// -/// - [var] uses [dotenvy] which loads the `.env` file from the current or -/// parent directories before returning the value. -/// -/// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable -/// wasn't set. -#[track_caller] -pub fn var(key: &str) -> anyhow::Result> { - match dotenvy::var(key) { - Ok(content) => Ok(Some(content)), - Err(dotenvy::Error::EnvVar(std::env::VarError::NotPresent)) => Ok(None), - Err(error) => Err(error.into()), - } -} - -/// Reads an environment variable for the current process, and fails if it was -/// not found. -/// -/// Compared to [std::env::var] there are a couple of differences: -/// -/// - [var] uses [dotenvy] which loads the `.env` file from the current or -/// parent directories before returning the value. -#[track_caller] -pub fn required_var(key: &str) -> anyhow::Result { - required(var(key), key) -} - -/// Reads an environment variable for the current process, and parses it if -/// it is set. -/// -/// Compared to [std::env::var] there are a couple of differences: -/// -/// - [var] uses [dotenvy] which loads the `.env` file from the current or -/// parent directories before returning the value. -/// -/// - [var] returns `Ok(None)` (instead of `Err`) if an environment variable -/// wasn't set. -#[track_caller] -pub fn var_parsed(key: &str) -> anyhow::Result> -where - R: FromStr, - R::Err: Error + Send + Sync + 'static, -{ - match var(key) { - Ok(Some(content)) => { - Ok(Some(content.parse().with_context(|| { - format!("Failed to parse {key} environment variable") - })?)) - } - Ok(None) => Ok(None), - Err(error) => Err(error), - } -} - -/// Reads an environment variable for the current process, and parses it if -/// it is set or fails otherwise. -/// -/// Compared to [std::env::var] there are a couple of differences: -/// -/// - [var] uses [dotenvy] which loads the `.env` file from the current or -/// parent directories before returning the value. -#[track_caller] -pub fn required_var_parsed(key: &str) -> anyhow::Result -where - R: FromStr, - R::Err: Error + Send + Sync + 'static, -{ - required(var_parsed(key), key) -} - -fn required(res: anyhow::Result>, key: &str) -> anyhow::Result { - match res { - Ok(opt) => opt.ok_or_else(|| anyhow!("Failed to find required {key} environment variable")), - Err(error) => Err(error), - } -} - -/// Reads an environment variable and parses it as a comma-separated list, or -/// returns an empty list if the variable is not set. -#[track_caller] -pub fn list(key: &str) -> anyhow::Result> { - let values = match var(key)? { - None => vec![], - Some(s) if s.is_empty() => vec![], - Some(s) => s.split(',').map(str::trim).map(String::from).collect(), - }; - - Ok(values) -} - -/// Reads an environment variable and parses it as a comma-separated list, or -/// returns an empty list if the variable is not set. Each individual value is -/// parsed using [FromStr]. -#[track_caller] -pub fn list_parsed(key: &str, f: F) -> anyhow::Result> -where - F: Fn(&str) -> C, - C: Context, -{ - let values = match var(key)? { - None => vec![], - Some(s) if s.is_empty() => vec![], - Some(s) => s - .split(',') - .map(str::trim) - .map(|s| { - f(s).with_context(|| { - format!("Failed to parse value \"{s}\" of {key} environment variable") - }) - }) - .collect::>()?, - }; - - Ok(values) -} - -#[cfg(test)] -mod tests { - use super::*; - use claims::*; - use std::sync::{LazyLock, Mutex}; - - const TEST_VAR: &str = "CRATES_IO_ENV_VARS_TEST_VAR"; - - /// A mutex to ensure that the tests don't run in parallel, since they all - /// modify the shared environment variable. - static MUTEX: LazyLock> = LazyLock::new(|| Mutex::new(())); - - #[test] - fn test_var() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "test") }; - assert_some_eq!(assert_ok!(var(TEST_VAR)), "test"); - - unsafe { std::env::remove_var(TEST_VAR) }; - assert_none!(assert_ok!(var(TEST_VAR))); - } - - #[test] - fn test_required_var() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "test") }; - assert_ok_eq!(required_var(TEST_VAR), "test"); - - unsafe { std::env::remove_var(TEST_VAR) }; - let error = assert_err!(required_var(TEST_VAR)); - assert_eq!( - error.to_string(), - "Failed to find required CRATES_IO_ENV_VARS_TEST_VAR environment variable" - ); - } - - #[test] - fn test_var_parsed() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "42") }; - assert_some_eq!(assert_ok!(var_parsed::(TEST_VAR)), 42); - - unsafe { std::env::set_var(TEST_VAR, "test") }; - let error = assert_err!(var_parsed::(TEST_VAR)); - assert_eq!( - error.to_string(), - "Failed to parse CRATES_IO_ENV_VARS_TEST_VAR environment variable" - ); - - unsafe { std::env::remove_var(TEST_VAR) }; - assert_none!(assert_ok!(var_parsed::(TEST_VAR))); - } - - #[test] - fn test_required_var_parsed() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "42") }; - assert_ok_eq!(required_var_parsed::(TEST_VAR), 42); - - unsafe { std::env::set_var(TEST_VAR, "test") }; - let error = assert_err!(required_var_parsed::(TEST_VAR)); - assert_eq!( - error.to_string(), - "Failed to parse CRATES_IO_ENV_VARS_TEST_VAR environment variable" - ); - - unsafe { std::env::remove_var(TEST_VAR) }; - let error = assert_err!(required_var_parsed::(TEST_VAR)); - assert_eq!( - error.to_string(), - "Failed to find required CRATES_IO_ENV_VARS_TEST_VAR environment variable" - ); - } - - #[test] - fn test_list() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "test") }; - assert_ok_eq!(list(TEST_VAR), vec!["test"]); - - unsafe { std::env::set_var(TEST_VAR, "test, foo, bar ") }; - assert_ok_eq!(list(TEST_VAR), vec!["test", "foo", "bar"]); - - unsafe { std::env::set_var(TEST_VAR, "") }; - assert_ok_eq!(list(TEST_VAR), Vec::::new()); - - unsafe { std::env::remove_var(TEST_VAR) }; - assert_ok_eq!(list(TEST_VAR), Vec::::new()); - } - - #[test] - fn test_list_parsed() { - let _guard = MUTEX.lock().unwrap(); - - unsafe { std::env::set_var(TEST_VAR, "42") }; - assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), vec![42]); - - unsafe { std::env::set_var(TEST_VAR, "42, 1, -53 ") }; - assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), vec![42, 1, -53]); - - unsafe { std::env::set_var(TEST_VAR, "42, what") }; - let error = assert_err!(list_parsed(TEST_VAR, i32::from_str)); - assert_eq!( - error.to_string(), - "Failed to parse value \"what\" of CRATES_IO_ENV_VARS_TEST_VAR environment variable" - ); - - unsafe { std::env::set_var(TEST_VAR, "") }; - assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), Vec::::new()); - - unsafe { std::env::remove_var(TEST_VAR) }; - assert_ok_eq!(list_parsed(TEST_VAR, i32::from_str), Vec::::new()); - } -} From 9a05e38494d7b0caf48bd1d216577c25fdacd564 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 8 Oct 2025 21:13:52 +0200 Subject: [PATCH 10/11] replace some arcs with ref --- src/bin/cratesfyi.rs | 25 ++++++------------------- src/cdn.rs | 2 +- src/storage/mod.rs | 2 +- src/utils/daemon.rs | 13 +++++-------- src/utils/queue_builder.rs | 17 +++++++---------- 5 files changed, 20 insertions(+), 39 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 37af7d487..8a8b82363 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -205,13 +205,9 @@ impl CommandLine { start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; - ctx.runtime.block_on(async { - docs_rs::utils::watch_registry( - ctx.async_build_queue.clone(), - ctx.config.clone(), - ctx.index.clone(), - ) - .await + ctx.runtime.block_on(async move { + docs_rs::utils::watch_registry(&ctx.async_build_queue, &ctx.config, ctx.index) + .await })?; } Self::StartBuildServer { @@ -219,10 +215,7 @@ impl CommandLine { } => { start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; - let build_queue = ctx.build_queue.clone(); - let config = ctx.config.clone(); - let rustwide_builder = RustwideBuilder::init(&ctx)?; - queue_builder(&ctx, rustwide_builder, build_queue, config)?; + queue_builder(&ctx, RustwideBuilder::init(&ctx)?)?; } Self::StartWebServer { socket_addr } => { // Blocks indefinitely @@ -654,14 +647,8 @@ impl DatabaseSubcommand { .runtime .block_on(async move { let mut conn = ctx.pool.get_async().await?; - db::delete_version( - &mut conn, - &ctx.async_storage, - &ctx.config, - &name, - &version, - ) - .await + db::delete_version(&mut conn, &ctx.async_storage, &ctx.config, &name, &version) + .await }) .context("failed to delete the version")?, Self::Delete { diff --git a/src/cdn.rs b/src/cdn.rs index 326c81225..10edd0765 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -51,7 +51,7 @@ pub enum CdnBackend { } impl CdnBackend { - pub async fn new(config: &Arc) -> CdnBackend { + pub async fn new(config: &Config) -> CdnBackend { match config.cdn_backend { CdnKind::CloudFront => { let shared_config = aws_config::load_defaults(BehaviorVersion::latest()).await; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 1b4dcdd92..aa3ac5460 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -187,7 +187,6 @@ impl AsyncStorage { config: Arc, ) -> Result { Ok(Self { - config: config.clone(), backend: match config.storage_backend { StorageKind::Database => { StorageBackend::Database(DatabaseBackend::new(pool, metrics)) @@ -196,6 +195,7 @@ impl AsyncStorage { StorageBackend::S3(Box::new(S3Backend::new(metrics, &config).await?)) } }, + config, }) } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index f5308b9aa..76ef0938d 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -19,8 +19,8 @@ use tracing::{debug, info}; /// NOTE: this should only be run once, otherwise crates would be added /// to the queue multiple times. pub async fn watch_registry( - build_queue: Arc, - config: Arc, + build_queue: &AsyncBuildQueue, + config: &Config, index: Arc, ) -> Result<(), Error> { let mut last_gc = Instant::now(); @@ -56,13 +56,12 @@ fn start_registry_watcher(context: &Context) -> Result<(), Error> { let build_queue = context.async_build_queue.clone(); let config = context.config.clone(); let index = context.index.clone(); - let runtime = context.runtime.clone(); - runtime.spawn(async { + context.runtime.spawn(async move { // space this out to prevent it from clashing against the queue-builder thread on launch tokio::time::sleep(Duration::from_secs(30)).await; - watch_registry(build_queue, config, index).await + watch_registry(&build_queue, &config, index).await }); Ok(()) @@ -195,14 +194,12 @@ pub fn start_daemon(context: Context, enable_registry_watcher: bool) -> Result<( } // build new crates every minute - let build_queue = context.build_queue.clone(); - let config = context.config.clone(); let rustwide_builder = RustwideBuilder::init(&context)?; thread::Builder::new() .name("build queue reader".to_string()) .spawn({ let context = context.clone(); - move || queue_builder(&context, rustwide_builder, build_queue, config).unwrap() + move || queue_builder(&context, rustwide_builder).unwrap() }) .unwrap(); diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 25e371dea..8a1d90502 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -1,26 +1,23 @@ use crate::Context; -use crate::{BuildQueue, Config, docbuilder::RustwideBuilder, utils::report_error}; +use crate::{docbuilder::RustwideBuilder, utils::report_error}; use anyhow::{Context as _, Error}; use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::sync::Arc; use std::time::Duration; use std::{fs, io, path::Path, thread}; use tracing::{debug, error, warn}; -pub fn queue_builder( - context: &Context, - mut builder: RustwideBuilder, - build_queue: Arc, - config: Arc, -) -> Result<(), Error> { +pub fn queue_builder(context: &Context, mut builder: RustwideBuilder) -> Result<(), Error> { loop { - if let Err(e) = remove_tempdirs(&config.temp_dir) { + let temp_dir = &context.config.temp_dir; + if let Err(e) = remove_tempdirs(temp_dir) { report_error(&anyhow::anyhow!(e).context(format!( "failed to clean temporary directory {:?}", - &config.temp_dir + temp_dir ))); } + let build_queue = &context.build_queue; + // check lock file match build_queue.is_locked().context("could not get queue lock") { Ok(true) => { From 5c00773474a0af557f237c2f6c36c18f9a1fe1be Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 8 Oct 2025 22:09:37 +0200 Subject: [PATCH 11/11] WIP --- Cargo.lock | 69 +-- Cargo.toml | 1 - src/build_queue.rs | 647 +++++++++++----------- src/cdn.rs | 837 +++++++++++++++-------------- src/docbuilder/limits.rs | 21 +- src/docbuilder/rustwide_builder.rs | 186 +++---- src/test/mod.rs | 43 +- 7 files changed, 887 insertions(+), 917 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb2c821a0..7d5cfa7a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1130,31 +1130,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bon" -version = "3.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537c317ddf588aab15c695bf92cf55dec159b93221c074180ca3e0e5a94da415" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5abbf2d4a4c6896197c9de13d6d7cb7eff438c63dacde1dde980569cb00248" -dependencies = [ - "darling 0.21.3", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.106", -] - [[package]] name = "borsh" version = "1.5.7" @@ -1795,18 +1770,8 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core", + "darling_macro", ] [[package]] @@ -1823,38 +1788,13 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.106", -] - [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core 0.21.3", + "darling_core", "quote", "syn 2.0.106", ] @@ -2020,7 +1960,6 @@ dependencies = [ "axum-extra", "backtrace", "base64 0.22.1", - "bon", "bzip2", "chrono", "clap", @@ -6726,7 +6665,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ - "darling 0.20.11", + "darling", "proc-macro2", "quote", "syn 2.0.106", diff --git a/Cargo.toml b/Cargo.toml index c522d0e46..4d431018f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,6 @@ sqlx = { version = "0.8", features = [ "runtime-tokio", "postgres", "sqlite", "c url = { version = "2.1.1", features = ["serde"] } docsrs-metadata = { path = "crates/metadata" } anyhow = { version = "1.0.42", features = ["backtrace"]} -bon = "3.7.1" backtrace = "0.3.61" thiserror = "2.0.3" comrak = { version = "0.41.0", default-features = false } diff --git a/src/build_queue.rs b/src/build_queue.rs index eab51e78e..84b5f6f64 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -730,7 +730,7 @@ pub async fn queue_rebuilds( #[cfg(test)] mod tests { - use crate::test::FakeBuild; + use crate::test::{FakeBuild, async_wrapper_with_config}; use super::*; use chrono::Utc; @@ -738,116 +738,119 @@ mod tests { #[test] fn test_rebuild_when_old() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.max_queued_rebuilds = Some(100); - }); - - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .builds(vec![ - FakeBuild::default() - .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), - ]) - .create() - .await?; - - let build_queue = env.async_build_queue(); - assert!(build_queue.queued_crates().await?.is_empty()); + }, + |env| async move { + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .builds(vec![ + FakeBuild::default() + .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), + ]) + .create() + .await?; + + let build_queue = env.async_build_queue(); + assert!(build_queue.queued_crates().await?.is_empty()); - let mut conn = env.async_db().async_conn().await; - queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; + let mut conn = env.async_db().async_conn().await; + queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; - let queue = build_queue.queued_crates().await?; - assert_eq!(queue.len(), 1); - assert_eq!(queue[0].name, "foo"); - assert_eq!(queue[0].version, "0.1.0"); - assert_eq!(queue[0].priority, REBUILD_PRIORITY); + let queue = build_queue.queued_crates().await?; + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].name, "foo"); + assert_eq!(queue[0].version, "0.1.0"); + assert_eq!(queue[0].priority, REBUILD_PRIORITY); - Ok(()) - }) + Ok(()) + }, + ) } #[test] fn test_still_rebuild_when_full_with_failed() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.max_queued_rebuilds = Some(1); - }); - - let build_queue = env.async_build_queue(); - build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) - .await?; - build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) - .await?; + }, + |env| async move { + let build_queue = env.async_build_queue(); + build_queue + .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .await?; + build_queue + .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .await?; - let mut conn = env.async_db().async_conn().await; - sqlx::query!("UPDATE queue SET attempt = 99") - .execute(&mut *conn) - .await?; + let mut conn = env.async_db().async_conn().await; + sqlx::query!("UPDATE queue SET attempt = 99") + .execute(&mut *conn) + .await?; - assert_eq!(build_queue.queued_crates().await?.len(), 0); + assert_eq!(build_queue.queued_crates().await?.len(), 0); - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .builds(vec![ - FakeBuild::default() - .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), - ]) - .create() - .await?; + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .builds(vec![ + FakeBuild::default() + .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), + ]) + .create() + .await?; - let build_queue = env.async_build_queue(); - queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; + let build_queue = env.async_build_queue(); + queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; - assert_eq!(build_queue.queued_crates().await?.len(), 1); + assert_eq!(build_queue.queued_crates().await?.len(), 1); - Ok(()) - }) + Ok(()) + }, + ) } #[test] fn test_dont_rebuild_when_full() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.max_queued_rebuilds = Some(1); - }); - - let build_queue = env.async_build_queue(); - build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) - .await?; - build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) - .await?; - - env.fake_release() - .await - .name("foo") - .version("0.1.0") - .builds(vec![ - FakeBuild::default() - .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), - ]) - .create() - .await?; - - let build_queue = env.async_build_queue(); - assert_eq!(build_queue.queued_crates().await?.len(), 2); + }, + |env| async move { + let build_queue = env.async_build_queue(); + build_queue + .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .await?; + build_queue + .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .await?; + + env.fake_release() + .await + .name("foo") + .version("0.1.0") + .builds(vec![ + FakeBuild::default() + .rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), + ]) + .create() + .await?; + + let build_queue = env.async_build_queue(); + assert_eq!(build_queue.queued_crates().await?.len(), 2); - let mut conn = env.async_db().async_conn().await; - queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; + let mut conn = env.async_db().async_conn().await; + queue_rebuilds(&mut conn, &env.config(), &build_queue).await?; - assert_eq!(build_queue.queued_crates().await?.len(), 2); + assert_eq!(build_queue.queued_crates().await?.len(), 2); - Ok(()) - }) + Ok(()) + }, + ) } #[test] @@ -868,43 +871,44 @@ mod tests { #[test] fn test_add_duplicate_resets_attempts_and_priority() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_attempts = 5; - }); - - let queue = env.async_build_queue(); + }, + |env| async move { + let queue = env.async_build_queue(); - let mut conn = env.async_db().async_conn().await; - sqlx::query!( - " + let mut conn = env.async_db().async_conn().await; + sqlx::query!( + " INSERT INTO queue (name, version, priority, attempt, last_attempt ) VALUES ('failed_crate', '0.1.1', 0, 99, NOW())", - ) - .execute(&mut *conn) - .await?; + ) + .execute(&mut *conn) + .await?; - assert_eq!(queue.pending_count().await?, 0); + assert_eq!(queue.pending_count().await?, 0); - queue.add_crate("failed_crate", "0.1.1", 9, None).await?; + queue.add_crate("failed_crate", "0.1.1", 9, None).await?; - assert_eq!(queue.pending_count().await?, 1); + assert_eq!(queue.pending_count().await?, 1); - let row = sqlx::query!( - "SELECT priority, attempt, last_attempt + let row = sqlx::query!( + "SELECT priority, attempt, last_attempt FROM queue WHERE name = $1 AND version = $2", - "failed_crate", - "0.1.1", - ) - .fetch_one(&mut *conn) - .await?; + "failed_crate", + "0.1.1", + ) + .fetch_one(&mut *conn) + .await?; - assert_eq!(row.priority, 9); - assert_eq!(row.attempt, 0); - assert!(row.last_attempt.is_none()); - Ok(()) - }) + assert_eq!(row.priority, 9); + assert_eq!(row.attempt, 0); + assert!(row.last_attempt.is_none()); + Ok(()) + }, + ) } #[test] @@ -930,200 +934,203 @@ mod tests { #[test] fn test_wait_between_build_attempts() { - crate::test::wrapper(|env| { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_attempts = 99; config.delay_between_build_attempts = Duration::from_secs(1); - }); - - let runtime = env.runtime(); + }, + |env| async move { + let runtime = env.runtime(); - let queue = env.build_queue(); + let queue = env.build_queue(); - queue.add_crate("krate", "1.0.0", 0, None)?; + queue.add_crate("krate", "1.0.0", 0, None)?; - // first let it fail - queue.process_next_crate(|krate| { - assert_eq!(krate.name, "krate"); - anyhow::bail!("simulate a failure"); - })?; + // first let it fail + queue.process_next_crate(|krate| { + assert_eq!(krate.name, "krate"); + anyhow::bail!("simulate a failure"); + })?; - queue.process_next_crate(|_| { - // this can't happen since we didn't wait between attempts - unreachable!(); - })?; + queue.process_next_crate(|_| { + // this can't happen since we didn't wait between attempts + unreachable!(); + })?; - runtime.block_on(async { - // fake the build-attempt timestamp so it's older - let mut conn = env.async_db().async_conn().await; - sqlx::query!( - "UPDATE queue SET last_attempt = $1", - Utc::now() - chrono::Duration::try_seconds(60).unwrap() - ) - .execute(&mut *conn) - .await - })?; + runtime.block_on(async { + // fake the build-attempt timestamp so it's older + let mut conn = env.async_db().async_conn().await; + sqlx::query!( + "UPDATE queue SET last_attempt = $1", + Utc::now() - chrono::Duration::try_seconds(60).unwrap() + ) + .execute(&mut *conn) + .await + })?; - let mut handled = false; - // now we can process it again - queue.process_next_crate(|krate| { - assert_eq!(krate.name, "krate"); - handled = true; - Ok(BuildPackageSummary::default()) - })?; + let mut handled = false; + // now we can process it again + queue.process_next_crate(|krate| { + assert_eq!(krate.name, "krate"); + handled = true; + Ok(BuildPackageSummary::default()) + })?; - assert!(handled); + assert!(handled); - Ok(()) - }) + Ok(()) + }, + ) } #[test] fn test_add_and_process_crates() { const MAX_ATTEMPTS: u16 = 3; - crate::test::wrapper(|env| { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_attempts = MAX_ATTEMPTS; config.delay_between_build_attempts = Duration::ZERO; - }); - - let queue = env.build_queue(); + }, + |env| async move { + let queue = env.build_queue(); + + let test_crates = [ + ("low-priority", "1.0.0", 1000), + ("high-priority-foo", "1.0.0", -1000), + ("medium-priority", "1.0.0", -10), + ("high-priority-bar", "1.0.0", -1000), + ("standard-priority", "1.0.0", 0), + ("high-priority-baz", "1.0.0", -1000), + ]; + for krate in &test_crates { + queue.add_crate(krate.0, krate.1, krate.2, None)?; + } - let test_crates = [ - ("low-priority", "1.0.0", 1000), - ("high-priority-foo", "1.0.0", -1000), - ("medium-priority", "1.0.0", -10), - ("high-priority-bar", "1.0.0", -1000), - ("standard-priority", "1.0.0", 0), - ("high-priority-baz", "1.0.0", -1000), - ]; - for krate in &test_crates { - queue.add_crate(krate.0, krate.1, krate.2, None)?; - } + let assert_next = |name| -> Result<()> { + queue.process_next_crate(|krate| { + assert_eq!(name, krate.name); + Ok(BuildPackageSummary::default()) + })?; + Ok(()) + }; + let assert_next_and_fail = |name| -> Result<()> { + queue.process_next_crate(|krate| { + assert_eq!(name, krate.name); + anyhow::bail!("simulate a failure"); + })?; + Ok(()) + }; + + // The first processed item is the one with the highest priority added first. + assert_next("high-priority-foo")?; + + // Simulate a failure in high-priority-bar. + assert_next_and_fail("high-priority-bar")?; + + // Continue with the next high priority crate. + assert_next("high-priority-baz")?; + + // After all the crates with the max priority are processed, before starting to process + // crates with a lower priority the failed crates with the max priority will be tried + // again. + assert_next("high-priority-bar")?; + + // Continue processing according to the priority. + assert_next("medium-priority")?; + assert_next("standard-priority")?; + + // Simulate the crate failing many times. + for _ in 0..MAX_ATTEMPTS { + assert_next_and_fail("low-priority")?; + } - let assert_next = |name| -> Result<()> { - queue.process_next_crate(|krate| { - assert_eq!(name, krate.name); + // Since low-priority failed many times it will be removed from the queue. Because of + // that the queue should now be empty. + let mut called = false; + queue.process_next_crate(|_| { + called = true; Ok(BuildPackageSummary::default()) })?; - Ok(()) - }; - let assert_next_and_fail = |name| -> Result<()> { - queue.process_next_crate(|krate| { - assert_eq!(name, krate.name); - anyhow::bail!("simulate a failure"); - })?; - Ok(()) - }; - - // The first processed item is the one with the highest priority added first. - assert_next("high-priority-foo")?; - - // Simulate a failure in high-priority-bar. - assert_next_and_fail("high-priority-bar")?; - - // Continue with the next high priority crate. - assert_next("high-priority-baz")?; - - // After all the crates with the max priority are processed, before starting to process - // crates with a lower priority the failed crates with the max priority will be tried - // again. - assert_next("high-priority-bar")?; - - // Continue processing according to the priority. - assert_next("medium-priority")?; - assert_next("standard-priority")?; + assert!(!called, "there were still items in the queue"); + + // Ensure metrics were recorded correctly + let metrics = env.instance_metrics(); + assert_eq!(metrics.total_builds.get(), 9); + assert_eq!(metrics.failed_builds.get(), 1); + assert_eq!(metrics.build_time.get_sample_count(), 9); + + // no invalidations were run since we don't have a distribution id configured + assert!( + env.runtime() + .block_on(async { + cdn::queued_or_active_crate_invalidations( + &mut *env.async_db().async_conn().await, + ) + .await + })? + .is_empty() + ); - // Simulate the crate failing many times. - for _ in 0..MAX_ATTEMPTS { - assert_next_and_fail("low-priority")?; - } - - // Since low-priority failed many times it will be removed from the queue. Because of - // that the queue should now be empty. - let mut called = false; - queue.process_next_crate(|_| { - called = true; - Ok(BuildPackageSummary::default()) - })?; - assert!(!called, "there were still items in the queue"); - - // Ensure metrics were recorded correctly - let metrics = env.instance_metrics(); - assert_eq!(metrics.total_builds.get(), 9); - assert_eq!(metrics.failed_builds.get(), 1); - assert_eq!(metrics.build_time.get_sample_count(), 9); - - // no invalidations were run since we don't have a distribution id configured - assert!( - env.runtime() - .block_on(async { - cdn::queued_or_active_crate_invalidations( - &mut *env.async_db().async_conn().await, - ) - .await - })? - .is_empty() - ); - - Ok(()) - }) + Ok(()) + }, + ) } #[test] fn test_invalidate_cdn_after_build_and_error() { - crate::test::wrapper(|env| { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); config.cloudfront_distribution_id_static = Some("distribution_id_static".into()); - }); - - let queue = env.build_queue(); + }, + |env| async move { + let queue = env.build_queue(); - queue.add_crate("will_succeed", "1.0.0", -1, None)?; - queue.add_crate("will_fail", "1.0.0", 0, None)?; + queue.add_crate("will_succeed", "1.0.0", -1, None)?; + queue.add_crate("will_fail", "1.0.0", 0, None)?; - let fetch_invalidations = || { - env.runtime() - .block_on(async { - let mut conn = env.async_db().async_conn().await; - cdn::queued_or_active_crate_invalidations(&mut conn).await - }) - .unwrap() - }; + let fetch_invalidations = || { + env.runtime() + .block_on(async { + let mut conn = env.async_db().async_conn().await; + cdn::queued_or_active_crate_invalidations(&mut conn).await + }) + .unwrap() + }; - assert!(fetch_invalidations().is_empty()); + assert!(fetch_invalidations().is_empty()); - queue.process_next_crate(|krate| { - assert_eq!("will_succeed", krate.name); - Ok(BuildPackageSummary::default()) - })?; + queue.process_next_crate(|krate| { + assert_eq!("will_succeed", krate.name); + Ok(BuildPackageSummary::default()) + })?; - let queued_invalidations = fetch_invalidations(); - assert_eq!(queued_invalidations.len(), 3); - assert!( - queued_invalidations - .iter() - .all(|i| i.krate == "will_succeed") - ); + let queued_invalidations = fetch_invalidations(); + assert_eq!(queued_invalidations.len(), 3); + assert!( + queued_invalidations + .iter() + .all(|i| i.krate == "will_succeed") + ); - queue.process_next_crate(|krate| { - assert_eq!("will_fail", krate.name); - anyhow::bail!("simulate a failure"); - })?; + queue.process_next_crate(|krate| { + assert_eq!("will_fail", krate.name); + anyhow::bail!("simulate a failure"); + })?; - let queued_invalidations = fetch_invalidations(); - assert_eq!(queued_invalidations.len(), 6); - assert!( - queued_invalidations - .iter() - .skip(3) - .all(|i| i.krate == "will_fail") - ); + let queued_invalidations = fetch_invalidations(); + assert_eq!(queued_invalidations.len(), 6); + assert!( + queued_invalidations + .iter() + .skip(3) + .all(|i| i.krate == "will_fail") + ); - Ok(()) - }) + Ok(()) + }, + ) } #[test] @@ -1198,72 +1205,76 @@ mod tests { #[test] fn test_failed_count_for_reattempts() { const MAX_ATTEMPTS: u16 = 3; - crate::test::wrapper(|env| { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_attempts = MAX_ATTEMPTS; config.delay_between_build_attempts = Duration::ZERO; - }); - let queue = env.build_queue(); - - assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; - assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + }, + |env| async move { + let queue = env.build_queue(); - for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); + queue.add_crate("foo", "1.0.0", -100, None)?; + assert_eq!(queue.failed_count()?, 0); + queue.add_crate("bar", "1.0.0", 0, None)?; + + for _ in 0..MAX_ATTEMPTS { + assert_eq!(queue.failed_count()?, 0); + queue.process_next_crate(|krate| { + assert_eq!("foo", krate.name); + Ok(BuildPackageSummary { + should_reattempt: true, + ..Default::default() + }) + })?; + } + assert_eq!(queue.failed_count()?, 1); + queue.process_next_crate(|krate| { - assert_eq!("foo", krate.name); - Ok(BuildPackageSummary { - should_reattempt: true, - ..Default::default() - }) + assert_eq!("bar", krate.name); + Ok(BuildPackageSummary::default()) })?; - } - assert_eq!(queue.failed_count()?, 1); + assert_eq!(queue.failed_count()?, 1); - queue.process_next_crate(|krate| { - assert_eq!("bar", krate.name); - Ok(BuildPackageSummary::default()) - })?; - assert_eq!(queue.failed_count()?, 1); - - Ok(()) - }); + Ok(()) + }, + ); } #[test] fn test_failed_count_after_error() { const MAX_ATTEMPTS: u16 = 3; - crate::test::wrapper(|env| { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_attempts = MAX_ATTEMPTS; config.delay_between_build_attempts = Duration::ZERO; - }); - let queue = env.build_queue(); - - assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; - assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + }, + |env| async move { + let queue = env.build_queue(); - for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); + queue.add_crate("foo", "1.0.0", -100, None)?; + assert_eq!(queue.failed_count()?, 0); + queue.add_crate("bar", "1.0.0", 0, None)?; + + for _ in 0..MAX_ATTEMPTS { + assert_eq!(queue.failed_count()?, 0); + queue.process_next_crate(|krate| { + assert_eq!("foo", krate.name); + anyhow::bail!("this failed"); + })?; + } + assert_eq!(queue.failed_count()?, 1); + queue.process_next_crate(|krate| { - assert_eq!("foo", krate.name); - anyhow::bail!("this failed"); + assert_eq!("bar", krate.name); + Ok(BuildPackageSummary::default()) })?; - } - assert_eq!(queue.failed_count()?, 1); - - queue.process_next_crate(|krate| { - assert_eq!("bar", krate.name); - Ok(BuildPackageSummary::default()) - })?; - assert_eq!(queue.failed_count()?, 1); + assert_eq!(queue.failed_count()?, 1); - Ok(()) - }); + Ok(()) + }, + ); } #[test] diff --git a/src/cdn.rs b/src/cdn.rs index 10edd0765..cb3d46417 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -642,7 +642,7 @@ mod tests { use std::time::Duration; use super::*; - use crate::test::async_wrapper; + use crate::test::{async_wrapper, async_wrapper_with_config}; use aws_sdk_cloudfront::{Config, config::Credentials}; use aws_smithy_runtime::client::http::test_util::{ReplayEvent, StaticReplayClient}; @@ -694,19 +694,18 @@ mod tests { #[test] fn create_cloudfront() { - async_wrapper(|env| async move { - env.override_config(|config| { - config.cdn_backend = CdnKind::CloudFront; - }); - - assert!(matches!(*env.cdn(), CdnBackend::CloudFront { .. })); - assert!(matches!( - CdnBackend::new(&env.config()).await, - CdnBackend::CloudFront { .. } - )); - - Ok(()) - }) + async_wrapper_with_config( + |config| config.cdn_backend = CdnKind::CloudFront, + |env| async move { + assert!(matches!(*env.cdn(), CdnBackend::CloudFront { .. })); + assert!(matches!( + CdnBackend::new(&env.config()).await, + CdnBackend::CloudFront { .. } + )); + + Ok(()) + }, + ) } #[test] @@ -724,481 +723,489 @@ mod tests { #[test] fn invalidation_counts_are_zero_with_empty_queue() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); config.cloudfront_distribution_id_static = Some("distribution_id_static".into()); - }); - - let config = env.config(); - let mut conn = env.async_db().async_conn().await; - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + }, + |env| async move { + let config = env.config(); + let mut conn = env.async_db().async_conn().await; + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); - let counts = - queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) - .await?; - assert_eq!(counts.len(), 2); - assert_eq!(*counts.get("distribution_id_web").unwrap(), 0); - assert_eq!(*counts.get("distribution_id_static").unwrap(), 0); - Ok(()) - }) + let counts = + queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) + .await?; + assert_eq!(counts.len(), 2); + assert_eq!(*counts.get("distribution_id_web").unwrap(), 0); + assert_eq!(*counts.get("distribution_id_static").unwrap(), 0); + Ok(()) + }, + ) } #[test] fn escalate_to_full_invalidation() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); config.cloudfront_distribution_id_static = Some("distribution_id_static".into()); config.cdn_max_queued_age = Duration::from_secs(0); - }); - - let cdn = env.cdn(); - let config = env.config(); - let mut conn = env.async_db().async_conn().await; - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + }, + |env| async move { + let cdn = env.cdn(); + let config = env.config(); + let mut conn = env.async_db().async_conn().await; + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); - queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; - - // invalidation paths are queued. - assert_eq!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .into_iter() - .map(|i| ( - i.cdn_distribution_id, - i.krate, - i.path_pattern, - i.cdn_reference - )) - .collect::>(), - vec![ - ( - "distribution_id_web".into(), - "krate".into(), - "/krate*".into(), - None - ), - ( - "distribution_id_web".into(), - "krate".into(), - "/crate/krate*".into(), - None - ), - ( - "distribution_id_static".into(), - "krate".into(), - "/rustdoc/krate*".into(), - None - ), - ] - ); + queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; + + // invalidation paths are queued. + assert_eq!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .into_iter() + .map(|i| ( + i.cdn_distribution_id, + i.krate, + i.path_pattern, + i.cdn_reference + )) + .collect::>(), + vec![ + ( + "distribution_id_web".into(), + "krate".into(), + "/krate*".into(), + None + ), + ( + "distribution_id_web".into(), + "krate".into(), + "/crate/krate*".into(), + None + ), + ( + "distribution_id_static".into(), + "krate".into(), + "/rustdoc/krate*".into(), + None + ), + ] + ); - let counts = - queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) - .await?; - assert_eq!(counts.len(), 2); - assert_eq!(*counts.get("distribution_id_web").unwrap(), 2); - assert_eq!(*counts.get("distribution_id_static").unwrap(), 1); - - // queueing the invalidation doesn't create it in the CDN - assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); - assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); - - let cdn = env.cdn(); - let config = env.config(); - - // now handle the queued invalidations - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_static", - ) - .await?; + let counts = + queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) + .await?; + assert_eq!(counts.len(), 2); + assert_eq!(*counts.get("distribution_id_web").unwrap(), 2); + assert_eq!(*counts.get("distribution_id_static").unwrap(), 1); + + // queueing the invalidation doesn't create it in the CDN + assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); + assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); + + let cdn = env.cdn(); + let config = env.config(); + + // now handle the queued invalidations + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_static", + ) + .await?; - // which creates them in the CDN - { - let ir_web = active_invalidations(&cdn, "distribution_id_web"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_web[0].path_patterns, vec!["/*"]); + // which creates them in the CDN + { + let ir_web = active_invalidations(&cdn, "distribution_id_web"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_web[0].path_patterns, vec!["/*"]); - let ir_static = active_invalidations(&cdn, "distribution_id_static"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_static[0].path_patterns, vec!["/*"]); - } + let ir_static = active_invalidations(&cdn, "distribution_id_static"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_static[0].path_patterns, vec!["/*"]); + } - // the queued entries got a CDN reference attached - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .iter() - .all(|i| i.cdn_reference.is_some() && i.created_in_cdn.is_some()) - ); + // the queued entries got a CDN reference attached + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .iter() + .all(|i| i.cdn_reference.is_some() && i.created_in_cdn.is_some()) + ); - Ok(()) - }); + Ok(()) + }, + ); } #[test] fn invalidate_a_crate() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); config.cloudfront_distribution_id_static = Some("distribution_id_static".into()); - }); - - let cdn = env.cdn(); - let config = env.config(); - let mut conn = env.async_db().async_conn().await; - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + }, + |env| async move { + let cdn = env.cdn(); + let config = env.config(); + let mut conn = env.async_db().async_conn().await; + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); - queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; - - // invalidation paths are queued. - assert_eq!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .into_iter() - .map(|i| ( - i.cdn_distribution_id, - i.krate, - i.path_pattern, - i.cdn_reference - )) - .collect::>(), - vec![ - ( - "distribution_id_web".into(), - "krate".into(), - "/krate*".into(), - None - ), - ( - "distribution_id_web".into(), - "krate".into(), - "/crate/krate*".into(), - None - ), - ( - "distribution_id_static".into(), - "krate".into(), - "/rustdoc/krate*".into(), - None - ), - ] - ); + queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; + + // invalidation paths are queued. + assert_eq!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .into_iter() + .map(|i| ( + i.cdn_distribution_id, + i.krate, + i.path_pattern, + i.cdn_reference + )) + .collect::>(), + vec![ + ( + "distribution_id_web".into(), + "krate".into(), + "/krate*".into(), + None + ), + ( + "distribution_id_web".into(), + "krate".into(), + "/crate/krate*".into(), + None + ), + ( + "distribution_id_static".into(), + "krate".into(), + "/rustdoc/krate*".into(), + None + ), + ] + ); - let counts = - queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) - .await?; - assert_eq!(counts.len(), 2); - assert_eq!(*counts.get("distribution_id_web").unwrap(), 2); - assert_eq!(*counts.get("distribution_id_static").unwrap(), 1); - - // queueing the invalidation doesn't create it in the CDN - assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); - assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); - - let cdn = env.cdn(); - let config = env.config(); - - // now handle the queued invalidations - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_static", - ) - .await?; + let counts = + queued_or_active_crate_invalidation_count_by_distribution(&mut conn, &config) + .await?; + assert_eq!(counts.len(), 2); + assert_eq!(*counts.get("distribution_id_web").unwrap(), 2); + assert_eq!(*counts.get("distribution_id_static").unwrap(), 1); + + // queueing the invalidation doesn't create it in the CDN + assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); + assert!(active_invalidations(&cdn, "distribution_id_static").is_empty()); + + let cdn = env.cdn(); + let config = env.config(); + + // now handle the queued invalidations + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_static", + ) + .await?; - // which creates them in the CDN - { - let ir_web = active_invalidations(&cdn, "distribution_id_web"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_web[0].path_patterns, vec!["/krate*", "/crate/krate*"]); + // which creates them in the CDN + { + let ir_web = active_invalidations(&cdn, "distribution_id_web"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_web[0].path_patterns, vec!["/krate*", "/crate/krate*"]); - let ir_static = active_invalidations(&cdn, "distribution_id_static"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_static[0].path_patterns, vec!["/rustdoc/krate*"]); - } + let ir_static = active_invalidations(&cdn, "distribution_id_static"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_static[0].path_patterns, vec!["/rustdoc/krate*"]); + } - // the queued entries got a CDN reference attached - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .iter() - .all(|i| i.cdn_reference.is_some() && i.created_in_cdn.is_some()) - ); + // the queued entries got a CDN reference attached + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .iter() + .all(|i| i.cdn_reference.is_some() && i.created_in_cdn.is_some()) + ); - // clear the active invalidations in the CDN to _fake_ them - // being completed on the CDN side. - cdn.clear_active_invalidations(); - - // now handle again - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; - handle_queued_invalidation_requests( - &config, - &cdn, - &env.instance_metrics(), - &mut conn, - "distribution_id_static", - ) - .await?; + // clear the active invalidations in the CDN to _fake_ them + // being completed on the CDN side. + cdn.clear_active_invalidations(); - // which removes them from the queue table - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + // now handle again + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; + handle_queued_invalidation_requests( + &config, + &cdn, + &env.instance_metrics(), + &mut conn, + "distribution_id_static", + ) + .await?; - Ok(()) - }); + // which removes them from the queue table + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); + + Ok(()) + }, + ); } #[test] fn only_add_some_invalidations_when_too_many_are_active() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); - }); + }, + |env| async move { + let cdn = env.cdn(); + + // create an invalidation with 15 paths, so we're over the limit + let already_running_invalidation = cdn + .create_invalidation( + "distribution_id_web", + &(0..(MAX_CLOUDFRONT_WILDCARD_INVALIDATIONS - 1)) + .map(|_| "/something*") + .collect::>(), + ) + .await?; - let cdn = env.cdn(); + let mut conn = env.async_db().async_conn().await; + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); + + // insert some completed invalidations into the queue & the CDN, these will be ignored + for i in 0..10 { + insert_running_invalidation( + &mut conn, + "distribution_id_web", + &format!("some_id_{i}"), + ) + .await?; + cdn.insert_completed_invalidation( + "distribution_id_web", + &format!("some_id_{i}"), + &["/*"], + ); + } - // create an invalidation with 15 paths, so we're over the limit - let already_running_invalidation = cdn - .create_invalidation( + // insert the CDN representation of the already running invalidation + insert_running_invalidation( + &mut conn, "distribution_id_web", - &(0..(MAX_CLOUDFRONT_WILDCARD_INVALIDATIONS - 1)) - .map(|_| "/something*") - .collect::>(), + &already_running_invalidation.invalidation_id, ) .await?; - let mut conn = env.async_db().async_conn().await; - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + // queue an invalidation + queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; - // insert some completed invalidations into the queue & the CDN, these will be ignored - for i in 0..10 { - insert_running_invalidation( + // handle the queued invalidations + handle_queued_invalidation_requests( + &env.config(), + &env.cdn(), + &env.instance_metrics(), &mut conn, "distribution_id_web", - &format!("some_id_{i}"), ) .await?; - cdn.insert_completed_invalidation( - "distribution_id_web", - &format!("some_id_{i}"), - &["/*"], - ); - } - - // insert the CDN representation of the already running invalidation - insert_running_invalidation( - &mut conn, - "distribution_id_web", - &already_running_invalidation.invalidation_id, - ) - .await?; - - // queue an invalidation - queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; - - // handle the queued invalidations - handle_queued_invalidation_requests( - &env.config(), - &env.cdn(), - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; - // only one path was added to the CDN - let q = queued_or_active_crate_invalidations(&mut conn).await?; - assert_eq!( - q.iter() - .filter_map(|i| i.cdn_reference.as_ref()) - .filter(|&reference| reference != &already_running_invalidation.invalidation_id) - .count(), - 1 - ); + // only one path was added to the CDN + let q = queued_or_active_crate_invalidations(&mut conn).await?; + assert_eq!( + q.iter() + .filter_map(|i| i.cdn_reference.as_ref()) + .filter( + |&reference| reference != &already_running_invalidation.invalidation_id + ) + .count(), + 1 + ); - // old invalidation is still active, new one is added - let ir_web = active_invalidations(&cdn, "distribution_id_web"); - assert_eq!(ir_web.len(), 2); - assert_eq!(ir_web[0].path_patterns.len(), 12); - assert_eq!(ir_web[1].path_patterns.len(), 1); + // old invalidation is still active, new one is added + let ir_web = active_invalidations(&cdn, "distribution_id_web"); + assert_eq!(ir_web.len(), 2); + assert_eq!(ir_web[0].path_patterns.len(), 12); + assert_eq!(ir_web[1].path_patterns.len(), 1); - Ok(()) - }); + Ok(()) + }, + ); } #[test] fn dont_create_invalidations_when_too_many_are_active() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); - }); - - let cdn = env.cdn(); + }, + |env| async move { + let cdn = env.cdn(); + + // create an invalidation with 15 paths, so we're over the limit + let already_running_invalidation = cdn + .create_invalidation( + "distribution_id_web", + &(0..15).map(|_| "/something*").collect::>(), + ) + .await?; - // create an invalidation with 15 paths, so we're over the limit - let already_running_invalidation = cdn - .create_invalidation( + let mut conn = env.async_db().async_conn().await; + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); + insert_running_invalidation( + &mut conn, "distribution_id_web", - &(0..15).map(|_| "/something*").collect::>(), + &already_running_invalidation.invalidation_id, ) .await?; - let mut conn = env.async_db().async_conn().await; - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); - insert_running_invalidation( - &mut conn, - "distribution_id_web", - &already_running_invalidation.invalidation_id, - ) - .await?; + // queue an invalidation + queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; - // queue an invalidation - queue_crate_invalidation(&mut conn, &env.config(), "krate").await?; + // handle the queued invalidations + handle_queued_invalidation_requests( + &env.config(), + &env.cdn(), + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; - // handle the queued invalidations - handle_queued_invalidation_requests( - &env.config(), - &env.cdn(), - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; + // nothing was added to the CDN + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .iter() + .filter(|i| !matches!( + &i.cdn_reference, + Some(val) if val == &already_running_invalidation.invalidation_id + )) + .all(|i| i.cdn_reference.is_none()) + ); - // nothing was added to the CDN - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .iter() - .filter(|i| !matches!( - &i.cdn_reference, - Some(val) if val == &already_running_invalidation.invalidation_id - )) - .all(|i| i.cdn_reference.is_none()) - ); + // old invalidations are still active + let ir_web = active_invalidations(&cdn, "distribution_id_web"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_web[0].path_patterns.len(), 15); - // old invalidations are still active - let ir_web = active_invalidations(&cdn, "distribution_id_web"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_web[0].path_patterns.len(), 15); - - // clear the active invalidations in the CDN to _fake_ them - // being completed on the CDN side. - cdn.clear_active_invalidations(); - - // now handle again - handle_queued_invalidation_requests( - &env.config(), - &env.cdn(), - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; + // clear the active invalidations in the CDN to _fake_ them + // being completed on the CDN side. + cdn.clear_active_invalidations(); - // which adds the CDN reference - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .iter() - .all(|i| i.cdn_reference.is_some()) - ); + // now handle again + handle_queued_invalidation_requests( + &env.config(), + &env.cdn(), + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; - // and creates them in the CDN too - let ir_web = active_invalidations(&cdn, "distribution_id_web"); - assert_eq!(ir_web.len(), 1); - assert_eq!(ir_web[0].path_patterns, vec!["/krate*", "/crate/krate*"]); + // which adds the CDN reference + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .iter() + .all(|i| i.cdn_reference.is_some()) + ); - Ok(()) - }); + // and creates them in the CDN too + let ir_web = active_invalidations(&cdn, "distribution_id_web"); + assert_eq!(ir_web.len(), 1); + assert_eq!(ir_web[0].path_patterns, vec!["/krate*", "/crate/krate*"]); + + Ok(()) + }, + ); } #[test] fn dont_create_invalidations_without_paths() { - crate::test::async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.cloudfront_distribution_id_web = Some("distribution_id_web".into()); - }); - - let cdn = env.cdn(); - - let mut conn = env.async_db().async_conn().await; - // no invalidation is queued - assert!( - queued_or_active_crate_invalidations(&mut conn) - .await? - .is_empty() - ); + }, + |env| async move { + let cdn = env.cdn(); + + let mut conn = env.async_db().async_conn().await; + // no invalidation is queued + assert!( + queued_or_active_crate_invalidations(&mut conn) + .await? + .is_empty() + ); - // run the handler - handle_queued_invalidation_requests( - &env.config(), - &env.cdn(), - &env.instance_metrics(), - &mut conn, - "distribution_id_web", - ) - .await?; + // run the handler + handle_queued_invalidation_requests( + &env.config(), + &env.cdn(), + &env.instance_metrics(), + &mut conn, + "distribution_id_web", + ) + .await?; - // no invalidation was created - assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); + // no invalidation was created + assert!(active_invalidations(&cdn, "distribution_id_web").is_empty()); - Ok(()) - }); + Ok(()) + }, + ); } async fn get_mock_config(http_client: StaticReplayClient) -> aws_sdk_cloudfront::Config { diff --git a/src/docbuilder/limits.rs b/src/docbuilder/limits.rs index ac8ef8af1..366038a62 100644 --- a/src/docbuilder/limits.rs +++ b/src/docbuilder/limits.rs @@ -155,19 +155,20 @@ mod test { #[test] fn config_default_memory_limit() { - async_wrapper(|env| async move { - env.override_config(|config| { + async_wrapper_with_config( + |config| { config.build_default_memory_limit = Some(6 * GB); - }); + }, + |env| async move { + let db = env.async_db(); + let mut conn = db.async_conn().await; - let db = env.async_db(); - let mut conn = db.async_conn().await; + let limits = Limits::for_crate(&env.config(), &mut conn, "krate").await?; + assert_eq!(limits.memory, 6 * GB); - let limits = Limits::for_crate(&env.config(), &mut conn, "krate").await?; - assert_eq!(limits.memory, 6 * GB); - - Ok(()) - }) + Ok(()) + }, + ) } #[test] diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 1b0edd9c1..a82b84a10 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1286,7 +1286,7 @@ mod tests { use crate::db::types::Feature; use crate::registry_api::ReleaseData; use crate::storage::{CompressionAlgorithm, compression}; - use crate::test::{AxumRouterTestExt, TestEnvironment, wrapper}; + use crate::test::{AxumRouterTestExt, TestEnvironment, wrapper, wrapper_with_config}; use std::{io, iter}; use test_case::test_case; @@ -1537,36 +1537,36 @@ mod tests { #[test] #[ignore] fn test_collect_metrics() { - wrapper(|env| { - let metrics_dir = tempfile::tempdir()?.keep(); - - env.override_config(|cfg| { + let metrics_dir = tempfile::tempdir().unwrap().keep(); + wrapper_with_config( + |cfg| { cfg.compiler_metrics_collection_path = Some(metrics_dir.clone()); cfg.include_default_targets = false; - }); - - let crate_ = DUMMY_CRATE_NAME; - let version = DUMMY_CRATE_VERSION; - - let mut builder = RustwideBuilder::init(&env.context).unwrap(); - builder.update_toolchain()?; - assert!( - builder - .build_package(crate_, version, PackageKind::CratesIo, true)? - .successful - ); + }, + |env| { + let crate_ = DUMMY_CRATE_NAME; + let version = DUMMY_CRATE_VERSION; + + let mut builder = RustwideBuilder::init(&env.context).unwrap(); + builder.update_toolchain()?; + assert!( + builder + .build_package(crate_, version, PackageKind::CratesIo, true)? + .successful + ); - let metric_files: Vec<_> = fs::read_dir(&metrics_dir)? - .filter_map(|di| di.ok()) - .map(|di| di.path()) - .collect(); + let metric_files: Vec<_> = fs::read_dir(&metrics_dir)? + .filter_map(|di| di.ok()) + .map(|di| di.path()) + .collect(); - assert_eq!(metric_files.len(), 1); + assert_eq!(metric_files.len(), 1); - let _: serde_json::Value = serde_json::from_slice(&fs::read(&metric_files[0])?)?; + let _: serde_json::Value = serde_json::from_slice(&fs::read(&metric_files[0])?)?; - Ok(()) - }) + Ok(()) + }, + ) } #[test] @@ -1794,58 +1794,64 @@ mod tests { #[test] #[ignore] fn test_locked_fails_unlocked_needs_new_deps() { - wrapper(|env| { - env.override_config(|cfg| cfg.include_default_targets = false); - - // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(env, "rand_core", "0.5.1")?; - - // Specific setup required: - // * crate has a binary so that it is published with a lockfile - // * crate has a library so that it is documented by docs.rs - // * crate has an optional dependency - // * metadata enables the optional dependency for docs.rs - // * `cargo doc` fails with the version of the dependency in the lockfile - // * there is a newer version of the dependency available that correctly builds - let crate_ = "docs_rs_test_incorrect_lockfile"; - let version = "0.1.2"; - let mut builder = RustwideBuilder::init(&env.context).unwrap(); - builder.update_toolchain()?; - assert!( - builder - .build_package(crate_, version, PackageKind::CratesIo, false)? - .successful - ); + wrapper_with_config( + |cfg| { + cfg.include_default_targets = false; + }, + |env| { + // if the corrected dependency of the crate was already downloaded we need to remove it + remove_cache_files(env, "rand_core", "0.5.1")?; + + // Specific setup required: + // * crate has a binary so that it is published with a lockfile + // * crate has a library so that it is documented by docs.rs + // * crate has an optional dependency + // * metadata enables the optional dependency for docs.rs + // * `cargo doc` fails with the version of the dependency in the lockfile + // * there is a newer version of the dependency available that correctly builds + let crate_ = "docs_rs_test_incorrect_lockfile"; + let version = "0.1.2"; + let mut builder = RustwideBuilder::init(&env.context).unwrap(); + builder.update_toolchain()?; + assert!( + builder + .build_package(crate_, version, PackageKind::CratesIo, false)? + .successful + ); - Ok(()) - }); + Ok(()) + }, + ); } #[test] #[ignore] fn test_locked_fails_unlocked_needs_new_unknown_deps() { - wrapper(|env| { - env.override_config(|cfg| cfg.include_default_targets = false); - - // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(env, "value-bag-sval2", "1.4.1")?; - - // Similar to above, this crate fails to build with the published - // lockfile, but generating a new working lockfile requires - // introducing a completely new dependency (not just version) which - // would not have had its details pulled down from the sparse-index. - let crate_ = "docs_rs_test_incorrect_lockfile"; - let version = "0.2.0"; - let mut builder = RustwideBuilder::init(&env.context).unwrap(); - builder.update_toolchain()?; - assert!( - builder - .build_package(crate_, version, PackageKind::CratesIo, false)? - .successful - ); + wrapper_with_config( + |cfg| { + cfg.include_default_targets = false; + }, + |env| { + // if the corrected dependency of the crate was already downloaded we need to remove it + remove_cache_files(env, "value-bag-sval2", "1.4.1")?; + + // Similar to above, this crate fails to build with the published + // lockfile, but generating a new working lockfile requires + // introducing a completely new dependency (not just version) which + // would not have had its details pulled down from the sparse-index. + let crate_ = "docs_rs_test_incorrect_lockfile"; + let version = "0.2.0"; + let mut builder = RustwideBuilder::init(&env.context).unwrap(); + builder.update_toolchain()?; + assert!( + builder + .build_package(crate_, version, PackageKind::CratesIo, false)? + .successful + ); - Ok(()) - }); + Ok(()) + }, + ); } #[test] @@ -2028,26 +2034,26 @@ mod tests { fn test_workspace_reinitialize_after_interval() { use std::thread::sleep; use std::time::Duration; - wrapper(|env: &TestEnvironment| { - env.override_config(|cfg: &mut Config| { - cfg.build_workspace_reinitialization_interval = Duration::from_secs(1) - }); - let mut builder = RustwideBuilder::init(&env.context)?; - builder.update_toolchain()?; - assert!( - builder - .build_local_package(Path::new("tests/crates/build-std"))? - .successful - ); - sleep(Duration::from_secs(1)); - builder.reinitialize_workspace_if_interval_passed(&env.context)?; - assert!( - builder - .build_local_package(Path::new("tests/crates/build-std"))? - .successful - ); - Ok(()) - }) + wrapper_with_config( + |cfg| cfg.build_workspace_reinitialization_interval = Duration::from_secs(1), + |env| { + let mut builder = RustwideBuilder::init(&env.context)?; + builder.update_toolchain()?; + assert!( + builder + .build_local_package(Path::new("tests/crates/build-std"))? + .successful + ); + sleep(Duration::from_secs(1)); + builder.reinitialize_workspace_if_interval_passed(&env.context)?; + assert!( + builder + .build_local_package(Path::new("tests/crates/build-std"))? + .successful + ); + Ok(()) + }, + ) } #[test] diff --git a/src/test/mod.rs b/src/test/mod.rs index 0fb36c932..824a80740 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -25,21 +25,42 @@ use tower::ServiceExt; use tracing::error; #[track_caller] -pub(crate) fn wrapper(f: impl FnOnce(&TestEnvironment) -> Result<()>) { - let env = TestEnvironment::new(); +pub(crate) fn wrapper_with_config( + override_config: impl FnOnce(&mut Config), + f: impl FnOnce(&TestEnvironment) -> Result<()>, +) { + let mut config = TestEnvironment::base_config(); + override_config(&mut config); + let env = TestEnvironment::with_config(config); f(&env).expect("test failed"); } -pub(crate) fn async_wrapper(f: F) +#[track_caller] +pub(crate) fn wrapper(f: impl FnOnce(&TestEnvironment) -> Result<()>) { + wrapper_with_config(|_| {}, f) +} + +pub(crate) fn async_wrapper_with_config(override_config: impl FnOnce(&mut Config), f: F) where F: FnOnce(Rc) -> Fut, Fut: Future>, { - let env = Rc::new(TestEnvironment::new()); + let mut config = TestEnvironment::base_config(); + override_config(&mut config); + + let env = Rc::new(TestEnvironment::with_config(config)); env.runtime().block_on(f(env.clone())).expect("test failed"); } +pub(crate) fn async_wrapper(f: F) +where + F: FnOnce(Rc) -> Fut, + Fut: Future>, +{ + async_wrapper_with_config(|_| {}, f) +} + pub(crate) trait AxumResponseTestExt { async fn text(self) -> Result; async fn bytes(self) -> Result; @@ -330,10 +351,6 @@ pub(crate) fn init_logger() { } impl TestEnvironment { - fn new() -> Self { - Self::with_config(Self::base_config()) - } - fn with_config(config: Config) -> Self { init_logger(); @@ -442,16 +459,6 @@ impl TestEnvironment { config } - pub(crate) fn override_config(&self, f: impl FnOnce(&mut Config)) { - let mut config = Self::base_config(); - f(&mut config); - - // FIXME: fix - // if self.context.config.set(Arc::new(config)).is_err() { - // panic!("can't call override_config after the configuration is accessed!"); - // } - } - pub(crate) fn async_build_queue(&self) -> Arc { self.context.async_build_queue.clone() }