Skip to content

Commit 445769c

Browse files
sypharsyphar
committed
make consistency check async
1 parent a96a6d4 commit 445769c

File tree

3 files changed

+112
-87
lines changed

3 files changed

+112
-87
lines changed

‎src/bin/cratesfyi.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -688,7 +688,8 @@ impl DatabaseSubcommand {
688688
Self::Limits { command } => command.handle_args(ctx)?,
689689

690690
Self::Synchronize { dry_run } => {
691-
docs_rs::utils::consistency::run_check(&ctx, dry_run)?;
691+
ctx.runtime()?
692+
.block_on(docs_rs::utils::consistency::run_check(&ctx, dry_run))?;
692693
}
693694
}
694695
Ok(())

‎src/utils/consistency/db.rs‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,23 +62,31 @@ pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Resu
6262
#[cfg(test)]
6363
mod tests {
6464
use super::*;
65-
use crate::test::wrapper;
65+
use crate::test::async_wrapper;
6666

6767
#[test]
6868
fn test_load() {
69-
wrapper(|env| {
70-
env.build_queue().add_crate("queued", "0.0.1", 0, None)?;
71-
env.fake_release().name("krate").version("0.0.2").create()?;
72-
env.fake_release()
69+
async_wrapper(|env| async move {
70+
env.async_build_queue()
71+
.await
72+
.add_crate("queued", "0.0.1", 0, None)
73+
.await?;
74+
env.async_fake_release()
75+
.await
76+
.name("krate")
77+
.version("0.0.2")
78+
.create_async()
79+
.await?;
80+
env.async_fake_release()
81+
.await
7382
.name("krate")
7483
.version("0.0.3")
7584
.yanked(true)
76-
.create()?;
85+
.create_async()
86+
.await?;
7787

78-
let result = env.runtime().block_on(async {
79-
let mut conn = env.async_db().await.async_conn().await;
80-
load(&mut conn, &env.config()).await
81-
})?;
88+
let mut conn = env.async_db().await.async_conn().await;
89+
let result = load(&mut conn, &env.config()).await?;
8290

8391
assert_eq!(
8492
result,

‎src/utils/consistency/mod.rs‎

Lines changed: 92 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::{db::delete, Context};
1+
use crate::{db::delete, utils::spawn_blocking, Context};
22
use anyhow::{Context as _, Result};
33
use itertools::Itertools;
44
use tracing::{info, warn};
@@ -24,24 +24,25 @@ const BUILD_PRIORITY: i32 = 15;
2424
///
2525
/// Even when activities fail, the command can just be re-run. While the diff calculation will
2626
/// be repeated, we won't re-execute fixing activities.
27-
pub fn run_check<C: Context>(ctx: &C, dry_run: bool) -> Result<()> {
27+
pub async fn run_check<C: Context>(ctx: &C, dry_run: bool) -> Result<()> {
2828
let index = ctx.index()?;
2929

3030
info!("Loading data from database...");
31-
let db_data = ctx
32-
.runtime()?
33-
.block_on(async {
34-
let mut conn = ctx.pool()?.get_async().await?;
35-
db::load(&mut conn, &*ctx.config()?).await
36-
})
31+
let mut conn = ctx.pool()?.get_async().await?;
32+
let db_data = db::load(&mut conn, &*ctx.config()?)
33+
.await
3734
.context("Loading crate data from database for consistency check")?;
3835

3936
tracing::info!("Loading data from index...");
40-
let index_data =
41-
index::load(&index).context("Loading crate data from index for consistency check")?;
37+
let index_data = spawn_blocking({
38+
let index = index.clone();
39+
move || index::load(&index)
40+
})
41+
.await
42+
.context("Loading crate data from index for consistency check")?;
4243

4344
let diff = diff::calculate_diff(db_data.iter(), index_data.iter());
44-
let result = handle_diff(ctx, diff.iter(), dry_run)?;
45+
let result = handle_diff(ctx, diff.iter(), dry_run).await?;
4546

4647
println!("============");
4748
println!("SUMMARY");
@@ -79,29 +80,27 @@ struct HandleResult {
7980
yanks_corrected: u32,
8081
}
8182

82-
fn handle_diff<'a, I, C>(ctx: &C, iter: I, dry_run: bool) -> Result<HandleResult>
83+
async fn handle_diff<'a, I, C>(ctx: &C, iter: I, dry_run: bool) -> Result<HandleResult>
8384
where
8485
I: Iterator<Item = &'a diff::Difference>,
8586
C: Context,
8687
{
8788
let mut result = HandleResult::default();
8889

8990
let config = ctx.config()?;
90-
let runtime = ctx.runtime()?;
9191

92-
let storage = runtime.block_on(ctx.async_storage())?;
93-
let build_queue = ctx.build_queue()?;
92+
let storage = ctx.async_storage().await?;
93+
let build_queue = ctx.async_build_queue().await?;
9494

95-
let mut conn = runtime.block_on(ctx.pool()?.get_async())?;
95+
let mut conn = ctx.pool()?.get_async().await?;
9696

9797
for difference in iter {
9898
println!("{difference}");
9999

100100
match difference {
101101
diff::Difference::CrateNotInIndex(name) => {
102102
if !dry_run {
103-
if let Err(err) =
104-
runtime.block_on(delete::delete_crate(&mut conn, &storage, &config, name))
103+
if let Err(err) = delete::delete_crate(&mut conn, &storage, &config, name).await
105104
{
106105
warn!("{:?}", err);
107106
}
@@ -111,7 +110,9 @@ where
111110
diff::Difference::CrateNotInDb(name, versions) => {
112111
for version in versions {
113112
if !dry_run {
114-
if let Err(err) = build_queue.add_crate(name, version, BUILD_PRIORITY, None)
113+
if let Err(err) = build_queue
114+
.add_crate(name, version, BUILD_PRIORITY, None)
115+
.await
115116
{
116117
warn!("{:?}", err);
117118
}
@@ -121,25 +122,28 @@ where
121122
}
122123
diff::Difference::ReleaseNotInIndex(name, version) => {
123124
if !dry_run {
124-
if let Err(err) = runtime.block_on(delete::delete_version(
125-
&mut conn, &storage, &config, name, version,
126-
)) {
125+
if let Err(err) =
126+
delete::delete_version(&mut conn, &storage, &config, name, version).await
127+
{
127128
warn!("{:?}", err);
128129
}
129130
}
130131
result.releases_deleted += 1;
131132
}
132133
diff::Difference::ReleaseNotInDb(name, version) => {
133134
if !dry_run {
134-
if let Err(err) = build_queue.add_crate(name, version, BUILD_PRIORITY, None) {
135+
if let Err(err) = build_queue
136+
.add_crate(name, version, BUILD_PRIORITY, None)
137+
.await
138+
{
135139
warn!("{:?}", err);
136140
}
137141
}
138142
result.builds_queued += 1;
139143
}
140144
diff::Difference::ReleaseYank(name, version, yanked) => {
141145
if !dry_run {
142-
if let Err(err) = build_queue.set_yanked(name, version, *yanked) {
146+
if let Err(err) = build_queue.set_yanked(name, version, *yanked).await {
143147
warn!("{:?}", err);
144148
}
145149
}
@@ -155,57 +159,55 @@ where
155159
mod tests {
156160
use super::diff::Difference;
157161
use super::*;
158-
use crate::test::{wrapper, TestEnvironment};
162+
use crate::test::{async_wrapper, wrapper, TestEnvironment};
159163
use sqlx::Row as _;
160164

161-
fn count(env: &TestEnvironment, sql: &str) -> Result<i64> {
162-
Ok(env.runtime().block_on(async {
163-
let mut conn = env.async_db().await.async_conn().await;
164-
sqlx::query_scalar(sql).fetch_one(&mut *conn).await
165-
})?)
165+
async fn count(env: &TestEnvironment, sql: &str) -> Result<i64> {
166+
let mut conn = env.async_db().await.async_conn().await;
167+
Ok(sqlx::query_scalar(sql).fetch_one(&mut *conn).await?)
166168
}
167169

168-
fn single_row<O>(env: &TestEnvironment, sql: &str) -> Result<Vec<O>>
170+
async fn single_row<O>(env: &TestEnvironment, sql: &str) -> Result<Vec<O>>
169171
where
170172
O: Send + Unpin + for<'r> sqlx::Decode<'r, sqlx::Postgres> + sqlx::Type<sqlx::Postgres>,
171173
{
172-
env.runtime().block_on(async {
173-
let mut conn = env.async_db().await.async_conn().await;
174-
Ok::<_, anyhow::Error>(
175-
sqlx::query(sql)
176-
.fetch_all(&mut *conn)
177-
.await?
178-
.into_iter()
179-
.map(|row| row.get(0))
180-
.collect(),
181-
)
182-
})
174+
let mut conn = env.async_db().await.async_conn().await;
175+
Ok::<_, anyhow::Error>(
176+
sqlx::query(sql)
177+
.fetch_all(&mut *conn)
178+
.await?
179+
.into_iter()
180+
.map(|row| row.get(0))
181+
.collect(),
182+
)
183183
}
184184

185185
#[test]
186186
fn test_delete_crate() {
187-
wrapper(|env| {
188-
env.fake_release()
187+
async_wrapper(|env| async move {
188+
env.async_fake_release()
189+
.await
189190
.name("krate")
190191
.version("0.1.1")
191192
.version("0.1.2")
192-
.create()?;
193+
.create_async()
194+
.await?;
193195

194196
let diff = [Difference::CrateNotInIndex("krate".into())];
195197

196198
// calling with dry-run leads to no change
197-
handle_diff(env, diff.iter(), true)?;
199+
handle_diff(&*env, diff.iter(), true).await?;
198200

199201
assert_eq!(
200-
count(env, "SELECT count(*) FROM crates WHERE name = 'krate'")?,
202+
count(&*env, "SELECT count(*) FROM crates WHERE name = 'krate'").await?,
201203
1
202204
);
203205

204206
// without dry-run the crate will be deleted
205-
handle_diff(env, diff.iter(), false)?;
207+
handle_diff(&*env, diff.iter(), false).await?;
206208

207209
assert_eq!(
208-
count(env, "SELECT count(*) FROM crates WHERE name = 'krate'")?,
210+
count(&*env, "SELECT count(*) FROM crates WHERE name = 'krate'").await?,
209211
0
210212
);
211213

@@ -215,25 +217,35 @@ mod tests {
215217

216218
#[test]
217219
fn test_delete_release() {
218-
wrapper(|env| {
219-
env.fake_release().name("krate").version("0.1.1").create()?;
220-
env.fake_release().name("krate").version("0.1.2").create()?;
220+
async_wrapper(|env| async move {
221+
env.async_fake_release()
222+
.await
223+
.name("krate")
224+
.version("0.1.1")
225+
.create_async()
226+
.await?;
227+
env.async_fake_release()
228+
.await
229+
.name("krate")
230+
.version("0.1.2")
231+
.create_async()
232+
.await?;
221233

222234
let diff = [Difference::ReleaseNotInIndex(
223235
"krate".into(),
224236
"0.1.1".into(),
225237
)];
226238

227-
assert_eq!(count(env, "SELECT count(*) FROM releases")?, 2);
239+
assert_eq!(count(&*env, "SELECT count(*) FROM releases").await?, 2);
228240

229-
handle_diff(env, diff.iter(), true)?;
241+
handle_diff(&*env, diff.iter(), true).await?;
230242

231-
assert_eq!(count(env, "SELECT count(*) FROM releases")?, 2);
243+
assert_eq!(count(&*env, "SELECT count(*) FROM releases").await?, 2);
232244

233-
handle_diff(env, diff.iter(), false)?;
245+
handle_diff(&*env, diff.iter(), false).await?;
234246

235247
assert_eq!(
236-
single_row::<String>(env, "SELECT version FROM releases")?,
248+
single_row::<String>(&*env, "SELECT version FROM releases").await?,
237249
vec!["0.1.2"]
238250
);
239251

@@ -243,30 +255,32 @@ mod tests {
243255

244256
#[test]
245257
fn test_wrong_yank() {
246-
wrapper(|env| {
247-
env.fake_release()
258+
async_wrapper(|env| async move {
259+
env.async_fake_release()
260+
.await
248261
.name("krate")
249262
.version("0.1.1")
250263
.yanked(true)
251-
.create()?;
264+
.create_async()
265+
.await?;
252266

253267
let diff = [Difference::ReleaseYank(
254268
"krate".into(),
255269
"0.1.1".into(),
256270
false,
257271
)];
258272

259-
handle_diff(env, diff.iter(), true)?;
273+
handle_diff(&*env, diff.iter(), true).await?;
260274

261275
assert_eq!(
262-
single_row::<bool>(env, "SELECT yanked FROM releases")?,
276+
single_row::<bool>(&*env, "SELECT yanked FROM releases").await?,
263277
vec![true]
264278
);
265279

266-
handle_diff(env, diff.iter(), false)?;
280+
handle_diff(&*env, diff.iter(), false).await?;
267281

268282
assert_eq!(
269-
single_row::<bool>(env, "SELECT yanked FROM releases")?,
283+
single_row::<bool>(&*env, "SELECT yanked FROM releases").await?,
270284
vec![false]
271285
);
272286

@@ -276,20 +290,21 @@ mod tests {
276290

277291
#[test]
278292
fn test_missing_release_in_db() {
279-
wrapper(|env| {
293+
async_wrapper(|env| async move {
280294
let diff = [Difference::ReleaseNotInDb("krate".into(), "0.1.1".into())];
281295

282-
handle_diff(env, diff.iter(), true)?;
296+
handle_diff(&*env, diff.iter(), true).await?;
283297

284-
let build_queue = env.build_queue();
298+
let build_queue = env.async_build_queue().await;
285299

286-
assert!(build_queue.queued_crates()?.is_empty());
300+
assert!(build_queue.queued_crates().await?.is_empty());
287301

288-
handle_diff(env, diff.iter(), false)?;
302+
handle_diff(&*env, diff.iter(), false).await?;
289303

290304
assert_eq!(
291305
build_queue
292-
.queued_crates()?
306+
.queued_crates()
307+
.await?
293308
.iter()
294309
.map(|c| (c.name.as_str(), c.version.as_str(), c.priority))
295310
.collect::<Vec<_>>(),
@@ -301,23 +316,24 @@ mod tests {
301316

302317
#[test]
303318
fn test_missing_crate_in_db() {
304-
wrapper(|env| {
319+
async_wrapper(|env| async move {
305320
let diff = [Difference::CrateNotInDb(
306321
"krate".into(),
307322
vec!["0.1.1".into(), "0.1.2".into()],
308323
)];
309324

310-
handle_diff(env, diff.iter(), true)?;
325+
handle_diff(&*env, diff.iter(), true).await?;
311326

312-
let build_queue = env.build_queue();
327+
let build_queue = env.async_build_queue().await;
313328

314-
assert!(build_queue.queued_crates()?.is_empty());
329+
assert!(build_queue.queued_crates().await?.is_empty());
315330

316-
handle_diff(env, diff.iter(), false)?;
331+
handle_diff(&*env, diff.iter(), false).await?;
317332

318333
assert_eq!(
319334
build_queue
320-
.queued_crates()?
335+
.queued_crates()
336+
.await?
321337
.iter()
322338
.map(|c| (c.name.as_str(), c.version.as_str(), c.priority))
323339
.collect::<Vec<_>>(),

0 commit comments

Comments
 (0)