diff --git a/Cargo.lock b/Cargo.lock index 5db3dc14d57..50944dbe42b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1136,7 +1136,6 @@ dependencies = [ "tracing", "tracing-subscriber", "typomania", - "unicode-xid", "url", "utoipa", "utoipa-axum", @@ -1168,11 +1167,28 @@ dependencies = [ name = "crates_io_database" version = "0.0.0" dependencies = [ + "bon", + "chrono", + "claims", + "crates_io_diesel_helpers", + "crates_io_index", "crates_io_test_db", "diesel", "diesel-async", "diesel_full_text_search", + "futures-util", + "googletest", + "insta", + "rand 0.9.0", + "secrecy", + "semver", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.11", "tokio", + "tracing", + "unicode-xid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 48f591c33ea..f9d1c0aed61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,7 +132,6 @@ tracing = "=0.1.41" tracing-subscriber = { version = "=0.3.19", features = ["env-filter", "json"] } typomania = { version = "=0.1.2", default-features = false } url = "=2.5.4" -unicode-xid = "=0.2.6" utoipa = { version = "=5.3.1", features = ["chrono"] } utoipa-axum = "=0.2.0" diff --git a/crates/crates_io_database/Cargo.toml b/crates/crates_io_database/Cargo.toml index c2948b9965b..ca2d1eff877 100644 --- a/crates/crates_io_database/Cargo.toml +++ b/crates/crates_io_database/Cargo.toml @@ -8,10 +8,28 @@ edition = "2021" workspace = true [dependencies] -diesel = "=2.2.7" +bon = "=3.3.2" +chrono = { version = "=0.4.39", default-features = false, features = ["serde"] } +crates_io_diesel_helpers = { path = "../crates_io_diesel_helpers" } +crates_io_index = { path = "../crates_io_index" } +diesel = { version = "=2.2.7", features = ["serde_json", "chrono", "numeric"] } +diesel-async = "=0.5.2" diesel_full_text_search = "=2.2.0" +futures-util = "=0.3.31" +rand = "=0.9.0" +secrecy = "=0.10.3" +semver = { version = "=1.0.25", features = ["serde"] } +serde = { version = "=1.0.217", features = ["derive"] } +serde_json = "=1.0.138" +sha2 = "=0.10.8" +thiserror = "=2.0.11" +tracing = "=0.1.41" +unicode-xid = "=0.2.6" [dev-dependencies] +claims = "=0.8.0" crates_io_test_db = { path = "../crates_io_test_db" } diesel-async = { version = "=0.5.2", features = ["postgres"] } +googletest = "=0.13.0" +insta = "=1.42.1" tokio = { version = "=1.43.0", features = ["macros", "rt"] } diff --git a/crates/crates_io_database/src/lib.rs b/crates/crates_io_database/src/lib.rs index 8d9d416a8c9..c548a1b12ec 100644 --- a/crates/crates_io_database/src/lib.rs +++ b/crates/crates_io_database/src/lib.rs @@ -1,3 +1,5 @@ #![doc = include_str!("../README.md")] +pub mod models; pub mod schema; +pub mod utils; diff --git a/src/models/action.rs b/crates/crates_io_database/src/models/action.rs similarity index 100% rename from src/models/action.rs rename to crates/crates_io_database/src/models/action.rs diff --git a/src/models/category.rs b/crates/crates_io_database/src/models/category.rs similarity index 94% rename from src/models/category.rs rename to crates/crates_io_database/src/models/category.rs index 22611321f38..d536913fffe 100644 --- a/src/models/category.rs +++ b/crates/crates_io_database/src/models/category.rs @@ -1,15 +1,12 @@ +use crate::models::Crate; +use crate::schema::*; use chrono::NaiveDateTime; -use diesel::{ - delete, dsl, insert_into, sql_query, ExpressionMethods, QueryDsl, QueryResult, - TextExpressionMethods, -}; +use diesel::dsl; +use diesel::prelude::*; use diesel_async::scoped_futures::ScopedFutureExt; use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; use std::future::Future; -use crate::models::Crate; -use crate::schema::*; - #[derive(Clone, Identifiable, Queryable, QueryableByName, Debug, Selectable)] #[diesel(table_name = categories, check_for_backend(diesel::pg::Pg))] pub struct Category { @@ -21,7 +18,7 @@ pub struct Category { pub created_at: NaiveDateTime, } -type WithSlug<'a> = diesel::dsl::Eq>; +type WithSlug<'a> = dsl::Eq>; #[derive(Associations, Insertable, Identifiable, Debug, Clone, Copy)] #[diesel( @@ -73,12 +70,12 @@ impl Category { }) .collect::>(); - delete(crates_categories::table) + diesel::delete(crates_categories::table) .filter(crates_categories::crate_id.eq(crate_id)) .execute(conn) .await?; - insert_into(crates_categories::table) + diesel::insert_into(crates_categories::table) .values(&crate_categories) .execute(conn) .await?; @@ -113,7 +110,7 @@ impl Category { // Collect all the top-level categories and sum up the crates_cnt of // the crates in all subcategories - sql_query(format!(include_str!("toplevel.sql"), sort_sql)) + diesel::sql_query(format!(include_str!("toplevel.sql"), sort_sql)) .bind::(limit) .bind::(offset) .load(conn) @@ -122,7 +119,7 @@ impl Category { pub async fn subcategories(&self, conn: &mut AsyncPgConnection) -> QueryResult> { use diesel::sql_types::Text; - sql_query(include_str!("../subcategories.sql")) + diesel::sql_query(include_str!("subcategories.sql")) .bind::(&self.category) .load(conn) .await @@ -138,7 +135,7 @@ impl Category { ) -> QueryResult> { use diesel::sql_types::Text; - sql_query(include_str!("../parent_categories.sql")) + diesel::sql_query(include_str!("parent_categories.sql")) .bind::(&self.slug) .load(conn) .await @@ -168,7 +165,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ ( categories::category.eq("Cat 2"), @@ -212,7 +209,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ new_cat("Cat 1", "cat1", 0), new_cat("Cat 2", "cat2", 2), @@ -243,7 +240,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ ( categories::category.eq("Cat 1"), @@ -292,7 +289,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ new_cat("Cat 1", "cat1", 1), new_cat("Cat 1::sub", "cat1::sub", 2), @@ -334,7 +331,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ new_cat("Cat 1", "cat1", 1), new_cat("Cat 1::sub", "cat1::sub", 2), @@ -381,7 +378,7 @@ mod tests { let test_db = TestDatabase::new(); let mut conn = test_db.async_connect().await; - insert_into(categories::table) + diesel::insert_into(categories::table) .values(&vec![ new_cat("Cat 1", "cat1", 1), new_cat("Cat 1::sub1", "cat1::sub1", 2), diff --git a/src/models/crate_owner_invitation.rs b/crates/crates_io_database/src/models/crate_owner_invitation.rs similarity index 100% rename from src/models/crate_owner_invitation.rs rename to crates/crates_io_database/src/models/crate_owner_invitation.rs diff --git a/src/models/default_versions.rs b/crates/crates_io_database/src/models/default_versions.rs similarity index 99% rename from src/models/default_versions.rs rename to crates/crates_io_database/src/models/default_versions.rs index a591ec29bd7..63ecce9f334 100644 --- a/src/models/default_versions.rs +++ b/crates/crates_io_database/src/models/default_versions.rs @@ -2,6 +2,7 @@ use crate::schema::{default_versions, versions}; use crates_io_diesel_helpers::SemverVersion; use diesel::prelude::*; use diesel_async::{AsyncPgConnection, RunQueryDsl}; +use tracing::{debug, instrument, warn}; /// A subset of the columns of the `versions` table. /// @@ -137,6 +138,7 @@ async fn calculate_default_version( mod tests { use super::*; use crate::schema::crates; + use claims::assert_some; use crates_io_test_db::TestDatabase; use insta::assert_snapshot; use std::fmt::Write; diff --git a/src/models/deleted_crate.rs b/crates/crates_io_database/src/models/deleted_crate.rs similarity index 83% rename from src/models/deleted_crate.rs rename to crates/crates_io_database/src/models/deleted_crate.rs index a21d667589f..553f1f8b5aa 100644 --- a/src/models/deleted_crate.rs +++ b/crates/crates_io_database/src/models/deleted_crate.rs @@ -1,9 +1,9 @@ +use crate::schema::deleted_crates; use bon::Builder; use chrono::{DateTime, Utc}; -use crates_io_database::schema::deleted_crates; /// Struct used to `INSERT` a new `deleted_crates` record into the database. -#[derive(Insertable, Debug, Builder)] +#[derive(diesel::Insertable, Debug, Builder)] #[diesel(table_name = deleted_crates, check_for_backend(diesel::pg::Pg))] pub struct NewDeletedCrate<'a> { #[builder(start_fn)] diff --git a/src/models/dependency.rs b/crates/crates_io_database/src/models/dependency.rs similarity index 98% rename from src/models/dependency.rs rename to crates/crates_io_database/src/models/dependency.rs index 15fd46c1f94..b84e722f7f9 100644 --- a/src/models/dependency.rs +++ b/crates/crates_io_database/src/models/dependency.rs @@ -1,9 +1,9 @@ -use diesel::sql_types::{BigInt, Text}; - use crate::models::{Crate, Version}; use crate::schema::*; use crates_io_diesel_helpers::pg_enum; use crates_io_index::DependencyKind as IndexDependencyKind; +use diesel::prelude::*; +use diesel::sql_types::{BigInt, Text}; #[derive(Identifiable, Associations, Debug, Queryable, QueryableByName, Selectable)] #[diesel( diff --git a/src/models/download.rs b/crates/crates_io_database/src/models/download.rs similarity index 97% rename from src/models/download.rs rename to crates/crates_io_database/src/models/download.rs index cd2d562e318..88359bacb5a 100644 --- a/src/models/download.rs +++ b/crates/crates_io_database/src/models/download.rs @@ -2,6 +2,7 @@ use crate::models::Version as FullVersion; use crate::schema::{version_downloads, versions}; use chrono::NaiveDate; use crates_io_diesel_helpers::SemverVersion; +use diesel::prelude::*; #[derive(Queryable, Identifiable, Associations, Debug, Clone, Copy)] #[diesel( diff --git a/src/models/email.rs b/crates/crates_io_database/src/models/email.rs similarity index 97% rename from src/models/email.rs rename to crates/crates_io_database/src/models/email.rs index c2c597247e8..6fa853b285f 100644 --- a/src/models/email.rs +++ b/crates/crates_io_database/src/models/email.rs @@ -1,6 +1,6 @@ use bon::Builder; use chrono::NaiveDateTime; -use diesel::{OptionalExtension, QueryResult}; +use diesel::prelude::*; use diesel_async::{AsyncPgConnection, RunQueryDsl}; use secrecy::SecretString; diff --git a/src/models/follow.rs b/crates/crates_io_database/src/models/follow.rs similarity index 93% rename from src/models/follow.rs rename to crates/crates_io_database/src/models/follow.rs index 11fc5e7ea87..9e501d95856 100644 --- a/src/models/follow.rs +++ b/crates/crates_io_database/src/models/follow.rs @@ -1,5 +1,6 @@ use crate::models::User; use crate::schema::follows; +use diesel::prelude::*; #[derive(Insertable, Queryable, Identifiable, Associations, Clone, Copy, Debug)] #[diesel( diff --git a/src/models/helpers.rs b/crates/crates_io_database/src/models/helpers.rs similarity index 100% rename from src/models/helpers.rs rename to crates/crates_io_database/src/models/helpers.rs diff --git a/src/models/helpers/with_count.rs b/crates/crates_io_database/src/models/helpers/with_count.rs similarity index 89% rename from src/models/helpers/with_count.rs rename to crates/crates_io_database/src/models/helpers/with_count.rs index 2409ef1da0c..697a9025cf0 100644 --- a/src/models/helpers/with_count.rs +++ b/crates/crates_io_database/src/models/helpers/with_count.rs @@ -1,11 +1,12 @@ +use diesel::prelude::*; use diesel::sql_types::BigInt; #[derive(QueryableByName, Queryable, Debug)] pub struct WithCount { #[diesel(embed)] - pub(crate) record: T, + pub record: T, #[diesel(sql_type = BigInt)] - pub(crate) total: i64, + pub total: i64, } pub trait WithCountExtension { diff --git a/src/models/keyword.rs b/crates/crates_io_database/src/models/keyword.rs similarity index 100% rename from src/models/keyword.rs rename to crates/crates_io_database/src/models/keyword.rs diff --git a/src/models/krate.rs b/crates/crates_io_database/src/models/krate.rs similarity index 99% rename from src/models/krate.rs rename to crates/crates_io_database/src/models/krate.rs index 01cf6937544..0031e958ac6 100644 --- a/src/models/krate.rs +++ b/crates/crates_io_database/src/models/krate.rs @@ -1,4 +1,9 @@ +use crate::models::helpers::with_count::*; +use crate::models::version::TopVersions; +use crate::models::{CrateOwner, Owner, OwnerKind, ReverseDependency, User, Version}; +use crate::schema::*; use chrono::NaiveDateTime; +use crates_io_diesel_helpers::canon_crate_name; use diesel::associations::Identifiable; use diesel::dsl; use diesel::pg::Pg; @@ -8,12 +13,7 @@ use diesel_async::scoped_futures::ScopedFutureExt; use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; use secrecy::SecretString; use thiserror::Error; - -use crate::models::helpers::with_count::*; -use crate::models::version::TopVersions; -use crate::models::{CrateOwner, Owner, OwnerKind, ReverseDependency, User, Version}; -use crate::schema::*; -use crates_io_diesel_helpers::canon_crate_name; +use tracing::instrument; use super::Team; @@ -422,7 +422,7 @@ impl Crate { /// Returns (dependency, dependent crate name, dependent crate downloads) #[instrument(skip_all, fields(krate.name = %self.name))] - pub(crate) async fn reverse_dependencies( + pub async fn reverse_dependencies( &self, conn: &mut AsyncPgConnection, offset: i64, @@ -546,6 +546,7 @@ pub enum InvalidDependencyName { #[cfg(test)] mod tests { use crate::models::Crate; + use claims::{assert_err_eq, assert_ok}; #[test] fn validate_crate_name() { diff --git a/src/models/krate_reverse_dependencies.sql b/crates/crates_io_database/src/models/krate_reverse_dependencies.sql similarity index 100% rename from src/models/krate_reverse_dependencies.sql rename to crates/crates_io_database/src/models/krate_reverse_dependencies.sql diff --git a/src/models.rs b/crates/crates_io_database/src/models/mod.rs similarity index 100% rename from src/models.rs rename to crates/crates_io_database/src/models/mod.rs diff --git a/src/models/owner.rs b/crates/crates_io_database/src/models/owner.rs similarity index 100% rename from src/models/owner.rs rename to crates/crates_io_database/src/models/owner.rs diff --git a/src/parent_categories.sql b/crates/crates_io_database/src/models/parent_categories.sql similarity index 100% rename from src/parent_categories.sql rename to crates/crates_io_database/src/models/parent_categories.sql diff --git a/src/subcategories.sql b/crates/crates_io_database/src/models/subcategories.sql similarity index 100% rename from src/subcategories.sql rename to crates/crates_io_database/src/models/subcategories.sql diff --git a/src/models/team.rs b/crates/crates_io_database/src/models/team.rs similarity index 95% rename from src/models/team.rs rename to crates/crates_io_database/src/models/team.rs index a04d3e65c02..8b01d5489bb 100644 --- a/src/models/team.rs +++ b/crates/crates_io_database/src/models/team.rs @@ -7,7 +7,7 @@ use crate::schema::{crate_owners, teams}; /// For now, just a Github Team. Can be upgraded to other teams /// later if desirable. -#[derive(Queryable, Identifiable, Serialize, Deserialize, Debug, Selectable)] +#[derive(Queryable, Identifiable, serde::Serialize, serde::Deserialize, Debug, Selectable)] pub struct Team { /// Unique table id pub id: i32, diff --git a/src/models/token.rs b/crates/crates_io_database/src/models/token.rs similarity index 96% rename from src/models/token.rs rename to crates/crates_io_database/src/models/token.rs index 231a634982d..03b8b883cf8 100644 --- a/src/models/token.rs +++ b/crates/crates_io_database/src/models/token.rs @@ -10,8 +10,8 @@ use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; pub use self::scopes::{CrateScope, EndpointScope}; use crate::models::User; use crate::schema::api_tokens; -use crate::util::rfc3339; -use crate::util::token::{HashedToken, PlainToken}; +use crate::utils::rfc3339; +use crate::utils::token::{HashedToken, PlainToken}; #[derive(Debug, Insertable, Builder)] #[diesel(table_name = api_tokens, check_for_backend(diesel::pg::Pg))] @@ -39,7 +39,7 @@ impl NewApiToken { } /// The model representing a row in the `api_tokens` database table. -#[derive(Debug, Identifiable, Queryable, Selectable, Associations, Serialize)] +#[derive(Debug, Identifiable, Queryable, Selectable, Associations, serde::Serialize)] #[diesel(belongs_to(User))] pub struct ApiToken { pub id: i32, @@ -99,6 +99,7 @@ impl ApiToken { mod tests { use super::*; use chrono::NaiveDate; + use claims::assert_some; #[test] fn api_token_serializes_to_rfc3339() { diff --git a/src/models/token/scopes.rs b/crates/crates_io_database/src/models/token/scopes.rs similarity index 97% rename from src/models/token/scopes.rs rename to crates/crates_io_database/src/models/token/scopes.rs index a3b9579935f..be677a2aa6c 100644 --- a/src/models/token/scopes.rs +++ b/crates/crates_io_database/src/models/token/scopes.rs @@ -5,7 +5,7 @@ use diesel::serialize::{self, IsNull, Output, ToSql}; use diesel::sql_types::Text; use std::io::Write; -#[derive(Clone, Copy, Debug, PartialEq, Eq, AsExpression, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, diesel::AsExpression, serde::Serialize)] #[diesel(sql_type = Text)] #[serde(rename_all = "kebab-case")] pub enum EndpointScope { @@ -54,7 +54,7 @@ impl FromSql for EndpointScope { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] #[serde(transparent)] pub struct CrateScope { pattern: String, @@ -126,6 +126,7 @@ impl CrateScope { #[cfg(test)] mod tests { use super::*; + use claims::assert_ok; use googletest::prelude::*; #[googletest::test] diff --git a/src/models/toplevel.sql b/crates/crates_io_database/src/models/toplevel.sql similarity index 100% rename from src/models/toplevel.sql rename to crates/crates_io_database/src/models/toplevel.sql diff --git a/src/models/user.rs b/crates/crates_io_database/src/models/user.rs similarity index 100% rename from src/models/user.rs rename to crates/crates_io_database/src/models/user.rs diff --git a/src/models/version.rs b/crates/crates_io_database/src/models/version.rs similarity index 100% rename from src/models/version.rs rename to crates/crates_io_database/src/models/version.rs diff --git a/crates/crates_io_database/src/utils/mod.rs b/crates/crates_io_database/src/utils/mod.rs new file mode 100644 index 00000000000..a03b958f225 --- /dev/null +++ b/crates/crates_io_database/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod rfc3339; +pub mod token; diff --git a/src/util/rfc3339.rs b/crates/crates_io_database/src/utils/rfc3339.rs similarity index 100% rename from src/util/rfc3339.rs rename to crates/crates_io_database/src/utils/rfc3339.rs diff --git a/src/util/token.rs b/crates/crates_io_database/src/utils/token.rs similarity index 93% rename from src/util/token.rs rename to crates/crates_io_database/src/utils/token.rs index c83b1b069c1..6bd37934eef 100644 --- a/src/util/token.rs +++ b/crates/crates_io_database/src/utils/token.rs @@ -1,4 +1,8 @@ -use diesel::{deserialize::FromSql, pg::Pg, serialize::ToSql, sql_types::Bytea}; +use diesel::deserialize::FromSql; +use diesel::pg::Pg; +use diesel::serialize::ToSql; +use diesel::sql_types::Bytea; +use diesel::{AsExpression, FromSqlRow}; use rand::distr::{Alphanumeric, SampleString}; use secrecy::{ExposeSecret, SecretSlice, SecretString}; use sha2::{Digest, Sha256}; @@ -60,7 +64,7 @@ impl FromSql for HashedToken { pub struct PlainToken(SecretString); impl PlainToken { - pub(crate) fn generate() -> Self { + pub fn generate() -> Self { let plaintext = format!( "{}{}", TOKEN_PREFIX, @@ -90,6 +94,7 @@ fn generate_secure_alphanumeric_string(len: usize) -> String { #[cfg(test)] mod tests { use super::*; + use claims::assert_err; use googletest::prelude::*; #[test] diff --git a/src/lib.rs b/src/lib.rs index 268feb21e1e..a40e3d32699 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ extern crate serde; extern crate tracing; pub use crate::{app::App, email::Emails}; -pub use crates_io_database::schema; +pub use crates_io_database::{models, schema}; use std::sync::Arc; use crate::app::AppState; @@ -41,7 +41,6 @@ pub mod index; mod licenses; pub mod metrics; pub mod middleware; -pub mod models; pub mod openapi; pub mod rate_limiter; mod real_ip; diff --git a/src/util.rs b/src/util.rs index 67a389851f8..7a8ae22bf73 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,11 +1,10 @@ pub use self::io_util::{read_fill, read_le_u32}; pub use self::request_helpers::*; +pub use crates_io_database::utils::{rfc3339, token}; pub mod diesel; pub mod errors; mod io_util; mod request_helpers; -pub mod rfc3339; pub mod string_excl_null; -pub mod token; pub mod tracing;