|
| 1 | +//! This module contains the glue code between our database and the index files |
| 2 | +//! and is used by the corresponding background jobs to generate the |
| 3 | +//! index files. |
| 4 | +
|
| 5 | +use crate::models::{Crate, CrateVersions, Dependency, Version}; |
| 6 | +use crate::schema::{crates, dependencies}; |
| 7 | +use crate::util::diesel::Conn; |
| 8 | +use anyhow::Context; |
| 9 | +use crates_io_index::features::split_features; |
| 10 | +use diesel::prelude::*; |
| 11 | +use sentry::Level; |
| 12 | + |
| 13 | +#[instrument(skip_all, fields(krate.name = ?name))] |
| 14 | +pub fn get_index_data(name: &str, conn: &mut impl Conn) -> anyhow::Result<Option<String>> { |
| 15 | + debug!("Looking up crate by name"); |
| 16 | + let Some(krate): Option<Crate> = Crate::by_exact_name(name).first(conn).optional()? else { |
| 17 | + return Ok(None); |
| 18 | + }; |
| 19 | + |
| 20 | + debug!("Gathering remaining index data"); |
| 21 | + let crates = index_metadata(&krate, conn).context("Failed to gather index metadata")?; |
| 22 | + |
| 23 | + // This can sometimes happen when we delete versions upon owner request |
| 24 | + // but don't realize that the crate is now left with no versions at all. |
| 25 | + // |
| 26 | + // In this case we will delete the crate from the index and log a warning to |
| 27 | + // Sentry to clean this up in the database. |
| 28 | + if crates.is_empty() { |
| 29 | + let message = format!("Crate `{name}` has no versions left"); |
| 30 | + sentry::capture_message(&message, Level::Warning); |
| 31 | + |
| 32 | + return Ok(None); |
| 33 | + } |
| 34 | + |
| 35 | + debug!("Serializing index data"); |
| 36 | + let mut bytes = Vec::new(); |
| 37 | + crates_io_index::write_crates(&crates, &mut bytes) |
| 38 | + .context("Failed to serialize index metadata")?; |
| 39 | + |
| 40 | + let str = String::from_utf8(bytes).context("Failed to decode index metadata as utf8")?; |
| 41 | + |
| 42 | + Ok(Some(str)) |
| 43 | +} |
| 44 | + |
| 45 | +/// Gather all the necessary data to write an index metadata file |
| 46 | +pub fn index_metadata( |
| 47 | + krate: &Crate, |
| 48 | + conn: &mut impl Conn, |
| 49 | +) -> QueryResult<Vec<crates_io_index::Crate>> { |
| 50 | + let mut versions: Vec<Version> = krate.all_versions().load(conn)?; |
| 51 | + |
| 52 | + // We sort by `created_at` by default, but since tests run within a |
| 53 | + // single database transaction the versions will all have the same |
| 54 | + // `created_at` timestamp, so we sort by semver as a secondary key. |
| 55 | + versions.sort_by_cached_key(|k| (k.created_at, semver::Version::parse(&k.num).ok())); |
| 56 | + |
| 57 | + let deps: Vec<(Dependency, String)> = Dependency::belonging_to(&versions) |
| 58 | + .inner_join(crates::table) |
| 59 | + .select((dependencies::all_columns, crates::name)) |
| 60 | + .load(conn)?; |
| 61 | + |
| 62 | + let deps = deps.grouped_by(&versions); |
| 63 | + |
| 64 | + versions |
| 65 | + .into_iter() |
| 66 | + .zip(deps) |
| 67 | + .map(|(version, deps)| { |
| 68 | + let mut deps = deps |
| 69 | + .into_iter() |
| 70 | + .map(|(dep, name)| { |
| 71 | + // If this dependency has an explicit name in `Cargo.toml` that |
| 72 | + // means that the `name` we have listed is actually the package name |
| 73 | + // that we're depending on. The `name` listed in the index is the |
| 74 | + // Cargo.toml-written-name which is what cargo uses for |
| 75 | + // `--extern foo=...` |
| 76 | + let (name, package) = match dep.explicit_name { |
| 77 | + Some(explicit_name) => (explicit_name, Some(name)), |
| 78 | + None => (name, None), |
| 79 | + }; |
| 80 | + |
| 81 | + crates_io_index::Dependency { |
| 82 | + name, |
| 83 | + req: dep.req, |
| 84 | + features: dep.features, |
| 85 | + optional: dep.optional, |
| 86 | + default_features: dep.default_features, |
| 87 | + kind: Some(dep.kind.into()), |
| 88 | + package, |
| 89 | + target: dep.target, |
| 90 | + } |
| 91 | + }) |
| 92 | + .collect::<Vec<_>>(); |
| 93 | + |
| 94 | + deps.sort(); |
| 95 | + |
| 96 | + let features = version.features().unwrap_or_default(); |
| 97 | + let (features, features2) = split_features(features); |
| 98 | + |
| 99 | + let (features2, v) = if features2.is_empty() { |
| 100 | + (None, None) |
| 101 | + } else { |
| 102 | + (Some(features2), Some(2)) |
| 103 | + }; |
| 104 | + |
| 105 | + let krate = crates_io_index::Crate { |
| 106 | + name: krate.name.clone(), |
| 107 | + vers: version.num.to_string(), |
| 108 | + cksum: version.checksum, |
| 109 | + yanked: Some(version.yanked), |
| 110 | + deps, |
| 111 | + features, |
| 112 | + links: version.links, |
| 113 | + rust_version: version.rust_version, |
| 114 | + features2, |
| 115 | + v, |
| 116 | + }; |
| 117 | + |
| 118 | + Ok(krate) |
| 119 | + }) |
| 120 | + .collect() |
| 121 | +} |
| 122 | + |
| 123 | +#[cfg(test)] |
| 124 | +mod tests { |
| 125 | + use super::*; |
| 126 | + use crate::schema::users; |
| 127 | + use crate::tests::builders::{CrateBuilder, VersionBuilder}; |
| 128 | + use chrono::{Days, Utc}; |
| 129 | + use crates_io_test_db::TestDatabase; |
| 130 | + use insta::assert_json_snapshot; |
| 131 | + |
| 132 | + #[test] |
| 133 | + fn test_index_metadata() { |
| 134 | + let test_db = TestDatabase::new(); |
| 135 | + let mut conn = test_db.connect(); |
| 136 | + |
| 137 | + let user_id = diesel::insert_into(users::table) |
| 138 | + .values(( |
| 139 | + users::name.eq("user1"), |
| 140 | + users::gh_login.eq("user1"), |
| 141 | + users::gh_id.eq(42), |
| 142 | + users::gh_access_token.eq("some random token"), |
| 143 | + )) |
| 144 | + .returning(users::id) |
| 145 | + .get_result::<i32>(&mut conn) |
| 146 | + .unwrap(); |
| 147 | + |
| 148 | + let created_at_1 = Utc::now() |
| 149 | + .checked_sub_days(Days::new(14)) |
| 150 | + .unwrap() |
| 151 | + .naive_utc(); |
| 152 | + |
| 153 | + let created_at_2 = Utc::now() |
| 154 | + .checked_sub_days(Days::new(7)) |
| 155 | + .unwrap() |
| 156 | + .naive_utc(); |
| 157 | + |
| 158 | + let fooo = CrateBuilder::new("foo", user_id) |
| 159 | + .version(VersionBuilder::new("0.1.0")) |
| 160 | + .expect_build(&mut conn); |
| 161 | + |
| 162 | + let metadata = index_metadata(&fooo, &mut conn).unwrap(); |
| 163 | + assert_json_snapshot!(metadata); |
| 164 | + |
| 165 | + let bar = CrateBuilder::new("bar", user_id) |
| 166 | + .version( |
| 167 | + VersionBuilder::new("1.0.0-beta.1") |
| 168 | + .created_at(created_at_1) |
| 169 | + .yanked(true), |
| 170 | + ) |
| 171 | + .version(VersionBuilder::new("1.0.0").created_at(created_at_1)) |
| 172 | + .version( |
| 173 | + VersionBuilder::new("2.0.0") |
| 174 | + .created_at(created_at_2) |
| 175 | + .dependency(&fooo, None), |
| 176 | + ) |
| 177 | + .version(VersionBuilder::new("1.0.1").checksum("0123456789abcdef")) |
| 178 | + .expect_build(&mut conn); |
| 179 | + |
| 180 | + let metadata = index_metadata(&bar, &mut conn).unwrap(); |
| 181 | + assert_json_snapshot!(metadata); |
| 182 | + } |
| 183 | +} |
0 commit comments