Merge pull request #9766 from Turbo87/num_no_build1

database: Add `versions.num_no_build` column

Author: Turbo87

crates/crates_io_database/src/schema.rs

@@ -989,6 +989,8 @@ diesel::table! {
         bin_names -> Nullable<Array<Nullable<Text>>>,
         /// message associated with a yanked version
         yank_message -> Nullable<Text>,
+        /// This is the same as `num` without the optional "build metadata" part (except for some versions that were published before we started validating this).
+        num_no_build -> Nullable<Varchar>,
     }
 }
 

crates/crates_io_database_dump/src/dump-db.toml

@@ -220,6 +220,7 @@ dependencies = ["crates", "users"]
 id = "public"
 crate_id = "public"
 num = "public"
+num_no_build = "public"
 updated_at = "public"
 created_at = "public"
 downloads = "public"

crates/crates_io_database_dump/src/snapshots/[email protected]

@@ -17,7 +17,7 @@ BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY;
     \copy "crates_keywords" ("crate_id", "keyword_id") TO 'data/crates_keywords.csv' WITH CSV HEADER
     \copy (SELECT "crate_id", "created_at", "created_by", "owner_id", "owner_kind" FROM "crate_owners" WHERE NOT deleted) TO 'data/crate_owners.csv' WITH CSV HEADER
 
-    \copy "versions" ("bin_names", "checksum", "crate_id", "crate_size", "created_at", "downloads", "features", "has_lib", "id", "license", "links", "num", "published_by", "rust_version", "updated_at", "yanked") TO 'data/versions.csv' WITH CSV HEADER
+    \copy "versions" ("bin_names", "checksum", "crate_id", "crate_size", "created_at", "downloads", "features", "has_lib", "id", "license", "links", "num", "num_no_build", "published_by", "rust_version", "updated_at", "yanked") TO 'data/versions.csv' WITH CSV HEADER
     \copy "default_versions" ("crate_id", "version_id") TO 'data/default_versions.csv' WITH CSV HEADER
     \copy "dependencies" ("crate_id", "default_features", "explicit_name", "features", "id", "kind", "optional", "req", "target", "version_id") TO 'data/dependencies.csv' WITH CSV HEADER
     \copy "version_downloads" ("date", "downloads", "version_id") TO 'data/version_downloads.csv' WITH CSV HEADER

crates/crates_io_database_dump/src/snapshots/[email protected]

@@ -59,7 +59,7 @@ BEGIN;
     \copy "crates_categories" ("category_id", "crate_id") FROM 'data/crates_categories.csv' WITH CSV HEADER
     \copy "crates_keywords" ("crate_id", "keyword_id") FROM 'data/crates_keywords.csv' WITH CSV HEADER
     \copy "crate_owners" ("crate_id", "created_at", "created_by", "owner_id", "owner_kind") FROM 'data/crate_owners.csv' WITH CSV HEADER
-    \copy "versions" ("bin_names", "checksum", "crate_id", "crate_size", "created_at", "downloads", "features", "has_lib", "id", "license", "links", "num", "published_by", "rust_version", "updated_at", "yanked") FROM 'data/versions.csv' WITH CSV HEADER
+    \copy "versions" ("bin_names", "checksum", "crate_id", "crate_size", "created_at", "downloads", "features", "has_lib", "id", "license", "links", "num", "num_no_build", "published_by", "rust_version", "updated_at", "yanked") FROM 'data/versions.csv' WITH CSV HEADER
     \copy "default_versions" ("crate_id", "version_id") FROM 'data/default_versions.csv' WITH CSV HEADER
     \copy "dependencies" ("crate_id", "default_features", "explicit_name", "features", "id", "kind", "optional", "req", "target", "version_id") FROM 'data/dependencies.csv' WITH CSV HEADER
     \copy "version_downloads" ("date", "downloads", "version_id") FROM 'data/version_downloads.csv' WITH CSV HEADER

migrations/2024-10-24-133507_add-num-no-build-column/down.sql

@@ -0,0 +1,2 @@
+alter table versions
+    drop num_no_build;

migrations/2024-10-24-133507_add-num-no-build-column/up.sql

@@ -0,0 +1,29 @@
+alter table versions
+    add num_no_build varchar;
+
+comment on column versions.num_no_build is 'This is the same as `num` without the optional "build metadata" part (except for some versions that were published before we started validating this).';
+
+-- to be run manually:
+
+-- update versions
+--     set num_no_build = split_part(num, '+', 1);
+--
+-- with duplicates as (
+--     -- find all versions that have the same `crate_id` and `num_no_build`
+--     select crate_id, num_no_build, array_agg(num ORDER BY id) as nums
+--     from versions
+--     group by crate_id, num_no_build
+--     having count(*) > 1
+-- ),
+-- duplicates_to_update as (
+--     -- for each group of duplicates, update all versions except the one that
+--     -- doesn't have "build metadata", or the first one that was published if
+--     -- all versions have "build metadata"
+--     select crate_id, num_no_build, unnest(case when array_position(nums, num_no_build) IS NOT NULL then array_remove(nums, num_no_build) else nums[2:] end) as num
+--     from duplicates
+-- )
+-- update versions
+--     set num_no_build = duplicates_to_update.num
+--     from duplicates_to_update
+--     where versions.crate_id = duplicates_to_update.crate_id
+--     and versions.num = duplicates_to_update.num;

src/models/version.rs

@@ -34,6 +34,7 @@ pub struct Version {
     pub has_lib: Option<bool>,
     pub bin_names: Option<Vec<Option<String>>>,
     pub yank_message: Option<String>,
+    pub num_no_build: Option<String>,
 }
 
 impl Version {