Add semver_ord(num) SQL function and version.semver_ord column

We currently rely on the Rust `semver` crate to implement our "sort by semantic versioning" functionality, that is used by web interface, but also to determine the "default version". This has the downside that we need to load the full list of version numbers for a crate from the database to the API server, sort it and then throw away the ones that we don't need.

This commit implements a `semver_ord(num)` pgSQL function that returns a JSONB array, which has the same ordering precedence as the Semantic Versioning spec (https://semver.org/#spec-item-11), with the small caveat that it only supports up to 15 prerelease parts. The maximum number of prerelease parts in our current dataset is 7, so 15 should be plenty.

The database migration in this commit also adds a new `semver_ord` column to the `versions` table, and an on-insert trigger function that automatically derives the `semver_ord` column from the `num` column value.

Once this migration has run, the existing versions can be backfilled by running the following SQL script, until all versions are processed:

```sql
with versions_to_update as (
    select id, num
    from versions
    where semver_ord = 'null'::jsonb
    limit 1000
)
update versions
    set semver_ord = semver_ord(num)
    where id in (select id from versions_to_update);
```

Author: Turbo87

crates/crates_io_database/tests/semver_ord.rs

@@ -0,0 +1,66 @@
+use crates_io_test_db::TestDatabase;
+use diesel::prelude::*;
+use diesel::sql_types::Text;
+use diesel_async::RunQueryDsl;
+use std::fmt::Debug;
+
+/// This test checks that the `semver_ord` function orders versions correctly.
+///
+/// The test data is a list of versions in a random order. The versions are then
+/// ordered by the `semver_ord` function and the result is compared to the
+/// expected order (see <https://semver.org/#spec-item-11>).
+///
+/// The test data was imported from <https://github.com/dtolnay/semver/blob/1.0.26/tests/test_version.rs#L223-L242>.
+#[tokio::test]
+async fn test_spec_order() {
+    let test_db = TestDatabase::new();
+    let mut conn = test_db.async_connect().await;
+
+    let query = r#"
+    with nums as (
+        select unnest(array[
+            '1.0.0-beta',
+            '1.0.0-alpha',
+            '1.0.0-rc.1',
+            '1.0.0',
+            '1.0.0-beta.2',
+            '1.0.0-alpha.1',
+            '1.0.0-alpha.beta',
+            '1.0.0-beta.11'
+        ]) as num
+    )
+    select num
+    from nums
+    order by semver_ord(num);
+    "#;
+
+    #[derive(QueryableByName)]
+    struct Row {
+        #[diesel(sql_type = Text)]
+        num: String,
+    }
+
+    impl Debug for Row {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.write_str(&self.num)
+        }
+    }
+
+    let nums = diesel::sql_query(query)
+        .load::<Row>(&mut conn)
+        .await
+        .unwrap();
+
+    insta::assert_debug_snapshot!(nums, @r"
+    [
+        1.0.0-alpha,
+        1.0.0-alpha.1,
+        1.0.0-alpha.beta,
+        1.0.0-beta,
+        1.0.0-beta.2,
+        1.0.0-beta.11,
+        1.0.0-rc.1,
+        1.0.0,
+    ]
+    ");
+}

migrations/2025-03-06-060640_semver_ord/down.sql

@@ -0,0 +1,4 @@
+drop trigger trigger_set_semver_ord on versions;
+drop function set_semver_ord();
+alter table versions drop column semver_ord;
+drop function semver_ord;

migrations/2025-03-06-060640_semver_ord/up.sql

@@ -0,0 +1,100 @@
+-- Add `semver_ord(num)` function to convert a semver string into a JSONB array for version comparison purposes.
+
+create or replace function semver_ord(num varchar) returns jsonb as $$
+declare
+    -- We need to ensure that the array has the same length for all versions
+    -- since shorter arrays have lower precedence in JSONB. Since we also need
+    -- to add a boolean value for each part of the prerelease string, this
+    -- results in us supporting up to 15 parts in the prerelease string.
+    -- Everything beyond that will be ignored.
+    prerelease_array_length constant int := 30;
+
+    -- We ignore the "build metadata" part of the semver string, since it has
+    -- no impact on the version ordering.
+    match_result text[] := regexp_match(num, '^(\d+).(\d+).(\d+)(?:-([0-9A-Za-z\-.]+))?');
+
+    prerelease jsonb;
+    prerelease_part text;
+    i int := 0;
+begin
+    if match_result[4] is null then
+        -- A JSONB object has higher precedence than an array, and versions with
+        -- prerelease specifiers should have lower precedence than those without.
+        prerelease := json_build_object();
+    else
+        prerelease := to_jsonb(array_fill(NULL::bool, ARRAY[prerelease_array_length]));
+
+        -- Split prerelease string by `.` and "append" items to
+        -- the `prerelease` array.
+        foreach prerelease_part in array string_to_array(match_result[4], '.')
+        loop
+            -- Parse parts as numbers if they consist of only digits.
+            if regexp_like(prerelease_part, '^\d+$') then
+                -- In JSONB a number has higher precedence than a string but in
+                -- semver it is the other way around, so we use true/false to
+                -- work around this.
+                prerelease := jsonb_set(prerelease, array[i::text], to_jsonb(false));
+                prerelease := jsonb_set(prerelease, array[(i + 1)::text], to_jsonb(prerelease_part::numeric));
+            else
+                prerelease := jsonb_set(prerelease, array[i::text], to_jsonb(true));
+                prerelease := jsonb_set(prerelease, array[(i + 1)::text], to_jsonb(prerelease_part));
+            end if;
+
+            -- Exit the loop if we have reached the maximum number of parts.
+            i := i + 2;
+            exit when i > prerelease_array_length;
+        end loop;
+    end if;
+
+    -- Return an array with the major, minor, patch, and prerelease parts.
+    return json_build_array(
+        match_result[1]::numeric,
+        match_result[2]::numeric,
+        match_result[3]::numeric,
+        prerelease
+    );
+end;
+$$ language plpgsql immutable;
+
+comment on function semver_ord is 'Converts a semver string into a JSONB array for version comparison purposes. The array has the following format: [major, minor, patch, prerelease] and when used for sorting follow the precedence rules defined in the semver specification (https://semver.org/#spec-item-11).';
+
+
+-- Add corresponding column to the `versions` table.
+
+alter table versions
+    add semver_ord jsonb default 'null'::jsonb not null;
+
+comment on column versions.semver_ord is 'JSONB representation of the version number for sorting purposes.';
+
+
+-- Create a trigger to set the `semver_ord` column when inserting a new version.
+-- Ideally, we would use a generated column for this, but introducing such a
+-- column would require a full table rewrite, which is not feasible for large
+-- tables.
+
+create or replace function set_semver_ord() returns trigger as $$
+begin
+    new.semver_ord := semver_ord(new.num);
+    return new;
+end
+$$ language plpgsql;
+
+create or replace trigger trigger_set_semver_ord
+    before insert on versions
+    for each row
+    execute procedure set_semver_ord();
+
+
+-- Populate the `semver_ord` column for existing versions.
+-- This query should be run manually in small batches to avoid locking the
+-- table for too long.
+
+-- with versions_to_update as (
+--     select id, num
+--     from versions
+--     where semver_ord = 'null'::jsonb
+--     limit 1000
+-- )
+-- update versions
+--     set semver_ord = semver_ord(num)
+--     where id in (select id from versions_to_update);