feat(semver): Add fields to Release to support semver filtering and sorting. (#25956)

This pr makes changes to the Release model so that we can support semver
sorting. The values for these fields will all be extracted from version if it fits our slightly
more permissive semver format:
<major>.<minor>.<patch>.<revision>-<prerelease>+<build_code>

These are all optional to varying extents. If a version does not match this format at all then
these fields will just be null. If we partially match, any fields not already included in the version
should be set to 0 or '' depending on field type. So 1.2 would have fields like
major=1, minor=2, patch=0, revision=0, prerelease=''. This is important, because we want to
use https://www.postgresql.org/docs/current/functions-comparisons.html#ROW-WISE-COMPARISON
to filter these rows, and nulls cause the comparison to return False where we wouldn't expect.

If build_code is entirely numeric we will parse it and store as build_number, which will allow
us to sort Android releases numerically.

I've added an index on (major, minor, patch, revision) with all the columns desc. We'll likely mostly be
sorting desc, so that seemed most helpful. This means that we'd sort in this order: 1.2.1, 1.2.0,
with 1.2.1 being the highest.

I've left prerelease out of the index, since I think that the number of prereleases per
patch or revision will be low, and it's probably nicer to keep the text field out of the index. To filter
correctly via semver, we need to add case when prerelease = '' then 1 else 0 end, prerelease to
our row comparison, so that rows without a prerelease end up being ranked as higher. This shouldn't
be slow, since most rows will have very few prereleases, but if we find that it is we can include these
fields in the index.

Author: wedamija

migrations_lockfile.txt

@@ -7,5 +7,5 @@ will then be regenerated, and you should be able to merge without conflicts.
 
 jira_ac: 0001_initial
 nodestore: 0002_nodestore_no_dictfield
-sentry: 0198_add_project_transaction_threshold
+sentry: 0199_release_semver
 social_auth: 0001_initial

src/sentry/migrations/0199_release_semver.py

@@ -0,0 +1,100 @@
+# Generated by Django 1.11.29 on 2021-05-20 20:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    # This flag is used to mark that a migration shouldn't be automatically run in
+    # production. We set this to True for operations that we think are risky and want
+    # someone from ops to run manually and monitor.
+    # General advice is that if in doubt, mark your migration as `is_dangerous`.
+    # Some things you should always mark as dangerous:
+    # - Large data migrations. Typically we want these to be run manually by ops so that
+    #   they can be monitored. Since data migrations will now hold a transaction open
+    #   this is even more important.
+    # - Adding columns to highly active tables, even ones that are NULL.
+    is_dangerous = True
+
+    # This flag is used to decide whether to run this migration in a transaction or not.
+    # By default we prefer to run in a transaction, but for migrations where you want
+    # to `CREATE INDEX CONCURRENTLY` this needs to be set to False. Typically you'll
+    # want to create an index concurrently when adding one to an existing table.
+    # You'll also usually want to set this to `False` if you're writing a data
+    # migration, since we don't want the entire migration to run in one long-running
+    # transaction.
+    atomic = False
+
+    dependencies = [
+        ("sentry", "0198_add_project_transaction_threshold"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="release",
+            name="build_code",
+            field=models.TextField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="build_number",
+            field=models.BigIntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="major",
+            field=models.BigIntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="minor",
+            field=models.BigIntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="patch",
+            field=models.BigIntegerField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="prerelease",
+            field=models.TextField(null=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="revision",
+            field=models.BigIntegerField(null=True),
+        ),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    """
+                    CREATE INDEX CONCURRENTLY IF NOT EXISTS "sentry_release_organization_id_major_mi_38715957_idx"
+                    ON "sentry_release" ("organization_id", "major" DESC, "minor" DESC, "patch" DESC, "revision" DESC);
+                    """,
+                    reverse_sql="DROP INDEX CONCURRENTLY IF EXISTS sentry_release_organization_id_major_mi_38715957_idx",
+                ),
+                migrations.RunSQL(
+                    """
+                    CREATE INDEX CONCURRENTLY IF NOT EXISTS "sentry_release_organization_id_build_code_f93815e5_idx" ON "sentry_release" ("organization_id", "build_code");
+                    """,
+                    reverse_sql="DROP INDEX CONCURRENTLY IF EXISTS sentry_release_organization_id_build_code_f93815e5_idx",
+                ),
+                migrations.RunSQL(
+                    """
+                    CREATE INDEX CONCURRENTLY IF NOT EXISTS "sentry_release_organization_id_build_number_e1646551_idx" ON "sentry_release" ("organization_id", "build_number");
+                    """,
+                    reverse_sql="DROP INDEX CONCURRENTLY IF EXISTS sentry_release_organization_id_build_number_e1646551_idx",
+                ),
+            ],
+            state_operations=[
+                migrations.AlterIndexTogether(
+                    name="release",
+                    index_together={
+                        ("organization", "build_code"),
+                        ("organization", "major", "minor", "patch", "revision"),
+                        ("organization", "build_number"),
+                    },
+                ),
+            ],
+        ),
+    ]

src/sentry/models/release.py

@@ -141,6 +141,19 @@ class Release(Model):
     total_deploys = BoundedPositiveIntegerField(null=True, default=0)
     last_deploy_id = BoundedPositiveIntegerField(null=True)
 
+    # Denormalized semver columns. These will be filled if `version` matches at least
+    # part of our more permissive model of semver:
+    # `<major>.<minor>.<patch>.<revision>-<prerelease>+<build_code>
+    major = models.BigIntegerField(null=True)
+    minor = models.BigIntegerField(null=True)
+    patch = models.BigIntegerField(null=True)
+    revision = models.BigIntegerField(null=True)
+    prerelease = models.TextField(null=True)
+    build_code = models.TextField(null=True)
+    # If `build_code` can be parsed as a 64 bit int we'll store it here as well for
+    # sorting/comparison purposes
+    build_number = models.BigIntegerField(null=True)
+
     # HACK HACK HACK
     # As a transitionary step we permit release rows to exist multiple times
     # where they are "specialized" for a specific project.  The goal is to
@@ -152,6 +165,14 @@ class Meta:
         app_label = "sentry"
         db_table = "sentry_release"
         unique_together = (("organization", "version"),)
+        # TODO(django2.2): Note that we create this index with each column ordered
+        # descending. Django 2.2 allows us to specify functional indexes, which should
+        # allow us to specify this on the model.
+        index_together = (
+            ("organization", "major", "minor", "patch", "revision"),
+            ("organization", "build_code"),
+            ("organization", "build_number"),
+        )
 
     __repr__ = sane_repr("organization_id", "version")