From 2511205fb830f1dd95b93459eaadb77dad095c52 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Wed, 5 Feb 2025 12:33:35 -0500 Subject: [PATCH 1/3] feat: update shared version --- requirements.in | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 667d55bbad..787bfdf546 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ freezegun google-cloud-pubsub gunicorn>=22.0.0 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem -https://github.com/codecov/shared/archive/7ba099fa0552244c77a8cc3a4b772216613c09c8.tar.gz#egg=shared +https://github.com/codecov/shared/archive/838bba5551758c1922e3328ea6b6e6ab6817c9ae.tar.gz#egg=shared https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz idna>=3.7 minio @@ -58,4 +58,4 @@ starlette==0.40.0 stripe>=11.4.1 urllib3>=1.26.19 vcrpy -whitenoise +whitenoise \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bcb25ecb15..760c3f2618 100644 --- a/requirements.txt +++ b/requirements.txt @@ -416,7 +416,7 @@ sentry-sdk[celery]==2.13.0 # shared setproctitle==1.1.10 # via -r requirements.in -shared @ https://github.com/codecov/shared/archive/7ba099fa0552244c77a8cc3a4b772216613c09c8.tar.gz +shared @ https://github.com/codecov/shared/archive/838bba5551758c1922e3328ea6b6e6ab6817c9ae.tar.gz # via -r requirements.in simplejson==3.17.2 # via -r requirements.in From cae56cd900706a0ee7dbf08f3af3b8432533dd43 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Wed, 5 Feb 2025 12:33:35 -0500 Subject: [PATCH 2/3] feat: move timeseries ORM related code to shared I want to move the models to shared because I want to be able to use the Django models in the worker. --- codecov/settings_base.py | 1 + timeseries/migrations/0001_initial.py | 47 ---- .../migrations/0002_continuous_aggregates.py | 37 --- timeseries/migrations/0003_cagg_policies.py | 24 -- .../migrations/0004_measurement_summaries.py | 84 ------- .../migrations/0005_uniqueness_constraints.py | 35 --- .../migrations/0006_auto_20220718_1311.py | 41 ---- .../migrations/0007_auto_20220727_2011.py | 45 ---- .../migrations/0008_auto_20220802_1838.py | 24 -- .../migrations/0009_auto_20220804_1305.py | 41 ---- .../migrations/0010_auto_20230123_1453.py | 23 -- .../0011_measurement_measurable_id.py | 17 -- .../migrations/0012_auto_20230501_1929.py | 16 -- .../0013_measurable_indexes_caggs.py | 219 ------------------ ...series_measurement_flag_unique_and_more.py | 43 ---- timeseries/migrations/__init__.py | 0 timeseries/models.py | 182 +-------------- timeseries/tests/factories.py | 29 +-- timeseries/tests/test_admin.py | 3 +- timeseries/tests/test_db.py | 31 --- timeseries/tests/test_helpers.py | 5 +- timeseries/tests/test_models.py | 132 ----------- 22 files changed, 8 insertions(+), 1071 deletions(-) delete mode 100644 timeseries/migrations/0001_initial.py delete mode 100644 timeseries/migrations/0002_continuous_aggregates.py delete mode 100644 timeseries/migrations/0003_cagg_policies.py delete mode 100644 timeseries/migrations/0004_measurement_summaries.py delete mode 100644 timeseries/migrations/0005_uniqueness_constraints.py delete mode 100644 timeseries/migrations/0006_auto_20220718_1311.py delete mode 100644 timeseries/migrations/0007_auto_20220727_2011.py delete mode 100644 timeseries/migrations/0008_auto_20220802_1838.py delete mode 100644 timeseries/migrations/0009_auto_20220804_1305.py delete mode 100644 timeseries/migrations/0010_auto_20230123_1453.py delete mode 100644 timeseries/migrations/0011_measurement_measurable_id.py delete mode 100644 timeseries/migrations/0012_auto_20230501_1929.py delete mode 100644 timeseries/migrations/0013_measurable_indexes_caggs.py delete mode 100644 timeseries/migrations/0014_remove_measurement_timeseries_measurement_flag_unique_and_more.py delete mode 100644 timeseries/migrations/__init__.py delete mode 100644 timeseries/tests/test_db.py delete mode 100644 timeseries/tests/test_models.py diff --git a/codecov/settings_base.py b/codecov/settings_base.py index 736d7c678d..b97571a57b 100644 --- a/codecov/settings_base.py +++ b/codecov/settings_base.py @@ -443,6 +443,7 @@ "profiling": "shared.django_apps.profiling.migrations", "reports": "shared.django_apps.reports.migrations", "staticanalysis": "shared.django_apps.staticanalysis.migrations", + "timeseries": "shared.django_apps.timeseries.migrations", } # to aid in debugging, print out this info on startup. If no license, prints nothing diff --git a/timeseries/migrations/0001_initial.py b/timeseries/migrations/0001_initial.py deleted file mode 100644 index 8a7e77314d..0000000000 --- a/timeseries/migrations/0001_initial.py +++ /dev/null @@ -1,47 +0,0 @@ -# Generated by Django 3.1.13 on 2022-05-23 20:35 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Measurement", - fields=[ - ("timestamp", models.DateTimeField(primary_key=True, serialize=False)), - ("owner_id", models.BigIntegerField()), - ("repo_id", models.BigIntegerField()), - ("flag_id", models.BigIntegerField(null=True)), - ("branch", models.TextField(null=True)), - ("commit_sha", models.TextField(null=True)), - ("name", models.TextField()), - ("value", models.FloatField()), - ], - ), - migrations.RunSQL( - "ALTER TABLE timeseries_measurement DROP CONSTRAINT timeseries_measurement_pkey;", - reverse_sql="", - ), - migrations.AddIndex( - model_name="measurement", - index=models.Index( - fields=[ - "owner_id", - "repo_id", - "flag_id", - "branch", - "name", - "timestamp", - ], - name="timeseries__owner_i_2cc713_idx", - ), - ), - migrations.RunSQL( - "SELECT create_hypertable('timeseries_measurement', 'timestamp');", - reverse_sql="", - ), - ] diff --git a/timeseries/migrations/0002_continuous_aggregates.py b/timeseries/migrations/0002_continuous_aggregates.py deleted file mode 100644 index 2cba7d1319..0000000000 --- a/timeseries/migrations/0002_continuous_aggregates.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 3.1.13 on 2022-05-23 20:46 - -from django.db import migrations - - -class Migration(migrations.Migration): - # cant create this views in a transaction - atomic = False - - dependencies = [ - ("timeseries", "0001_initial"), - ] - - operations = [ - migrations.RunSQL( - f""" - create materialized view timeseries_measurement_summary_{days}day - with (timescaledb.continuous) as - select - owner_id, - repo_id, - flag_id, - branch, - name, - time_bucket(interval '{days} days', timestamp) as timestamp_bin, - avg(value) as value_avg, - max(value) as value_max, - min(value) as value_min, - count(value) as value_count - from timeseries_measurement - group by - owner_id, repo_id, flag_id, branch, name, timestamp_bin; - """, - reverse_sql=f"drop materialized view timeseries_measurement_summary_{days}day;", - ) - for days in [1, 7, 30] - ] diff --git a/timeseries/migrations/0003_cagg_policies.py b/timeseries/migrations/0003_cagg_policies.py deleted file mode 100644 index 51d35611ec..0000000000 --- a/timeseries/migrations/0003_cagg_policies.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.1.13 on 2022-05-24 14:51 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0002_continuous_aggregates"), - ] - - operations = [ - migrations.RunSQL( - f""" - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_{name}', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '24 hours' - ); - """, - reverse_sql=f"select remove_continuous_aggregate_policy('timeseries_measurement_summary_{name}');", - ) - for name in ["1day", "7day", "30day"] - ] diff --git a/timeseries/migrations/0004_measurement_summaries.py b/timeseries/migrations/0004_measurement_summaries.py deleted file mode 100644 index c755684067..0000000000 --- a/timeseries/migrations/0004_measurement_summaries.py +++ /dev/null @@ -1,84 +0,0 @@ -# Generated by Django 3.1.13 on 2022-05-25 20:02 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0003_cagg_policies"), - ] - - operations = [ - migrations.CreateModel( - name="MeasurementSummary1Day", - fields=[ - ( - "timestamp_bin", - models.DateTimeField(primary_key=True, serialize=False), - ), - ("owner_id", models.BigIntegerField()), - ("repo_id", models.BigIntegerField()), - ("flag_id", models.BigIntegerField()), - ("branch", models.TextField()), - ("name", models.TextField()), - ("value_avg", models.FloatField()), - ("value_max", models.FloatField()), - ("value_min", models.FloatField()), - ("value_count", models.FloatField()), - ], - options={ - "db_table": "timeseries_measurement_summary_1day", - "ordering": ["timestamp_bin"], - "abstract": False, - "managed": False, - }, - ), - migrations.CreateModel( - name="MeasurementSummary30Day", - fields=[ - ( - "timestamp_bin", - models.DateTimeField(primary_key=True, serialize=False), - ), - ("owner_id", models.BigIntegerField()), - ("repo_id", models.BigIntegerField()), - ("flag_id", models.BigIntegerField()), - ("branch", models.TextField()), - ("name", models.TextField()), - ("value_avg", models.FloatField()), - ("value_max", models.FloatField()), - ("value_min", models.FloatField()), - ("value_count", models.FloatField()), - ], - options={ - "db_table": "timeseries_measurement_summary_30day", - "ordering": ["timestamp_bin"], - "abstract": False, - "managed": False, - }, - ), - migrations.CreateModel( - name="MeasurementSummary7Day", - fields=[ - ( - "timestamp_bin", - models.DateTimeField(primary_key=True, serialize=False), - ), - ("owner_id", models.BigIntegerField()), - ("repo_id", models.BigIntegerField()), - ("flag_id", models.BigIntegerField()), - ("branch", models.TextField()), - ("name", models.TextField()), - ("value_avg", models.FloatField()), - ("value_max", models.FloatField()), - ("value_min", models.FloatField()), - ("value_count", models.FloatField()), - ], - options={ - "db_table": "timeseries_measurement_summary_7day", - "ordering": ["timestamp_bin"], - "abstract": False, - "managed": False, - }, - ), - ] diff --git a/timeseries/migrations/0005_uniqueness_constraints.py b/timeseries/migrations/0005_uniqueness_constraints.py deleted file mode 100644 index 6aba3f03eb..0000000000 --- a/timeseries/migrations/0005_uniqueness_constraints.py +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by Django 3.1.13 on 2022-06-07 19:07 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0004_measurement_summaries"), - ] - - operations = [ - migrations.AddConstraint( - model_name="measurement", - constraint=models.UniqueConstraint( - condition=models.Q(flag_id__isnull=False), - fields=( - "name", - "owner_id", - "repo_id", - "flag_id", - "commit_sha", - "timestamp", - ), - name="timeseries_measurement_flag_unique", - ), - ), - migrations.AddConstraint( - model_name="measurement", - constraint=models.UniqueConstraint( - condition=models.Q(flag_id__isnull=True), - fields=("name", "owner_id", "repo_id", "commit_sha", "timestamp"), - name="timeseries_measurement_noflag_unique", - ), - ), - ] diff --git a/timeseries/migrations/0006_auto_20220718_1311.py b/timeseries/migrations/0006_auto_20220718_1311.py deleted file mode 100644 index 3008c36396..0000000000 --- a/timeseries/migrations/0006_auto_20220718_1311.py +++ /dev/null @@ -1,41 +0,0 @@ -# Generated by Django 3.2.12 on 2022-07-18 13:11 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0005_uniqueness_constraints"), - ] - - operations = [ - migrations.CreateModel( - name="Dataset", - fields=[ - ( - "id", - models.AutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.TextField()), - ("repository_id", models.IntegerField()), - ("backfilled", models.BooleanField(default=False)), - ], - ), - migrations.AddIndex( - model_name="dataset", - index=models.Index( - fields=["name", "repository_id"], name="timeseries__name_f96a15_idx" - ), - ), - migrations.AddConstraint( - model_name="dataset", - constraint=models.UniqueConstraint( - fields=("name", "repository_id"), name="name_repository_id_unique" - ), - ), - ] diff --git a/timeseries/migrations/0007_auto_20220727_2011.py b/timeseries/migrations/0007_auto_20220727_2011.py deleted file mode 100644 index 2697f871dd..0000000000 --- a/timeseries/migrations/0007_auto_20220727_2011.py +++ /dev/null @@ -1,45 +0,0 @@ -# Generated by Django 3.2.12 on 2022-07-27 20:11 - -from django.db import migrations, models - -import core.models - - -class Migration(migrations.Migration): - """ - BEGIN; - -- - -- Add field created_at to dataset - -- - ALTER TABLE "timeseries_dataset" ADD COLUMN "created_at" timestamp NULL; - -- - -- Add field updated_at to dataset - -- - ALTER TABLE "timeseries_dataset" ADD COLUMN "updated_at" timestamp NULL; - -- - -- Alter field id on dataset - -- - COMMIT; - """ - - dependencies = [ - ("timeseries", "0006_auto_20220718_1311"), - ] - - operations = [ - migrations.AddField( - model_name="dataset", - name="created_at", - field=core.models.DateTimeWithoutTZField(null=True), - ), - migrations.AddField( - model_name="dataset", - name="updated_at", - field=core.models.DateTimeWithoutTZField(null=True), - ), - migrations.AlterField( - model_name="dataset", - name="id", - field=models.AutoField(primary_key=True, serialize=False), - ), - ] diff --git a/timeseries/migrations/0008_auto_20220802_1838.py b/timeseries/migrations/0008_auto_20220802_1838.py deleted file mode 100644 index a717133089..0000000000 --- a/timeseries/migrations/0008_auto_20220802_1838.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.2.12 on 2022-08-02 18:38 - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0007_auto_20220727_2011"), - ] - - operations = [ - migrations.RunSQL( - f""" - select remove_continuous_aggregate_policy('timeseries_measurement_summary_{name}'); - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_{name}', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '1 h' - ); - """, - ) - for name in ["1day", "7day", "30day"] - ] diff --git a/timeseries/migrations/0009_auto_20220804_1305.py b/timeseries/migrations/0009_auto_20220804_1305.py deleted file mode 100644 index 561087a325..0000000000 --- a/timeseries/migrations/0009_auto_20220804_1305.py +++ /dev/null @@ -1,41 +0,0 @@ -# Generated by Django 3.2.12 on 2022-08-04 13:05 - -import datetime - -from django.db import migrations - -import core.models - - -class Migration(migrations.Migration): - """ - BEGIN; - -- - -- Alter field created_at on dataset - -- - -- - -- Alter field updated_at on dataset - -- - COMMIT; - """ - - dependencies = [ - ("timeseries", "0008_auto_20220802_1838"), - ] - - operations = [ - migrations.AlterField( - model_name="dataset", - name="created_at", - field=core.models.DateTimeWithoutTZField( - default=datetime.datetime.now, null=True - ), - ), - migrations.AlterField( - model_name="dataset", - name="updated_at", - field=core.models.DateTimeWithoutTZField( - default=datetime.datetime.now, null=True - ), - ), - ] diff --git a/timeseries/migrations/0010_auto_20230123_1453.py b/timeseries/migrations/0010_auto_20230123_1453.py deleted file mode 100644 index c725e7857a..0000000000 --- a/timeseries/migrations/0010_auto_20230123_1453.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.2.12 on 2023-01-23 14:53 - -from django.conf import settings -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0009_auto_20220804_1305"), - ] - - # disable real time aggregates - # https://docs.timescale.com/timescaledb/latest/how-to-guides/continuous-aggregates/real-time-aggregates/#real-time-aggregates - - operations = [ - migrations.RunSQL( - f""" - alter materialized view timeseries_measurement_summary_{name} set (timescaledb.materialized_only = true); - """, - ) - for name in ["1day", "7day", "30day"] - if not settings.TIMESERIES_REAL_TIME_AGGREGATES - ] diff --git a/timeseries/migrations/0011_measurement_measurable_id.py b/timeseries/migrations/0011_measurement_measurable_id.py deleted file mode 100644 index 8f21456df7..0000000000 --- a/timeseries/migrations/0011_measurement_measurable_id.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.1.7 on 2023-04-28 19:45 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0010_auto_20230123_1453"), - ] - - operations = [ - migrations.AddField( - model_name="measurement", - name="measurable_id", - field=models.TextField(null=True), - ), - ] diff --git a/timeseries/migrations/0012_auto_20230501_1929.py b/timeseries/migrations/0012_auto_20230501_1929.py deleted file mode 100644 index 29b208a919..0000000000 --- a/timeseries/migrations/0012_auto_20230501_1929.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 4.1.7 on 2023-05-01 19:29 - -from django.db import migrations -from shared.django_apps.migration_utils import RiskyRunSQL - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0011_measurement_measurable_id"), - ] - - operations = [ - RiskyRunSQL( - "update timeseries_measurement set measurable_id = case when name = 'coverage' then repo_id::text when name = 'flag_coverage' then flag_id::text end where measurable_id is null;" - ), - ] diff --git a/timeseries/migrations/0013_measurable_indexes_caggs.py b/timeseries/migrations/0013_measurable_indexes_caggs.py deleted file mode 100644 index 7bd9dcef7c..0000000000 --- a/timeseries/migrations/0013_measurable_indexes_caggs.py +++ /dev/null @@ -1,219 +0,0 @@ -# Generated by Django 4.1.7 on 2023-05-05 13:23 - -from django.conf import settings -from django.db import migrations, models -from shared.django_apps.migration_utils import RiskyAddConstraint, RiskyAddIndex - - -class Migration(migrations.Migration): - """ - BEGIN; - -- - -- Alter field measurable_id on measurement - -- - ALTER TABLE "timeseries_measurement" ALTER COLUMN "measurable_id" SET NOT NULL; - -- - -- Remove index timeseries__owner_i_2cc713_idx from measurement - -- - DROP INDEX IF EXISTS "timeseries__owner_i_2cc713_idx"; - -- - -- Create index timeseries__owner_i_08d6fe_idx on field(s) owner_id, repo_id, measurable_id, branch, name, timestamp of model measurement - -- - CREATE INDEX "timeseries__owner_i_08d6fe_idx" ON "timeseries_measurement" ("owner_id", "repo_id", "measurable_id", "branch", "name", "timestamp"); - -- - -- Create constraint timeseries_measurement_unique on model measurement - -- - ALTER TABLE "timeseries_measurement" ADD CONSTRAINT "timeseries_measurement_unique" UNIQUE ("name", "owner_id", "repo_id", "measurable_id", "commit_sha", "timestamp"); - -- - -- Raw SQL operation - -- - - drop materialized view timeseries_measurement_summary_1day; - create materialized view timeseries_measurement_summary_1day - with (timescaledb.continuous) as - select - owner_id, - repo_id, - measurable_id, - branch, - name, - time_bucket(interval '1 days', timestamp) as timestamp_bin, - avg(value) as value_avg, - max(value) as value_max, - min(value) as value_min, - count(value) as value_count - from timeseries_measurement - group by - owner_id, repo_id, measurable_id, branch, name, timestamp_bin - with no data; - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_1day', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '1 h' - ); - - -- - -- Raw SQL operation - -- - - drop materialized view timeseries_measurement_summary_7day; - create materialized view timeseries_measurement_summary_7day - with (timescaledb.continuous) as - select - owner_id, - repo_id, - measurable_id, - branch, - name, - time_bucket(interval '7 days', timestamp) as timestamp_bin, - avg(value) as value_avg, - max(value) as value_max, - min(value) as value_min, - count(value) as value_count - from timeseries_measurement - group by - owner_id, repo_id, measurable_id, branch, name, timestamp_bin - with no data; - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_7day', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '1 h' - ); - - -- - -- Raw SQL operation - -- - - drop materialized view timeseries_measurement_summary_30day; - create materialized view timeseries_measurement_summary_30day - with (timescaledb.continuous) as - select - owner_id, - repo_id, - measurable_id, - branch, - name, - time_bucket(interval '30 days', timestamp) as timestamp_bin, - avg(value) as value_avg, - max(value) as value_max, - min(value) as value_min, - count(value) as value_count - from timeseries_measurement - group by - owner_id, repo_id, measurable_id, branch, name, timestamp_bin - with no data; - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_30day', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '1 h' - ); - - -- - -- Raw SQL operation - -- - - alter materialized view timeseries_measurement_summary_1day set (timescaledb.materialized_only = true); - - -- - -- Raw SQL operation - -- - - alter materialized view timeseries_measurement_summary_7day set (timescaledb.materialized_only = true); - - -- - -- Raw SQL operation - -- - - alter materialized view timeseries_measurement_summary_30day set (timescaledb.materialized_only = true); - - COMMIT; - """ - - dependencies = [ - ("timeseries", "0012_auto_20230501_1929"), - ] - - operations = ( - [ - migrations.AlterField( - model_name="measurement", - name="measurable_id", - field=models.TextField(), - ), - migrations.RemoveIndex( - model_name="measurement", - name="timeseries__owner_i_2cc713_idx", - ), - RiskyAddIndex( - model_name="measurement", - index=models.Index( - fields=[ - "owner_id", - "repo_id", - "measurable_id", - "branch", - "name", - "timestamp", - ], - name="timeseries__owner_i_08d6fe_idx", - ), - ), - RiskyAddConstraint( - model_name="measurement", - constraint=models.UniqueConstraint( - fields=( - "name", - "owner_id", - "repo_id", - "measurable_id", - "commit_sha", - "timestamp", - ), - name="timeseries_measurement_unique", - ), - ), - ] - + [ - migrations.RunSQL( - f""" - drop materialized view timeseries_measurement_summary_{days}day; - create materialized view timeseries_measurement_summary_{days}day - with (timescaledb.continuous) as - select - owner_id, - repo_id, - measurable_id, - branch, - name, - time_bucket(interval '{days} days', timestamp) as timestamp_bin, - avg(value) as value_avg, - max(value) as value_max, - min(value) as value_min, - count(value) as value_count - from timeseries_measurement - group by - owner_id, repo_id, measurable_id, branch, name, timestamp_bin - with no data; - select add_continuous_aggregate_policy( - 'timeseries_measurement_summary_{days}day', - start_offset => NULL, - end_offset => NULL, - schedule_interval => INTERVAL '1 h' - ); - """ - ) - for days in [1, 7, 30] - ] - + [ - migrations.RunSQL( - f""" - alter materialized view timeseries_measurement_summary_{days}day set (timescaledb.materialized_only = true); - """ - ) - for days in [1, 7, 30] - if not settings.TIMESERIES_REAL_TIME_AGGREGATES - ] - ) diff --git a/timeseries/migrations/0014_remove_measurement_timeseries_measurement_flag_unique_and_more.py b/timeseries/migrations/0014_remove_measurement_timeseries_measurement_flag_unique_and_more.py deleted file mode 100644 index 27a4ec9dc6..0000000000 --- a/timeseries/migrations/0014_remove_measurement_timeseries_measurement_flag_unique_and_more.py +++ /dev/null @@ -1,43 +0,0 @@ -import django.utils.timezone -from django.db import migrations -from shared.django_apps.migration_utils import ( # Generated by Django 4.1.7 on 2023-05-15 20:46 - RiskyRemoveConstraint, - RiskyRemoveField, -) - -import core.models - - -class Migration(migrations.Migration): - dependencies = [ - ("timeseries", "0013_measurable_indexes_caggs"), - ] - - operations = [ - RiskyRemoveConstraint( - model_name="measurement", - name="timeseries_measurement_flag_unique", - ), - RiskyRemoveConstraint( - model_name="measurement", - name="timeseries_measurement_noflag_unique", - ), - RiskyRemoveField( - model_name="measurement", - name="flag_id", - ), - migrations.AlterField( - model_name="dataset", - name="created_at", - field=core.models.DateTimeWithoutTZField( - default=django.utils.timezone.now, null=True - ), - ), - migrations.AlterField( - model_name="dataset", - name="updated_at", - field=core.models.DateTimeWithoutTZField( - default=django.utils.timezone.now, null=True - ), - ), - ] diff --git a/timeseries/migrations/__init__.py b/timeseries/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/timeseries/models.py b/timeseries/models.py index c0ca2d95d6..b4f7267db5 100644 --- a/timeseries/models.py +++ b/timeseries/models.py @@ -1,181 +1 @@ -from datetime import datetime, timedelta -from enum import Enum - -import django.db.models as models -from django.utils import timezone -from django_prometheus.models import ExportModelOperationsMixin - -from core.models import DateTimeWithoutTZField - - -class Interval(Enum): - INTERVAL_1_DAY = 1 - INTERVAL_7_DAY = 7 - INTERVAL_30_DAY = 30 - - -class MeasurementName(Enum): - COVERAGE = "coverage" - FLAG_COVERAGE = "flag_coverage" - COMPONENT_COVERAGE = "component_coverage" - # For tracking the entire size of a bundle report by its name - BUNDLE_ANALYSIS_REPORT_SIZE = "bundle_analysis_report_size" - # For tracking the size of a category of assets of a bundle report by its name - BUNDLE_ANALYSIS_JAVASCRIPT_SIZE = "bundle_analysis_javascript_size" - BUNDLE_ANALYSIS_STYLESHEET_SIZE = "bundle_analysis_stylesheet_size" - BUNDLE_ANALYSIS_FONT_SIZE = "bundle_analysis_font_size" - BUNDLE_ANALYSIS_IMAGE_SIZE = "bundle_analysis_image_size" - # For tracking individual asset size via its UUID - BUNDLE_ANALYSIS_ASSET_SIZE = "bundle_analysis_asset_size" - - -class Measurement(ExportModelOperationsMixin("timeseries.measurement"), models.Model): - # TimescaleDB requires that `timestamp` be part of every index (since data is - # partitioned by `timestamp`). Since an auto-incrementing primary key would - # not satisfy this requirement we can make `timestamp` the primary key. - # `timestamp` may not be unique though so we drop the uniqueness constraint in - # a migration. - timestamp = models.DateTimeField(null=False, primary_key=True) - - owner_id = models.BigIntegerField(null=False) - repo_id = models.BigIntegerField(null=False) - measurable_id = models.TextField(null=False) - branch = models.TextField(null=True) - - # useful for updating a measurement if needed - commit_sha = models.TextField(null=True) - - # the name of the measurement (i.e. "coverage") - name = models.TextField(null=False, blank=False) - value = models.FloatField(null=False) - - class Meta: - indexes = [ - # for querying measurements - models.Index( - fields=[ - "owner_id", - "repo_id", - "measurable_id", - "branch", - "name", - "timestamp", - ] - ), - ] - constraints = [ - # for updating measurements - models.UniqueConstraint( - fields=[ - "name", - "owner_id", - "repo_id", - "measurable_id", - "commit_sha", - "timestamp", - ], - name="timeseries_measurement_unique", - ), - ] - - -class MeasurementSummary( - ExportModelOperationsMixin("timeseries.measurement_summary"), models.Model -): - timestamp_bin = models.DateTimeField(primary_key=True) - owner_id = models.BigIntegerField() - repo_id = models.BigIntegerField() - measurable_id = models.TextField() - branch = models.TextField() - name = models.TextField() - value_avg = models.FloatField() - value_max = models.FloatField() - value_min = models.FloatField() - value_count = models.FloatField() - - @classmethod - def agg_by(cls, interval: Interval) -> models.Manager: - model_classes = { - Interval.INTERVAL_1_DAY: MeasurementSummary1Day, - Interval.INTERVAL_7_DAY: MeasurementSummary7Day, - Interval.INTERVAL_30_DAY: MeasurementSummary30Day, - } - - model_class = model_classes.get(interval) - if not model_class: - raise ValueError(f"cannot aggregate by '{interval}'") - return model_class.objects - - class Meta: - abstract = True - # these are backed by TimescaleDB "continuous aggregates" - # (materialized views) - managed = False - ordering = ["timestamp_bin"] - - -class MeasurementSummary1Day(MeasurementSummary): - class Meta(MeasurementSummary.Meta): - db_table = "timeseries_measurement_summary_1day" - - -# Timescale's origin for time buckets is Monday 2000-01-03 -# Weekly aggregate bins will thus be Monday-Sunday -class MeasurementSummary7Day(MeasurementSummary): - class Meta(MeasurementSummary.Meta): - db_table = "timeseries_measurement_summary_7day" - - -# Timescale's origin for time buckets is 2000-01-03 -# 30 day offsets will be aligned on that origin -class MeasurementSummary30Day(MeasurementSummary): - class Meta(MeasurementSummary.Meta): - db_table = "timeseries_measurement_summary_30day" - - -class Dataset(ExportModelOperationsMixin("timeseries.dataset"), models.Model): - id = models.AutoField(primary_key=True) - - # this will likely correspond to a measurement name above - name = models.TextField(null=False, blank=False) - - # not a true foreign key since repositories are in a - # different database - repository_id = models.IntegerField(null=False) - - # indicates whether the backfill task has completed for this dataset - # TODO: We're not really using this field anymore as a backfill task takes very long for this to be populated when finished. - # The solution would be to somehow have a celery task return when it's done, hence the TODO - backfilled = models.BooleanField(null=False, default=False) - - created_at = DateTimeWithoutTZField(default=timezone.now, null=True) - updated_at = DateTimeWithoutTZField(default=timezone.now, null=True) - - class Meta: - indexes = [ - models.Index( - fields=[ - "name", - "repository_id", - ] - ), - ] - constraints = [ - models.UniqueConstraint( - fields=[ - "name", - "repository_id", - ], - name="name_repository_id_unique", - ), - ] - - def is_backfilled(self) -> bool: - """ - Returns `False` for an hour after creation. - - TODO: this should eventually read `self.backfilled` which will be updated via the worker - """ - if not self.created_at: - return False - return datetime.now() > self.created_at + timedelta(hours=1) +from shared.django_apps.timeseries.models import * diff --git a/timeseries/tests/factories.py b/timeseries/tests/factories.py index 157dba715e..7e4b78ffcd 100644 --- a/timeseries/tests/factories.py +++ b/timeseries/tests/factories.py @@ -1,28 +1 @@ -import random -from datetime import datetime - -import factory -from factory.django import DjangoModelFactory - -from timeseries import models - - -class MeasurementFactory(DjangoModelFactory): - class Meta: - model = models.Measurement - - owner_id = 1 - repo_id = 1 - name = "testing" - branch = "master" - value = factory.LazyAttribute(lambda _: random.random() * 1000) - timestamp = factory.LazyAttribute(lambda _: datetime.now()) - - -class DatasetFactory(DjangoModelFactory): - class Meta: - model = models.Dataset - - repository_id = 1 - name = "testing" - backfilled = False +from shared.django_apps.timeseries.tests.factories import * diff --git a/timeseries/tests/test_admin.py b/timeseries/tests/test_admin.py index fc28ef489c..0387fe4985 100644 --- a/timeseries/tests/test_admin.py +++ b/timeseries/tests/test_admin.py @@ -8,8 +8,7 @@ from django.utils import timezone from shared.django_apps.codecov_auth.tests.factories import UserFactory from shared.django_apps.core.tests.factories import RepositoryFactory - -from timeseries.tests.factories import DatasetFactory +from shared.django_apps.timeseries.tests.factories import DatasetFactory @pytest.mark.skipif( diff --git a/timeseries/tests/test_db.py b/timeseries/tests/test_db.py deleted file mode 100644 index 66d845e01e..0000000000 --- a/timeseries/tests/test_db.py +++ /dev/null @@ -1,31 +0,0 @@ -from unittest.mock import patch - -import pytest -from django.conf import settings -from django.db import connections -from django.test import TransactionTestCase - - -@pytest.mark.skipif( - not settings.TIMESERIES_ENABLED, reason="requires timeseries data storage" -) -class DatabaseTests(TransactionTestCase): - databases = {"timeseries"} - - @patch("django.db.backends.postgresql.base.DatabaseWrapper.is_usable") - def test_db_reconnect(self, is_usable): - timeseries_database_engine = settings.DATABASES["timeseries"]["ENGINE"] - settings.DATABASES["timeseries"]["ENGINE"] = "codecov.db" - - is_usable.return_value = True - - with connections["timeseries"].cursor() as cursor: - cursor.execute("SELECT 1") - - is_usable.return_value = False - - # it should reconnect and not raise an error - with connections["timeseries"].cursor() as cursor: - cursor.execute("SELECT 1") - - settings.DATABASES["timeseries"]["ENGINE"] = timeseries_database_engine diff --git a/timeseries/tests/test_helpers.py b/timeseries/tests/test_helpers.py index 90a67e1f46..2762ceb085 100644 --- a/timeseries/tests/test_helpers.py +++ b/timeseries/tests/test_helpers.py @@ -11,6 +11,10 @@ OwnerFactory, RepositoryFactory, ) +from shared.django_apps.timeseries.tests.factories import ( + DatasetFactory, + MeasurementFactory, +) from shared.reports.resources import Report, ReportFile, ReportLine from shared.utils.sessions import Session @@ -22,7 +26,6 @@ repository_coverage_measurements_with_fallback, ) from timeseries.models import Dataset, Interval, MeasurementName -from timeseries.tests.factories import DatasetFactory, MeasurementFactory def sample_report(): diff --git a/timeseries/tests/test_models.py b/timeseries/tests/test_models.py deleted file mode 100644 index 14003b2696..0000000000 --- a/timeseries/tests/test_models.py +++ /dev/null @@ -1,132 +0,0 @@ -from datetime import datetime, timezone - -import pytest -from django.conf import settings -from django.test import TransactionTestCase -from freezegun import freeze_time - -from timeseries.models import Dataset, Interval, MeasurementSummary - -from .factories import DatasetFactory, MeasurementFactory - - -@pytest.mark.skipif( - not settings.TIMESERIES_ENABLED, reason="requires timeseries data storage" -) -class MeasurementTests(TransactionTestCase): - databases = {"timeseries"} - - def test_measurement_agg_1day(self): - MeasurementFactory( - timestamp=datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc), value=1 - ) - MeasurementFactory( - timestamp=datetime(2022, 1, 1, 1, 0, 0, tzinfo=timezone.utc), value=2 - ) - MeasurementFactory( - timestamp=datetime(2022, 1, 1, 1, 0, 1, tzinfo=timezone.utc), value=3 - ) - MeasurementFactory( - timestamp=datetime(2022, 1, 2, 0, 0, 0, tzinfo=timezone.utc), value=4 - ) - MeasurementFactory( - timestamp=datetime(2022, 1, 2, 0, 1, 0, tzinfo=timezone.utc), value=5 - ) - - results = MeasurementSummary.agg_by(Interval.INTERVAL_1_DAY).all() - - assert len(results) == 2 - assert results[0].value_avg == 2 - assert results[0].value_min == 1 - assert results[0].value_max == 3 - assert results[0].value_count == 3 - assert results[1].value_avg == 4.5 - assert results[1].value_min == 4 - assert results[1].value_max == 5 - assert results[1].value_count == 2 - - def test_measurement_agg_7day(self): - # Week 1: Monday, Tuesday, Sunday - MeasurementFactory(timestamp=datetime(2022, 1, 3), value=1) - MeasurementFactory(timestamp=datetime(2022, 1, 4), value=2) - MeasurementFactory(timestamp=datetime(2022, 1, 9), value=3) - - # Week 2: Monday, Sunday - MeasurementFactory(timestamp=datetime(2022, 1, 10), value=4) - MeasurementFactory(timestamp=datetime(2022, 1, 16), value=5) - - results = MeasurementSummary.agg_by(Interval.INTERVAL_7_DAY).all() - - assert len(results) == 2 - assert results[0].value_avg == 2 - assert results[0].value_min == 1 - assert results[0].value_max == 3 - assert results[0].value_count == 3 - assert results[1].value_avg == 4.5 - assert results[1].value_min == 4 - assert results[1].value_max == 5 - assert results[1].value_count == 2 - - def test_measurement_agg_30day(self): - # Timescale's origin for time buckets is 2000-01-03 - # 30 day offsets will be aligned on that origin - - MeasurementFactory(timestamp=datetime(2000, 1, 3), value=1) - MeasurementFactory(timestamp=datetime(2000, 1, 4), value=2) - MeasurementFactory(timestamp=datetime(2000, 2, 1), value=3) - - MeasurementFactory(timestamp=datetime(2000, 2, 2), value=4) - MeasurementFactory(timestamp=datetime(2000, 2, 11), value=5) - - results = MeasurementSummary.agg_by(Interval.INTERVAL_30_DAY).all() - - assert len(results) == 2 - assert results[0].value_avg == 2 - assert results[0].value_min == 1 - assert results[0].value_max == 3 - assert results[0].value_count == 3 - assert results[1].value_avg == 4.5 - assert results[1].value_min == 4 - assert results[1].value_max == 5 - assert results[1].value_count == 2 - - def test_measurement_agg_invalid(self): - with self.assertRaises(ValueError): - MeasurementSummary.agg_by("invalid").all() - - -@pytest.mark.skipif( - not settings.TIMESERIES_ENABLED, reason="requires timeseries data storage" -) -class DatasetTests(TransactionTestCase): - databases = {"timeseries"} - - @freeze_time("2022-01-01T01:00:01+0000") - def test_is_backfilled_true(self): - dataset = DatasetFactory() - - Dataset.objects.filter(pk=dataset.pk).update( - created_at=datetime(2022, 1, 1, 0, 0, 0) - ) - - dataset.refresh_from_db() - assert dataset.is_backfilled() == True - - @freeze_time("2022-01-01T00:59:59+0000") - def test_is_backfilled_false(self): - dataset = DatasetFactory() - - Dataset.objects.filter(pk=dataset.pk).update( - created_at=datetime(2022, 1, 1, 0, 0, 0) - ) - - dataset.refresh_from_db() - assert dataset.is_backfilled() == False - - def test_is_backfilled_no_created_at(self): - dataset = DatasetFactory() - - Dataset.objects.filter(pk=dataset.pk).update(created_at=None) - - dataset.refresh_from_db() - assert dataset.is_backfilled() == False From 040c20f759225182c5ce7da1aabbb9a45ad55fd7 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Fri, 7 Feb 2025 09:24:11 -0500 Subject: [PATCH 3/3] update shared to include latest commit --- requirements.in | 2 +- requirements.txt | 29 +++++++++++++---------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/requirements.in b/requirements.in index 787bfdf546..3cc80afdca 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ freezegun google-cloud-pubsub gunicorn>=22.0.0 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem -https://github.com/codecov/shared/archive/838bba5551758c1922e3328ea6b6e6ab6817c9ae.tar.gz#egg=shared +https://github.com/codecov/shared/archive/9235672f89d2810fcf8c37905f0645f8ac77f4d4.tar.gz#egg=shared https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz idna>=3.7 minio diff --git a/requirements.txt b/requirements.txt index 760c3f2618..0f53fce64e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile requirements.in -# +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt aiodataloader==0.4.0 # via -r requirements.in amplitude-analytics==1.1.4 @@ -88,11 +84,11 @@ click-repl==0.2.0 # via celery codecov-ribs==0.1.18 # via shared -codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz +codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem # via -r requirements.in colour==0.1.5 # via shared -coverage[toml]==7.5.1 +coverage==7.5.1 # via # codecovopentelem # pytest-cov @@ -170,7 +166,7 @@ filelock==3.0.12 # via virtualenv freezegun==1.1.0 # via -r requirements.in -google-api-core[grpc]==2.23.0 +google-api-core==2.23.0 # via # google-cloud-core # google-cloud-pubsub @@ -195,7 +191,7 @@ google-crc32c==1.0.0 # google-resumable-media google-resumable-media==2.7.2 # via google-cloud-storage -googleapis-common-protos[grpc]==1.59.1 +googleapis-common-protos==1.59.1 # via # google-api-core # grpc-google-iam-v1 @@ -404,19 +400,23 @@ requests==2.32.3 # google-cloud-storage # shared # stripe -rfc3986[idna2008]==1.4.0 +rfc3986==1.4.0 # via httpx rsa==4.7.2 # via google-auth s3transfer==0.5.0 # via boto3 -sentry-sdk[celery]==2.13.0 +sentry-sdk==2.13.0 # via # -r requirements.in # shared setproctitle==1.1.10 # via -r requirements.in -shared @ https://github.com/codecov/shared/archive/838bba5551758c1922e3328ea6b6e6ab6817c9ae.tar.gz +setuptools==75.8.0 + # via + # ddtrace + # opentelemetry-instrumentation +shared @ https://github.com/codecov/shared/archive/9235672f89d2810fcf8c37905f0645f8ac77f4d4.tar.gz#egg=shared # via -r requirements.in simplejson==3.17.2 # via -r requirements.in @@ -499,6 +499,3 @@ zipp==3.19.2 # via importlib-metadata zstandard==0.23.0 # via shared - -# The following packages are considered to be unsafe in a requirements file: -# setuptools