From 4d1167b841a39afc21b27e36772430007ca3af94 Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Fri, 31 Jan 2025 15:26:54 -0800 Subject: [PATCH 1/6] change to update or create from create --- core/management/commands/insert_data_to_db_from_csv.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/management/commands/insert_data_to_db_from_csv.py b/core/management/commands/insert_data_to_db_from_csv.py index cc88388d0e..8a6dcea30d 100644 --- a/core/management/commands/insert_data_to_db_from_csv.py +++ b/core/management/commands/insert_data_to_db_from_csv.py @@ -51,8 +51,14 @@ def handle(self, *args, **kwargs): ) continue - Model.objects.create(**model_data) - self.stdout.write(self.style.SUCCESS(f"Inserted row: {row}")) + # Use update_or_create + Model.objects.update_or_create( + defaults=model_data, + # Specify the fields to identify the record + # For example, if 'id' is the unique field: + id=row.get("id"), + ) + self.stdout.write(self.style.SUCCESS(f"Processed row: {row}")) self.stdout.write( self.style.SUCCESS( From 7ca85f37e1955f1cfc9b1bca0de1e9727a3c5201 Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Thu, 6 Feb 2025 13:02:10 -0800 Subject: [PATCH 2/6] add commands for seeding DB --- core/management/commands/insert_data_to_db_from_csv.py | 3 +-- dev.sh | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/management/commands/insert_data_to_db_from_csv.py b/core/management/commands/insert_data_to_db_from_csv.py index 8a6dcea30d..9005e6c778 100644 --- a/core/management/commands/insert_data_to_db_from_csv.py +++ b/core/management/commands/insert_data_to_db_from_csv.py @@ -51,14 +51,13 @@ def handle(self, *args, **kwargs): ) continue - # Use update_or_create Model.objects.update_or_create( defaults=model_data, # Specify the fields to identify the record # For example, if 'id' is the unique field: id=row.get("id"), ) - self.stdout.write(self.style.SUCCESS(f"Processed row: {row}")) + self.stdout.write(self.style.SUCCESS(f"Inserted row: {row}")) self.stdout.write( self.style.SUCCESS( diff --git a/dev.sh b/dev.sh index be4b78ffe0..0bcdadd9d7 100755 --- a/dev.sh +++ b/dev.sh @@ -2,7 +2,7 @@ # starts the development server using gunicorn # NEVER run production with the --reload option command -echo "Starting gunicorn in dev mode" +echo "API: Starting gunicorn in dev mode" _start_gunicorn() { if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then @@ -19,6 +19,8 @@ _start_gunicorn() { python manage.py migrate python manage.py migrate --database "timeseries" timeseries python manage.py pgpartition --yes --skip-delete + python manage.py insert_data_to_db_from_csv core/management/commands/codecovTiers-Jan25.csv --model tiers + python manage.py insert_data_to_db_from_csv core/management/commands/codecovPlans-Jan25.csv --model plans fi if [[ "$DEBUGPY" ]]; then pip install debugpy From 63bc4525eeb14d208c44fe54d86e8d33f9a8bdc0 Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Thu, 6 Feb 2025 13:03:58 -0800 Subject: [PATCH 3/6] remove the startup command from app --- core/apps.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/core/apps.py b/core/apps.py index c23030afc7..4efb124f0f 100644 --- a/core/apps.py +++ b/core/apps.py @@ -1,7 +1,6 @@ import logging from django.apps import AppConfig -from django.core.management import call_command from shared.helpers.cache import RedisBackend from services.redis_configuration import get_redis_connection @@ -17,24 +16,6 @@ class CoreConfig(AppConfig): def ready(self): import core.signals # noqa: F401 - if RUN_ENV == "DEV": - try: - # Call your management command here - call_command( - "insert_data_to_db_from_csv", - "core/management/commands/codecovTiers-Jan25.csv", - "--model", - "tiers", - ) - call_command( - "insert_data_to_db_from_csv", - "core/management/commands/codecovPlans-Jan25.csv", - "--model", - "plans", - ) - except Exception as e: - logger.error(f"Failed to run startup command: {e}") - if RUN_ENV not in ["DEV", "TESTING"]: cache_backend = RedisBackend(get_redis_connection()) cache.configure(cache_backend) From 635f7f6813bd0f9aa0af5bebe093c859d22ae059 Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Thu, 6 Feb 2025 13:16:35 -0800 Subject: [PATCH 4/6] update stripe_ids to stage values --- core/management/commands/codecovPlans-Jan25.csv | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/management/commands/codecovPlans-Jan25.csv b/core/management/commands/codecovPlans-Jan25.csv index 083550a76f..9e3eae9597 100644 --- a/core/management/commands/codecovPlans-Jan25.csv +++ b/core/management/commands/codecovPlans-Jan25.csv @@ -2,12 +2,12 @@ 10,2025-01-16 04:40:55.162 -0800,2025-01-24 11:33:46.043 -0800,0,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories""}",,true,Developer,,,false,users-trial,6, 9,2025-01-16 04:39:59.759 -0800,2025-01-24 11:34:10.038 -0800,10,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",annually,true,Enterprise Cloud,,,true,users-enterprisey,4,price_1LmjzwGlVGuVgOrkIwlM46EU 8,2025-01-16 04:39:15.877 -0800,2025-01-24 11:34:31.904 -0800,12,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",monthly,true,Enterprise Cloud,,,true,users-enterprisem,4,price_1LmjypGlVGuVgOrkzKtNqhwW -7,2025-01-16 04:38:12.544 -0800,2025-01-24 11:34:53.935 -0800,4,"{""Up to 10 users"",""Unlimited repositories"",""2500 private repo uploads"",""Patch coverage analysis""}",annually,true,Team,10,2500,true,users-teamy,2,price_1NrlXiGlVGuVgOrkgMTw5yno -6,2025-01-16 04:37:08.918 -0800,2025-01-24 11:35:15.346 -0800,5,"{""Up to 10 users"",""Unlimited repositories"",""2500 private repo uploads"",""Patch coverage analysis""}",monthly,true,Team,10,2500,true,users-teamm,2,price_1NqPKdGlVGuVgOrkm9OFvtz8 -5,2025-01-16 04:35:34.152 -0800,2025-01-24 11:35:42.724 -0800,10,"{""Includes 5 seats"",""$10 per additional seat"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",annually,true,Sentry Pro,5,,true,users-sentryy,5,price_1MlYAYGlVGuVgOrke9SdbBUn -4,2025-01-16 04:34:33.867 -0800,2025-01-24 11:35:48.218 -0800,12,"{""Includes 5 seats"",""$10 per additional seat"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",monthly,true,Sentry Pro,5,,true,users-sentrym,5,price_1MlY9yGlVGuVgOrkHluurBtJ -3,2025-01-16 04:32:44.655 -0800,2025-01-24 11:36:09.660 -0800,10,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",annually,true,Pro,,,true,users-pr-inappy,3,price_1Gv2COGlVGuVgOrkuOYVLIj7 -2,2025-01-16 04:30:42.897 -0800,2025-01-24 11:36:14.651 -0800,12,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",monthly,true,Pro,,,true,users-pr-inappm,3,price_1Gv2B8GlVGuVgOrkFnLunCgc +7,2025-01-16 04:38:12.544 -0800,2025-01-24 11:34:53.935 -0800,4,"{""Up to 10 users"",""Unlimited repositories"",""2500 private repo uploads"",""Patch coverage analysis""}",annually,true,Team,10,2500,true,users-teamy,2,price_1OCM2cGlVGuVgOrkMWUFjPFz +6,2025-01-16 04:37:08.918 -0800,2025-01-24 11:35:15.346 -0800,5,"{""Up to 10 users"",""Unlimited repositories"",""2500 private repo uploads"",""Patch coverage analysis""}",monthly,true,Team,10,2500,true,users-teamm,2,price_1OCM0gGlVGuVgOrkWDYEBtSL +5,2025-01-16 04:35:34.152 -0800,2025-01-24 11:35:42.724 -0800,10,"{""Includes 5 seats"",""$10 per additional seat"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",annually,true,Sentry Pro,5,,true,users-sentryy,5,price_1Mj1mMGlVGuVgOrkC0ORc6iW +4,2025-01-16 04:34:33.867 -0800,2025-01-24 11:35:48.218 -0800,12,"{""Includes 5 seats"",""$10 per additional seat"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",monthly,true,Sentry Pro,5,,true,users-sentrym,5,price_1Mj1kYGlVGuVgOrk7jucaZAa +3,2025-01-16 04:32:44.655 -0800,2025-01-24 11:36:09.660 -0800,10,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",annually,true,Pro,,,true,users-pr-inappy,3,plan_H6P16wij3lUuxg +2,2025-01-16 04:30:42.897 -0800,2025-01-24 11:36:14.651 -0800,12,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories"",""Priority Support""}",monthly,true,Pro,,,true,users-pr-inappm,3,plan_H6P3KZXwmAbqPS 13,2025-01-23 14:25:04.793 -0800,2025-01-23 14:25:04.793 -0800,12,"{""Configurable # of users"",""Unlimited public repositories"",""Unlimited private repositories""}",,true,Github Marketplace,,,false,users,3, 12,2025-01-16 04:44:51.064 -0800,2025-01-24 11:33:14.405 -0800,0,"{""Up to 1 user"",""Unlimited public repositories"",""Unlimited private repositories""}",,true,Developer,1,250,false,users-developer,2, 11,2025-01-16 04:44:01.249 -0800,2025-01-24 11:33:28.532 -0800,0,"{""Up to 1 user"",""Unlimited public repositories"",""Unlimited private repositories""}",,true,Developer,1,250,false,users-basic,1, From bdf81a3990eaeadd7adbfd22f7750990090e61dc Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Thu, 6 Feb 2025 14:32:01 -0800 Subject: [PATCH 5/6] catch each row that fails --- .../commands/insert_data_to_db_from_csv.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/management/commands/insert_data_to_db_from_csv.py b/core/management/commands/insert_data_to_db_from_csv.py index 9005e6c778..122f714a43 100644 --- a/core/management/commands/insert_data_to_db_from_csv.py +++ b/core/management/commands/insert_data_to_db_from_csv.py @@ -51,13 +51,15 @@ def handle(self, *args, **kwargs): ) continue - Model.objects.update_or_create( - defaults=model_data, - # Specify the fields to identify the record - # For example, if 'id' is the unique field: - id=row.get("id"), - ) - self.stdout.write(self.style.SUCCESS(f"Inserted row: {row}")) + try: + Model.objects.update_or_create( + defaults=model_data, + id=row.get("id"), + ) + self.stdout.write(self.style.SUCCESS(f"Inserted row: {row}")) + except Exception as e: + self.stdout.write(self.style.ERROR(f"Error inserting row: {e}")) + continue self.stdout.write( self.style.SUCCESS( From 0c6c0fcfb8761e55a8c925e6b4f849bfa1f64ccc Mon Sep 17 00:00:00 2001 From: Ajay Singh Date: Fri, 7 Feb 2025 10:30:49 -0800 Subject: [PATCH 6/6] update shared hash --- requirements.in | 2 +- requirements.txt | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/requirements.in b/requirements.in index 3cc80afdca..cbb170b2f6 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ freezegun google-cloud-pubsub gunicorn>=22.0.0 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem -https://github.com/codecov/shared/archive/9235672f89d2810fcf8c37905f0645f8ac77f4d4.tar.gz#egg=shared +https://github.com/codecov/shared/archive/abf6180de5cee3ac99b7fbf909bd82c7325063fd.tar.gz#egg=shared https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz idna>=3.7 minio diff --git a/requirements.txt b/requirements.txt index 0f53fce64e..0d2dc25a9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,9 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile requirements.in -o requirements.txt +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements.in +# aiodataloader==0.4.0 # via -r requirements.in amplitude-analytics==1.1.4 @@ -84,11 +88,11 @@ click-repl==0.2.0 # via celery codecov-ribs==0.1.18 # via shared -codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem +codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz # via -r requirements.in colour==0.1.5 # via shared -coverage==7.5.1 +coverage[toml]==7.5.1 # via # codecovopentelem # pytest-cov @@ -166,7 +170,7 @@ filelock==3.0.12 # via virtualenv freezegun==1.1.0 # via -r requirements.in -google-api-core==2.23.0 +google-api-core[grpc]==2.23.0 # via # google-cloud-core # google-cloud-pubsub @@ -191,7 +195,7 @@ google-crc32c==1.0.0 # google-resumable-media google-resumable-media==2.7.2 # via google-cloud-storage -googleapis-common-protos==1.59.1 +googleapis-common-protos[grpc]==1.59.1 # via # google-api-core # grpc-google-iam-v1 @@ -400,23 +404,19 @@ requests==2.32.3 # google-cloud-storage # shared # stripe -rfc3986==1.4.0 +rfc3986[idna2008]==1.4.0 # via httpx rsa==4.7.2 # via google-auth s3transfer==0.5.0 # via boto3 -sentry-sdk==2.13.0 +sentry-sdk[celery]==2.13.0 # via # -r requirements.in # shared setproctitle==1.1.10 # via -r requirements.in -setuptools==75.8.0 - # via - # ddtrace - # opentelemetry-instrumentation -shared @ https://github.com/codecov/shared/archive/9235672f89d2810fcf8c37905f0645f8ac77f4d4.tar.gz#egg=shared +shared @ https://github.com/codecov/shared/archive/abf6180de5cee3ac99b7fbf909bd82c7325063fd.tar.gz # via -r requirements.in simplejson==3.17.2 # via -r requirements.in @@ -499,3 +499,6 @@ zipp==3.19.2 # via importlib-metadata zstandard==0.23.0 # via shared + +# The following packages are considered to be unsafe in a requirements file: +# setuptools