Merge feature/dataset-abc into dev

api/scpca_portal/management/commands/create_ccdl_datasets.py

@@ -4,7 +4,7 @@
 from django.template.defaultfilters import pluralize
 
 from scpca_portal.config.logging import get_and_configure_logger
-from scpca_portal.models import Dataset, Job
+from scpca_portal.models import CCDLDataset, Job
 
 logger = get_and_configure_logger(__name__)
 
@@ -44,7 +44,7 @@ def handle(self, *args, **kwargs):
         self.create_ccdl_datasets(**kwargs)
 
     def create_ccdl_datasets(self, ignore_hash, retry_failed_jobs, **kwargs) -> None:
-        created_datasets, updated_datasets = Dataset.create_or_update_ccdl_datasets(
+        created_datasets, updated_datasets = CCDLDataset.create_or_update_ccdl_datasets(
             ignore_hash=ignore_hash
         )
         if created_datasets:

api/scpca_portal/migrations/0080_ccdldataset_userdataset.py

@@ -0,0 +1,236 @@
+# Generated by Django 3.2.25 on 2026-01-09 18:35
+
+import uuid
+
+import django.contrib.postgres.fields
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scpca_portal", "0079_auto_20251119_1503"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="UserDataset",
+            fields=[
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4, editable=False, primary_key=True, serialize=False
+                    ),
+                ),
+                (
+                    "format",
+                    models.TextField(
+                        choices=[
+                            ("ANN_DATA", "AnnData"),
+                            ("SINGLE_CELL_EXPERIMENT", "Single-cell experiment"),
+                            ("METADATA", "Metadata"),
+                        ]
+                    ),
+                ),
+                ("data", models.JSONField(default=dict)),
+                ("email", models.EmailField(max_length=254, null=True)),
+                ("start", models.BooleanField(default=False)),
+                ("data_hash", models.CharField(max_length=32, null=True)),
+                ("metadata_hash", models.CharField(max_length=32, null=True)),
+                ("readme_hash", models.CharField(max_length=32, null=True)),
+                ("combined_hash", models.CharField(max_length=32, null=True)),
+                ("includes_files_bulk", models.BooleanField(default=False)),
+                ("includes_files_cite_seq", models.BooleanField(default=False)),
+                ("includes_files_merged", models.BooleanField(default=False)),
+                ("includes_files_multiplexed", models.BooleanField(default=False)),
+                ("estimated_size_in_bytes", models.BigIntegerField(default=0)),
+                ("started_at", models.DateTimeField(null=True)),
+                ("is_started", models.BooleanField(default=False)),
+                ("pending_at", models.DateTimeField(null=True)),
+                ("is_pending", models.BooleanField(default=False)),
+                ("processing_at", models.DateTimeField(null=True)),
+                ("is_processing", models.BooleanField(default=False)),
+                ("succeeded_at", models.DateTimeField(null=True)),
+                ("is_succeeded", models.BooleanField(default=False)),
+                ("failed_at", models.DateTimeField(null=True)),
+                ("is_failed", models.BooleanField(default=False)),
+                ("failed_reason", models.TextField(null=True)),
+                ("expires_at", models.DateTimeField(null=True)),
+                ("is_expired", models.BooleanField(default=False)),
+                ("terminated_at", models.DateTimeField(null=True)),
+                ("is_terminated", models.BooleanField(default=False)),
+                ("terminated_reason", models.TextField(null=True)),
+                ("total_sample_count", models.BigIntegerField(default=0)),
+                ("diagnoses_summary", models.JSONField(default=dict)),
+                ("files_summary", models.JSONField(default=list)),
+                ("project_diagnoses", models.JSONField(default=dict)),
+                ("project_modality_counts", models.JSONField(default=dict)),
+                (
+                    "modality_count_mismatch_projects",
+                    django.contrib.postgres.fields.ArrayField(
+                        base_field=models.TextField(), default=list, size=None
+                    ),
+                ),
+                ("project_sample_counts", models.JSONField(default=dict)),
+                ("project_titles", models.JSONField(default=dict)),
+                (
+                    "computed_file",
+                    models.OneToOneField(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="userdataset",
+                        to="scpca_portal.computedfile",
+                    ),
+                ),
+                (
+                    "download_tokens",
+                    models.ManyToManyField(
+                        related_name="downloaded_userdataset_set", to="scpca_portal.APIToken"
+                    ),
+                ),
+                (
+                    "regenerated_from",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="regenerated_userdataset_set",
+                        to="scpca_portal.userdataset",
+                    ),
+                ),
+                (
+                    "token",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="userdataset_set",
+                        to="scpca_portal.apitoken",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "user_datasets",
+                "ordering": ["updated_at"],
+                "get_latest_by": "updated_at",
+            },
+        ),
+        migrations.CreateModel(
+            name="CCDLDataset",
+            fields=[
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4, editable=False, primary_key=True, serialize=False
+                    ),
+                ),
+                (
+                    "format",
+                    models.TextField(
+                        choices=[
+                            ("ANN_DATA", "AnnData"),
+                            ("SINGLE_CELL_EXPERIMENT", "Single-cell experiment"),
+                            ("METADATA", "Metadata"),
+                        ]
+                    ),
+                ),
+                ("data", models.JSONField(default=dict)),
+                ("email", models.EmailField(max_length=254, null=True)),
+                ("start", models.BooleanField(default=False)),
+                ("data_hash", models.CharField(max_length=32, null=True)),
+                ("metadata_hash", models.CharField(max_length=32, null=True)),
+                ("readme_hash", models.CharField(max_length=32, null=True)),
+                ("combined_hash", models.CharField(max_length=32, null=True)),
+                ("includes_files_bulk", models.BooleanField(default=False)),
+                ("includes_files_cite_seq", models.BooleanField(default=False)),
+                ("includes_files_merged", models.BooleanField(default=False)),
+                ("includes_files_multiplexed", models.BooleanField(default=False)),
+                ("estimated_size_in_bytes", models.BigIntegerField(default=0)),
+                ("started_at", models.DateTimeField(null=True)),
+                ("is_started", models.BooleanField(default=False)),
+                ("pending_at", models.DateTimeField(null=True)),
+                ("is_pending", models.BooleanField(default=False)),
+                ("processing_at", models.DateTimeField(null=True)),
+                ("is_processing", models.BooleanField(default=False)),
+                ("succeeded_at", models.DateTimeField(null=True)),
+                ("is_succeeded", models.BooleanField(default=False)),
+                ("failed_at", models.DateTimeField(null=True)),
+                ("is_failed", models.BooleanField(default=False)),
+                ("failed_reason", models.TextField(null=True)),
+                ("expires_at", models.DateTimeField(null=True)),
+                ("is_expired", models.BooleanField(default=False)),
+                ("terminated_at", models.DateTimeField(null=True)),
+                ("is_terminated", models.BooleanField(default=False)),
+                ("terminated_reason", models.TextField(null=True)),
+                (
+                    "ccdl_name",
+                    models.TextField(
+                        choices=[
+                            ("ALL_METADATA", "All Metadata"),
+                            (
+                                "SINGLE_CELL_SINGLE_CELL_EXPERIMENT",
+                                "Single Cell Single Cell Experiment",
+                            ),
+                            (
+                                "SINGLE_CELL_SINGLE_CELL_EXPERIMENT_NO_MULTIPLEXED",
+                                "Single Cell Single Cell Experiment No Multiplexed",
+                            ),
+                            (
+                                "SINGLE_CELL_SINGLE_CELL_EXPERIMENT_MERGED",
+                                "Single Cell Single Cell Experiment Merged",
+                            ),
+                            ("SINGLE_CELL_ANN_DATA", "Single Cell Ann Data"),
+                            ("SINGLE_CELL_ANN_DATA_MERGED", "Single Cell Ann Data Merged"),
+                            ("SPATIAL_SPATIAL_SPACERANGER", "Spatial Spatial Spaceranger"),
+                        ],
+                    ),
+                ),
+                ("ccdl_project_id", models.TextField(null=True)),
+                (
+                    "ccdl_modality",
+                    models.TextField(
+                        choices=[
+                            ("BULK_RNA_SEQ", "Bulk RNA-seq"),
+                            ("CITE_SEQ", "CITE-seq"),
+                            ("MULTIPLEXED", "Multiplexed"),
+                            ("SINGLE_CELL", "Single-cell"),
+                            ("SPATIAL", "Spatial Data"),
+                        ],
+                        null=True,
+                    ),
+                ),
+                (
+                    "computed_file",
+                    models.OneToOneField(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="ccdldataset",
+                        to="scpca_portal.computedfile",
+                    ),
+                ),
+                (
+                    "download_tokens",
+                    models.ManyToManyField(
+                        related_name="downloaded_ccdldataset_set", to="scpca_portal.APIToken"
+                    ),
+                ),
+                (
+                    "token",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="ccdldataset_set",
+                        to="scpca_portal.apitoken",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "ccdl_datasets",
+                "ordering": ["updated_at"],
+                "get_latest_by": "updated_at",
+            },
+        ),
+    ]