Merge pull request #259 from EBI-Metagenomics/feature/coassembly-schema

Coassembly schema support

Author: SandyRogers

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+This PR:
+*
+
+
+---
+
+#### Checklist
+- The tests are passing on Github Actions (checked automatically)
+- The test coverage is at least as good as before (checked automatically)
+- Any model changes are reflected by migrations (checked automatically)
+- [ ] If `.talismanrc` was changed, it does not contain any duplicates (each file appears at most once)
+- [ ] The command `task make-dev-data` still works
+- [ ] The local docker-compose dev environment still works (`task run`)
+- [ ] Any new prefect flows activate Django before importing any models (`from activate_django_first import EMG_CONFIG`)
+- [ ] The code style guide in `README.md` has been followed

.gitignore

@@ -29,5 +29,6 @@ slurm-dev-environment/fs/nfs/ftp/public/databases/metagenomics/mgnify_results/*
 genomes/temp/*
 slurm-dev-environment/fs/nfs/public/tests/assembly_v6_output/ERP106708/MGYS*
 slurm-dev-environment/fs/nfs/public/tests/amplicon_v6_output/dwca/*
+.pytest-cache/
 .claude/*
 CLAUDE.md

.pre-commit-config.yaml

@@ -24,3 +24,7 @@ repos:
     hooks:
       - id: talisman-commit
         entry: cmd --githook pre-commit
+  - repo: https://github.com/ebi-metagenomics/no_talisman_dupes
+    rev: 'v0.1.0'
+    hooks:
+      - id: talisman-no-duplicate-filenames

.talismanrc

@@ -1,7 +1,4 @@
 fileignoreconfig:
-- filename: slurm-dev-environment/configs/jwt_hs256.key
-  checksum: cf9887754600f97f20b4afb223bcd666c3b0aa47dab9b067669c24997f398c26
-
 - filename: .github/workflows/release_k8s.yml
   checksum: 0e93196d244417801fd40ad6bfa01f53a6bf04257371378a1dc3e14cbc7a04d8
 
@@ -54,7 +51,7 @@ fileignoreconfig:
   checksum: 3afe465b36e4e141124bd84c8459c5aa81332eedf49b691eff22c74df63a0f18
 
 - filename: emgapiv2/settings.py
-  checksum: ab0fa7cdb6cd720aedd0a4f6bb9e2940187f9de91a39967e93ed4b91ca5050c5
+  checksum: 4113e9045da277c139530cbf71de334dc83b49a4f93a570b559e694bdd5cb0c8
 
 - filename: emgapiv2/settings_test.py
   checksum: ded898400b4eda7e3cf75694af1b66235975a85eb5b8f49f3a2c708163a7aad4
@@ -65,6 +62,12 @@ fileignoreconfig:
 - filename: emgapiv2/widgets.py
   allowed_patterns: [key]
 
+- filename: pyproject.toml
+  allowed_patterns: [Config key]
+
+- filename: slurm-dev-environment/configs/jwt_hs256.key
+  checksum: cf9887754600f97f20b4afb223bcd666c3b0aa47dab9b067669c24997f398c26
+
 - filename: slurm-dev-environment/configs/private-data-nginx.conf
   checksum: e3bb31e2a0f97c62ee684dd7124e5b72db364c52088ddd98331310909c6a0493
 
@@ -155,9 +158,6 @@ fileignoreconfig:
 - filename: workflows/flows/update_ena_accession_from_json_flow.py
   allowed_patterns: [total_missing_key_or_parse_error]
 
-- filename: workflows/flows/upload_assembly.py
-  checksum: f82f92058dabd50b9b6caf557d4eadae227ec4f6a0233135653251c20cf3c64b
-
 - filename: workflows/prefect_utils/env_context.py
   checksum: 8f5e11c7e0038bc4a648dc61ec657a679356857f77e557f744e2dd769ad88def
   ignore_detectors: [filename]
@@ -186,9 +186,6 @@ fileignoreconfig:
 - filename: workflows/data_io_utils/schemas/base.py
   checksum: 27b03a470da785c58f8294d2aa0aa3ef5f4e5ec5cffd97984ab446a5b8a9f337
 
-- filename: Taskfile.yaml
-  checksum: 592e19dbbafe04c136aa73a4921ac04b2d0473e654416b08d09e81d7544b8b09
-
 - filename: workflows/data_io_utils/schemas/assembly.py
   checksum: 12b45869a636431bb6e1bfe193f02f301a6b484a0c4503bc793f5450c36c932a
 

Taskfile.yaml

@@ -17,7 +17,7 @@ tasks:
     cmds:
       - docker compose exec -e PREFECT_CLIENT_RETRY_EXTRA_CODES=0 app python manage.py prefectcli work-pool create --type {{.TYPE}} {{.CLI_ARGS}} {{.NAME}}
     vars:
-      TYPE: '{{.TYPE | default "process"}}'
+      TYPE: '{{.TYPE | default "slurm"}}'
       NAME: '{{.NAME | default "slurm"}}'
 
   deploy-flow:
@@ -82,7 +82,7 @@ tasks:
     interactive: true
 
   test:
-    desc: "Run pytest tests. E.g. `task test` or `task test -- -k study`"
+    desc: "Run pytest tests. E.g. `task test` or `task test -- -k study` (for test selection), or `task test -- --lf` (for last failures only)"
     cmds:
       - docker compose run --entrypoint /bin/bash app -c "pytest {{.CLI_ARGS}}"
 

analyses/admin/assembly.py

@@ -43,7 +43,7 @@ class StudyFilterForAssembly(StudyFilter):
     ]
     search_fields = [
         "id",
-        "run__ena_accessions",
+        "runs__ena_accessions",
         "ena_study__title",
         "ena_study__accession",
         "ena_study__additional_accessions",
@@ -56,7 +56,7 @@ class StudyFilterForAssembly(StudyFilter):
         "reads_study__ena_study__additional_accessions",
         "ena_accessions",
     ]
-    autocomplete_fields = ["ena_study", "reads_study", "assembly_study", "run"]
+    autocomplete_fields = ["ena_study", "reads_study", "assembly_study", "runs"]
     readonly_fields = ["created_at", "updated_at"]
 
     def status_summary(self, obj):
@@ -76,7 +76,7 @@ def status_summary(self, obj):
             "Reads",
             {
                 "classes": ["tab"],
-                "fields": ["reads_study", "run"],
+                "fields": ["reads_study", "runs", "sample"],
             },
         ),
         (

analyses/admin/sample.py

@@ -2,8 +2,19 @@
 from unfold.admin import ModelAdmin
 from unfold.decorators import display
 
-from analyses.admin.base import ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin
-from analyses.models import Sample
+from analyses.admin.base import (
+    ENABrowserLinkMixin,
+    JSONFieldWidgetOverridesMixin,
+    TabularInlinePaginatedWithTabSupport,
+)
+from analyses.models import Sample, SampleRelatedSample
+
+
+class SampleRelatedSamplesInline(TabularInlinePaginatedWithTabSupport):
+    model = SampleRelatedSample
+    autocomplete_fields = ["declaring_sample", "related_sample"]
+    fk_name = "declaring_sample"
+    extra = 0
 
 
 @admin.register(Sample)
@@ -17,6 +28,7 @@ class SampleAdmin(ENABrowserLinkMixin, JSONFieldWidgetOverridesMixin, ModelAdmin
     autocomplete_fields = ["ena_sample", "ena_study", "studies"]
     list_display = ["first_accession", "updated_at", "display_accessions"]
     list_filter = ["updated_at", "created_at", "is_private"]
+    inlines = [SampleRelatedSamplesInline]
 
     @display(description="ENA Accessions", label=True)
     def display_accessions(self, instance: Sample):

analyses/admin/study.py

@@ -61,8 +61,8 @@ class StudyAssembliesInline(TabularInlinePaginatedWithTabSupport):
     verbose_name = "Assembly in this study"
     verbose_name_plural = "Assemblies in this study"
     show_change_link = True
-    fields = ["run", "status", "dir"]
-    readonly_fields = ["run"]
+    fields = ["runs", "status", "dir"]
+    readonly_fields = ["runs"]
     max_num = 0
     fk_name = "assembly_study"
     formfield_overrides = {
@@ -87,8 +87,8 @@ class StudyReadsInline(TabularInlinePaginatedWithTabSupport):
     verbose_name = "Assembly of this studies' read"
     verbose_name_plural = "Assemblies of this studies' reads"
     show_change_link = True
-    fields = ["run", "status", "dir"]
-    readonly_fields = ["run"]
+    fields = ["runs", "status", "dir"]
+    readonly_fields = ["runs"]
     max_num = 0
     fk_name = "reads_study"
     formfield_overrides = {

analyses/fixtures/assembly/conftest.py

@@ -20,32 +20,34 @@ def mgnify_assemblies(raw_read_run, raw_reads_mgnify_study, assemblers):
     ]
     assembly_objects = []
     # create metaspades assemblies
-    for run in assembleable_runs:
-        assembly_obj, _ = mg_models.Assembly.objects.get_or_create(
-            run=run,
-            reads_study=raw_reads_mgnify_study,
-            ena_study=raw_reads_mgnify_study.ena_study,
-            assembler=assembler_metaspades,
-            dir="slurm-dev-environment/fs/hps/tests/assembly_uploader",
-            metadata={"coverage": 20},
-            sample=run.sample,
-            ena_accessions=[
-                "ERZ857107",
-            ],
+    for i, run in enumerate(assembleable_runs):
+        assembly_obj, created = (
+            mg_models.Assembly.objects.get_or_create_for_run_and_sample(
+                run=run,
+                sample=run.sample,
+                reads_study=raw_reads_mgnify_study,
+                ena_study=raw_reads_mgnify_study.ena_study,
+                assembler=assembler_metaspades,
+                dir="slurm-dev-environment/fs/hps/tests/assembly_uploader",
+                metadata={"coverage": 20},
+                ena_accessions=[
+                    f"ERZ_METASPADES_{i}",
+                ],
+            )
         )
         assembly_objects.append(assembly_obj)
 
     # create one megahit assembly
-    for run in assembleable_runs[:1]:
-        assembly, _ = mg_models.Assembly.objects.get_or_create(
+    for i, run in enumerate(assembleable_runs[:1]):
+        assembly, created = mg_models.Assembly.objects.get_or_create_for_run_and_sample(
             run=run,
+            sample=run.sample,
             reads_study=raw_reads_mgnify_study,
             ena_study=raw_reads_mgnify_study.ena_study,
             assembler=assembler_megahit,
             dir="/hps/tests/assembly_uploader",
             metadata={"coverage": 10},
-            sample=run.sample,
-            ena_accessions=["ERZ857108"],
+            ena_accessions=[f"ERZ_MEGAHIT_{i}"],
         )
         assembly_objects.append(assembly)
     return assembly_objects
@@ -55,7 +57,7 @@ def mgnify_assemblies(raw_read_run, raw_reads_mgnify_study, assemblers):
 def mgnify_assemblies_completed(mgnify_assemblies):
     run_accession = "SRR6180434"
     metaspades_assemblies = mg_models.Assembly.objects.filter(
-        assembler__name="metaspades", run__ena_accessions__contains=[run_accession]
+        assembler__name="metaspades", runs__ena_accessions__contains=[run_accession]
     )
     for item in metaspades_assemblies:
         item.mark_status("assembly_started")
@@ -67,7 +69,7 @@ def mgnify_assemblies_completed(mgnify_assemblies):
 def mgnify_assembly_completed_uploader_sanity_check(mgnify_assemblies):
     run_accession = "SRR6180435"
     metaspades_assemblies = mg_models.Assembly.objects.filter(
-        assembler__name="metaspades", run__ena_accessions__contains=[run_accession]
+        assembler__name="metaspades", runs__ena_accessions__contains=[run_accession]
     )
     for item in metaspades_assemblies:
         item.mark_status("assembly_completed")

analyses/migrations/0053_add_assembly_runs_m2m_fields.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.2.7 on 2026-01-22 14:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyses", "0052_update_download_groups_to_pathways_and_systems"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="assembly",
+            name="runs",
+            field=models.ManyToManyField(
+                blank=True, related_name="assemblies_m2m", to="analyses.run"
+            ),
+        ),
+    ]