Schema Migration: Shared FluorState MTI for protein and dye states, introduce FluorescenceMeasurement

backend/config/settings/base.py

@@ -63,7 +63,6 @@
 
 THIRD_PARTY_APPS = [
     "django_structlog",  # Structured logging
-    "scout_apm.django",  # APM monitoring
     "crispy_forms",  # Form layouts
     "crispy_bootstrap5",
     "allauth",  # registration

backend/config/settings/production.py

@@ -211,6 +211,7 @@ def WHITENOISE_IMMUTABLE_FILE_TEST(path, url):
 # Scout APM Configuration
 # ------------------------------------------------------------------------------
 # SCOUT_MONITOR and SCOUT_KEY are automatically set by the Heroku addon
+INSTALLED_APPS += ["scout_apm.django"]
 SCOUT_NAME = "FPbase"
 
 # Structlog Configuration for Production

backend/favit/models.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
@@ -6,19 +10,24 @@
 
 from .managers import FavoriteManager
 
+if TYPE_CHECKING:
+    from fpbase.users.models import User  # noqa F401
+
 
 class Favorite(models.Model):
-    user = models.ForeignKey(
+    user_id: int
+    user = models.ForeignKey["User"](
         getattr(settings, "AUTH_USER_MODEL", "auth.User"),
         related_name="favorites",
         on_delete=models.CASCADE,
     )
-    target_content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
+    target_content_type_id: int
+    target_content_type = models.ForeignKey[ContentType](ContentType, on_delete=models.CASCADE)
     target_object_id = models.PositiveIntegerField()
     target = GenericForeignKey("target_content_type", "target_object_id")
     timestamp = models.DateTimeField(auto_now_add=True, db_index=True)
 
-    objects = FavoriteManager()
+    objects: FavoriteManager = FavoriteManager()
 
     class Meta:
         ordering = ["-timestamp"]

backend/fpbase/cache_utils.py

@@ -62,6 +62,7 @@ def _invalidate_optical_config_cache() -> None:
 SPECTRUM_OWNER_MODELS = {
     "proteins.Camera",
     "proteins.Dye",
+    "proteins.DyeState",
     "proteins.Filter",
     "proteins.Light",
     "proteins.Protein",

backend/fpbase/celery.py

@@ -10,7 +10,8 @@
 app = Celery("fpbase", namespace="CELERY")
 app.config_from_object("django.conf:settings", namespace="CELERY")
 
-scout_apm.celery.install(app)
+if getattr(settings, "SCOUT_NAME", False):
+    scout_apm.celery.install(app)
 
 # Load task modules from all registered Django app configs.
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

backend/fpbase/users/models.py

@@ -1,9 +1,18 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from django.contrib.auth.models import AbstractUser
 from django.contrib.auth.signals import user_logged_in
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
+if TYPE_CHECKING:
+    from favit.models import Favorite
+    from proteins.models import Microscope, OpticalConfig, ProteinCollection
+    from references.models import Reference
+
 
 class User(AbstractUser):
     # # AbstractUser Fields
@@ -19,6 +28,15 @@ class User(AbstractUser):
     # around the globe.
     name = models.CharField(_("Name of User"), blank=True, max_length=255)
 
+    if TYPE_CHECKING:
+        logins: models.QuerySet[UserLogin]
+        favorites: models.QuerySet[Favorite]
+        reference_author: models.QuerySet[Reference]
+        reference_modifier: models.QuerySet[Reference]
+        microscopes: models.QuerySet[Microscope]
+        opticalconfigs: models.QuerySet[OpticalConfig]
+        proteincollections: models.QuerySet[ProteinCollection]
+
     def __str__(self):
         return self.username
 
@@ -29,7 +47,8 @@ def get_absolute_url(self):
 class UserLogin(models.Model):
     """Represent users' logins, one per record"""
 
-    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="logins")
+    user_id: int
+    user = models.ForeignKey[User](User, on_delete=models.CASCADE, related_name="logins")
     timestamp = models.DateTimeField(auto_now_add=True)
 
     def __str__(self) -> str:

backend/fpbase/views.py

@@ -149,7 +149,7 @@ def get_context_data(self):
         data = super().get_context_data()
         data["stats"] = {
             "proteins": Protein.objects.count(),
-            "protspectra": Spectrum.objects.exclude(owner_state=None).count(),
+            "protspectra": Spectrum.objects.exclude(owner_fluor=None).count(),
         }
         return data
 

backend/proteins/admin.py

@@ -15,6 +15,7 @@
     BleachMeasurement,
     Camera,
     Dye,
+    DyeState,
     Excerpt,
     Filter,
     FilterPlacement,
@@ -41,7 +42,7 @@
 class SpectrumOwner:
     list_display = ("__str__", "spectra", "created_by", "created")
     list_select_related = ("created_by",)
-    list_filter = ("created", "manufacturer")
+    list_filter = ("created",)
     search_fields = ("name",)
 
     def __init__(self, *args, **kwargs):
@@ -113,7 +114,7 @@ class StateInline(MultipleSpectraOwner, admin.StackedInline):
                 "fields": (
                     ("ex_max", "em_max"),
                     ("ext_coeff", "qy"),
-                    ("twop_ex_max", "twop_peakGM", "twop_qy"),
+                    ("twop_ex_max", "twop_peak_gm", "twop_qy"),
                     ("pka", "maturation"),
                     "lifetime",
                     "bleach_links",
@@ -165,9 +166,16 @@ class LightAdmin(SpectrumOwner, admin.ModelAdmin):
 
 
 @admin.register(Dye)
-class DyeAdmin(MultipleSpectraOwner, VersionAdmin):
+class DyeAdmin(admin.ModelAdmin):
     model = Dye
     ordering = ("-created",)
+    list_filter = ("created", "manufacturer")
+
+
+@admin.register(DyeState)
+class DyeStateAdmin(MultipleSpectraOwner, VersionAdmin):
+    model = DyeState
+    ordering = ("-created",)
 
 
 @admin.register(Filter)
@@ -198,37 +206,37 @@ class SpectrumAdmin(VersionAdmin):
     model = Spectrum
     autocomplete_fields = ["reference"]
     list_select_related = (
-        "owner_state__protein",
+        "owner_fluor",
         "owner_filter",
         "owner_camera",
         "owner_light",
-        "owner_dye",
         "created_by",
     )
     list_display = ("__str__", "category", "subtype", "owner", "created_by")
     list_filter = ("status", "created", "category", "subtype")
     readonly_fields = ("owner", "name", "created", "modified", "spectrum_preview")
     search_fields = (
-        "owner_state__protein__name",
+        "owner_fluor__name",
         "owner_filter__name",
         "owner_camera__name",
         "owner_light__name",
-        "owner_dye__name",
     )
 
     def get_fields(self, request, obj=None):
         fields = []
         if not obj or not obj.category:
+            # If no category yet, allow selecting any owner type
             own = [
-                "owner_state",
+                "owner_fluor",
                 "owner_filter",
                 "owner_camera",
                 "owner_light",
-                "owner_dye",
             ]
-        elif obj.category == Spectrum.PROTEIN:
-            own = ["owner_state"]
+        elif obj.category in (Spectrum.PROTEIN, Spectrum.DYE):
+            # Protein and Dye both use owner_fluor (Fluorophore)
+            own = ["owner_fluor"]
         else:
+            # Filter, Camera, Light
             own = ["owner_" + obj.get_category_display().split(" ")[0].lower()]
         fields.extend(own)
         self.autocomplete_fields.extend(own)
@@ -342,7 +350,7 @@ class StateAdmin(CompareVersionAdmin):
                 "fields": (
                     ("ex_max", "em_max"),
                     ("ext_coeff", "qy"),
-                    ("twop_ex_max", "twop_peakGM", "twop_qy"),
+                    ("twop_ex_max", "twop_peak_gm", "twop_qy"),
                     ("pka", "maturation"),
                     "lifetime",
                 )

backend/proteins/api/serializers.py

@@ -1,7 +1,7 @@
 from rest_framework import serializers
 
-from ..models import Protein, Spectrum, State, StateTransition
-from ._tweaks import ModelSerializer
+from proteins.api._tweaks import ModelSerializer
+from proteins.models import Fluorophore, Protein, Spectrum, State, StateTransition
 
 
 class SpectrumSerializer(serializers.ModelSerializer):
@@ -29,12 +29,16 @@ class Meta:
         )
 
     def get_protein_name(self, obj):
-        if obj.owner_state:
-            return obj.owner_state.protein.name
+        # Check if owner_fluor is a State (has protein attribute)
+        if obj.owner_fluor and obj.owner_fluor.entity_type == Fluorophore.EntityTypes.PROTEIN:
+            return obj.owner_fluor.protein.name
+        return None
 
     def get_protein_slug(self, obj):
-        if obj.owner_state:
-            return obj.owner_state.protein.slug
+        # Check if owner_fluor is a State (has protein attribute)
+        if obj.owner_fluor and obj.owner_fluor.entity_type == Fluorophore.EntityTypes.PROTEIN:
+            return obj.owner_fluor.protein.slug
+        return None
 
 
 class StateTransitionSerializer(serializers.ModelSerializer):

backend/proteins/api/views.py

@@ -74,7 +74,7 @@ class SpectrumList(ListAPIView):
 
 
 class SpectrumDetail(RetrieveAPIView):
-    queryset = pm.Spectrum.objects.prefetch_related("owner_state")
+    queryset = pm.Spectrum.objects.prefetch_related("owner_fluor")
     permission_classes = (AllowAny,)
     serializer_class = SpectrumSerializer
 
@@ -122,7 +122,7 @@ def dispatch(self, *args, **kwargs):
 
 class BasicProteinListAPIView(ProteinListAPIView):
     queryset = (
-        pm.Protein.visible.filter(switch_type=pm.Protein.BASIC)
+        pm.Protein.visible.filter(switch_type=pm.Protein.SwitchingChoices.BASIC)
         .select_related("default_state")
         .annotate(rate=Max(F("default_state__bleach_measurements__rate")))
     )

backend/proteins/factories.py

@@ -1,6 +1,6 @@
 # pyright: reportPrivateImportUsage=false
 import random
-from typing import TypeVar, cast
+from typing import TYPE_CHECKING, TypeVar, cast
 
 import factory
 import factory.builder
@@ -9,10 +9,12 @@
 from django.utils.text import slugify
 
 from fpseq import FPSeq
+from proteins.models import Camera, Filter, FilterPlacement, Light, Microscope, OpticalConfig, Protein, Spectrum, State
 from proteins.util.helpers import wave_to_hex
 from references.factories import ReferenceFactory
 
-from .models import Camera, Filter, FilterPlacement, Light, Microscope, OpticalConfig, Protein, Spectrum, State
+if TYPE_CHECKING:
+    from proteins.models import Fluorophore
 
 T = TypeVar("T")
 
@@ -120,14 +122,14 @@ def _mock_edge_filter(edge, subtype, min_wave=300, max_wave=900, transmission=0.
 def _build_spectral_data(resolver: factory.builder.Resolver):
     subtype = getattr(resolver, "subtype", None)
 
-    if (owner_state := getattr(resolver, "owner_state", None)) is not None:
-        owner_state = cast("State", owner_state)
+    if (owner_fluor := getattr(resolver, "owner_fluor", None)) is not None:
+        owner_fluor = cast("Fluorophore", owner_fluor)
         if subtype == "ex":
-            return _mock_spectrum(owner_state.ex_max, type="ex")
+            return _mock_spectrum(owner_fluor.ex_max, type="ex")
         elif subtype == "em":
-            return _mock_spectrum(owner_state.em_max, type="em")
-        elif subtype == "2p" and owner_state.twop_ex_max:
-            return _mock_spectrum(owner_state.twop_ex_max, type="ex", min_wave=600, max_wave=1100)
+            return _mock_spectrum(owner_fluor.em_max, type="em")
+        elif subtype == "2p" and getattr(owner_fluor, "twop_ex_max", None):
+            return _mock_spectrum(owner_fluor.twop_ex_max, type="ex", min_wave=600, max_wave=1100)
 
     if (owner_filter := getattr(resolver, "owner_filter", None)) is not None:
         owner_filter = cast("Filter", owner_filter)
@@ -192,27 +194,31 @@ class Meta:
 
     ex_spectrum = factory.RelatedFactory(
         "proteins.factories.SpectrumFactory",
-        factory_related_name="owner_state",
+        factory_related_name="owner_fluor",
         subtype="ex",
         category="p",
     )
     em_spectrum = factory.RelatedFactory(
         "proteins.factories.SpectrumFactory",
-        factory_related_name="owner_state",
+        factory_related_name="owner_fluor",
         subtype="em",
         category="p",
     )
     twop_spectrum = factory.RelatedFactory(
         "proteins.factories.SpectrumFactory",
-        factory_related_name="owner_state",
+        factory_related_name="owner_fluor",
         subtype="2p",
         category="p",
     )
 
 
-class DyeFactory(FluorophoreFactory):
+class DyeFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = "proteins.Dye"
+        django_get_or_create = ("name", "slug")
+
+    name = factory.Sequence(lambda n: f"TestDye{n}")
+    slug = factory.LazyAttribute(lambda o: slugify(o.name))
 
 
 class ProteinFactory(factory.django.DjangoModelFactory[Protein]):
@@ -225,7 +231,7 @@ class Meta:
     slug = factory.LazyAttribute(lambda o: slugify(o.name))
     seq = factory.LazyFunction(_protein_seq)
     seq_validated = factory.Faker("boolean", chance_of_getting_true=75)
-    agg = factory.fuzzy.FuzzyChoice(Protein.AGG_CHOICES, getter=lambda c: c[0])
+    agg = factory.fuzzy.FuzzyChoice(Protein.AggChoices, getter=lambda c: c[0])
     pdb = factory.LazyFunction(lambda: random.choices(REAL_PDBS, k=random.randint(0, 2)))
     parent_organism = factory.SubFactory(OrganismFactory)
     primary_reference = factory.SubFactory("references.factories.ReferenceFactory")
@@ -352,7 +358,7 @@ def create_egfp() -> Protein:
             "DHYQQNTPIGDGPVLLPDNHYLSTQSALSKDPNEKRDHMVLLEFVTAAGITLGMDELYK"
         ),
         seq_validated=True,
-        agg=Protein.MONOMER,
+        agg=Protein.AggChoices.MONOMER,
         pdb=["4EUL", "2Y0G"],
         parent_organism=OrganismFactory(scientific_name="Aequorea victoria", id=6100, division="hydrozoans"),
         primary_reference=ReferenceFactory(doi="10.1016/0378-1119(95)00685-0"),
@@ -367,7 +373,7 @@ def create_egfp() -> Protein:
         default_state__maturation=25,
         default_state__lifetime=2.8,
         default_state__twop_ex_max=927,
-        default_state__twop_peakGM=39.64,
+        default_state__twop_peak_gm=39.64,
     )
 
     return egfp

backend/proteins/filters.py

@@ -10,7 +10,7 @@
 class SpectrumFilter(filters.FilterSet):
     class Meta:
         model = Spectrum
-        fields = ("category", "subtype", "id", "owner_state")
+        fields = ("category", "subtype", "id", "owner_fluor")
 
 
 class StateFilter(filters.FilterSet):
@@ -107,8 +107,8 @@ class ProteinFilter(filters.FilterSet):
     name__icontains = django_filters.CharFilter(
         field_name="name", method="name_or_alias_icontains", lookup_expr="icontains"
     )
-    switch_type__ne = django_filters.ChoiceFilter(choices=Protein.SWITCHING_CHOICES, method="switch_type__notequal")
-    cofactor__ne = django_filters.ChoiceFilter(choices=Protein.COFACTOR_CHOICES, method="cofactor__notequal")
+    switch_type__ne = django_filters.ChoiceFilter(choices=Protein.SwitchingChoices, method="switch_type__notequal")
+    cofactor__ne = django_filters.ChoiceFilter(choices=Protein.CofactorChoices, method="cofactor__notequal")
     parent_organism__ne = django_filters.ModelChoiceFilter(
         queryset=Organism.objects.all(), method="parent_organism__notequal"
     )