Added new tables for additional BIL identifiers pertaining to Instrum…

ingest/forms.py

@@ -4,7 +4,7 @@
 
 
 class UploadForm(forms.Form):
-    associated_collection = forms.ModelChoiceField(queryset=Collection.objects.all())
+    associated_submission = forms.ModelChoiceField(queryset=Collection.objects.all())
 
 class DescriptiveMetadataForm(forms.ModelForm):
 

ingest/migrations/0027_auto_20240205_1642.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.18 on 2024-02-05 16:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ingest', '0026_datasetlinkage'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='bil_id',
+            options={'verbose_name': 'BIL ID', 'verbose_name_plural': 'BIL IDs'},
+        ),
+        migrations.AlterModelOptions(
+            name='dataset',
+            options={'verbose_name': 'Dataset', 'verbose_name_plural': 'Datasets'},
+        ),
+        migrations.CreateModel(
+            name='BIL_Specimen_ID',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('bil_spc_id', models.CharField(blank=True, max_length=256, null=True)),
+                ('specimen_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.specimen')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='BIL_Project_ID',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('bil_prj_id', models.CharField(blank=True, max_length=256, null=True)),
+                ('project_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.project')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='BIL_Instrument_ID',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('bil_ins_id', models.CharField(blank=True, max_length=256, null=True)),
+                ('instrument_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.instrument')),
+            ],
+        ),
+    ]

ingest/mne.py

@@ -171,6 +171,97 @@ def  dataset_num_to_mne(num):
              rstring=rstring + '-'
            rstring=rstring+Mne.DATAA[element]
        return rstring
+
+  @staticmethod
+  def  specimen_num_to_mne(num):
+
+       #-----------------------------------
+       #store 256 decoded elements in stack
+       #-----------------------------------
+       stack=[]
+       q=num
+       if q < 256 :
+          q,r = divmod(num,256)
+          stack.append(r)     
+       else:
+           while q > 255 :
+               q,r = divmod(q,256)
+               stack.append(r)
+           stack.append(q)
+       #----------------------
+       #Finally code the stack
+       #----------------------
+       rstring=""
+       first=True
+       for element in reversed(stack):
+           if first:
+             first=False
+           else:
+             rstring=rstring + '-'
+           rstring=rstring+Mne.DATAA[element]
+       rstring = "spc_"+rstring
+       return rstring
+
+  @staticmethod
+  def  instrument_num_to_mne(num):
+
+       #-----------------------------------
+       #store 256 decoded elements in stack
+       #-----------------------------------
+       stack=[]
+       q=num
+       if q < 256 :
+          q,r = divmod(num,256)
+          stack.append(r)     
+       else:
+           while q > 255 :
+               q,r = divmod(q,256)
+               stack.append(r)
+           stack.append(q)
+       #----------------------
+       #Finally code the stack
+       #----------------------
+       rstring=""
+       first=True
+       for element in reversed(stack):
+           if first:
+             first=False
+           else:
+             rstring=rstring + '-'
+           rstring=rstring+Mne.DATAA[element]
+       rstring = "ins_" + rstring
+       return rstring
+
+  @staticmethod
+  def  project_num_to_mne(num):
+
+       #-----------------------------------
+       #store 256 decoded elements in stack
+       #-----------------------------------
+       stack=[]
+       q=num
+       if q < 256 :
+          q,r = divmod(num,256)
+          stack.append(r)     
+       else:
+           while q > 255 :
+               q,r = divmod(q,256)
+               stack.append(r)
+           stack.append(q)
+       #----------------------
+       #Finally code the stack
+       #----------------------
+       rstring=""
+       first=True
+       for element in reversed(stack):
+           if first:
+             first=False
+           else:
+             rstring=rstring + '-'
+           rstring=rstring+Mne.DATAA[element]
+       rstring = "prj_" + rstring
+       return rstring
+
 
   def  mne_to_num(mme):
        stack=[]

ingest/models.py

@@ -508,4 +508,16 @@ class DatasetLinkage(models.Model):
     data_id_2 = models.CharField(max_length=256, blank=True, null=True)
     relationship = models.CharField(max_length=64, default="", choices=[('sequence data', 'Sequence Data'), ('neuron tracing', 'Neuron Tracing'), ('derived_data', 'Derived Data'), ('raw', 'Raw'), ('aligned', 'Aligned')]) 
     description = models.TextField(blank=True, null=True)
-    linkage_date = models.DateField(null=True, blank=True)
+    linkage_date = models.DateField(null=True, blank=True)
+
+class BIL_Specimen_ID(models.Model):
+    bil_spc_id = models.CharField(max_length=256, blank=True, null=True)
+    specimen_id = models.ForeignKey(Specimen, on_delete=models.SET_NULL, null = True, blank=True)
+
+class BIL_Instrument_ID(models.Model):
+    bil_ins_id = models.CharField(max_length=256, blank=True, null=True)
+    instrument_id = models.ForeignKey(Instrument, on_delete=models.SET_NULL, null = True, blank=True)
+
+class BIL_Project_ID(models.Model):
+    bil_prj_id = models.CharField(max_length=256, blank=True, null=True)
+    project_id = models.ForeignKey(Project, on_delete=models.SET_NULL, null = True, blank=True)

ingest/templates/ingest/descriptive_metadata_upload.html

@@ -26,7 +26,7 @@ <h3>Step 3 of 3: Upload metadata for associated submission</h3>
 {% if collections %}
 <ul class="list-group">
   <li class="list-group-item">Download and fill out either the <a href="{% static 'ingest/BIL_Metadata_TemplateV3.xlsx' %}">Excel</a> or <a href="{% static '/ingest/BIL_Metadata_TemplateV3.ods' %}">LibreOffice Calc</a> template. </li>
-  <li class="list-group-item">Choose a collection, then upload your metadata.</li>
+  <li class="list-group-item">Choose a submission, then upload your metadata.</li>
 </ul>
   <hr>  
     {% csrf_token %}

ingest/views.py

@@ -28,7 +28,7 @@
 from .field_list import required_metadata
 from .filters import CollectionFilter
 from .forms import CollectionForm, ImageMetadataForm, DescriptiveMetadataForm, UploadForm, collection_send
-from .models import UUID, Collection, ImageMetadata, DescriptiveMetadata, Project, ProjectPeople, People, Project, EventsLog, Contributor, Funder, Publication, Instrument, Dataset, Specimen, Image, Sheet, Consortium, ProjectConsortium, SWC, ProjectAssociation, BIL_ID, DatasetEventsLog
+from .models import UUID, Collection, ImageMetadata, DescriptiveMetadata, Project, ProjectPeople, People, Project, EventsLog, Contributor, Funder, Publication, Instrument, Dataset, Specimen, Image, Sheet, Consortium, ProjectConsortium, SWC, ProjectAssociation, BIL_ID, DatasetEventsLog, BIL_Specimen_ID, BIL_Instrument_ID, BIL_Project_ID
 from .tables import CollectionTable, DescriptiveMetadataTable, CollectionRequestTable
 import uuid
 import datetime
@@ -75,6 +75,7 @@ def index(request):
         project.funded_by = ''
         project.is_biccn = False
         project.save()
+        save_project_id(project)
         project_people = ProjectPeople()
         project_people.project_id = project
         project_people.people_id = people
@@ -290,7 +291,7 @@ def create_project(request):
         # write project to the project table   
         project = Project(funded_by=funded_by, name=name)
         project.save()
-
+        save_project_id(project)
         proj_id = project.id
 
         for c in consortia_ids:
@@ -2460,6 +2461,54 @@ def save_bil_ids(datasets):
         saved_bil_id.save()
     return
 
+def save_specimen_ids(specimens):
+    """
+    This function iterates through the provided list of specimens, generates and saves bil_specimen_IDs
+    using the bil_specimen_ID model. It also associates an MNE ID with each bil_specimen_ID and saves the updated
+    bil_specimen_ID object in the database.
+    """
+    for specimen in specimens:
+        #create placeholder for BIL_specimen_ID
+        bil_spc_id = BIL_Specimen_ID(specimen_id = specimen)
+        bil_spc_id.save()
+        #grab the just created database ID and generate an mne id
+        saved_bil_spc_id = BIL_Specimen_ID.objects.get(specimen_id = specimen.id)
+        mne_id = Mne.specimen_num_to_mne(saved_bil_spc_id.id)
+        saved_bil_spc_id.bil_spc_id = mne_id
+        #final save
+        saved_bil_spc_id.save()
+    return
+
+def save_instrument_ids(instruments):
+    """
+    This function iterates through the provided list of instruments, generates and saves bil_instrument_IDs
+    using the bil_instrument_ID model. It also associates an MNE ID with each bil_instrument_ID and saves the updated
+    bil_instrument_ID object in the database.
+    """
+    for instrument in instruments:
+        #create placeholder for BIL_instrument_ID
+        bil_ins_id = BIL_Instrument_ID(instrument_id = instrument)
+        bil_ins_id.save()
+        #grab the just created database ID and generate an mne id
+        saved_bil_ins_id = BIL_Instrument_ID.objects.get(instrument_id = instrument.id)
+        mne_id = Mne.instrument_num_to_mne(saved_bil_ins_id.id)
+        saved_bil_ins_id.bil_ins_id = mne_id
+        #final save
+        saved_bil_ins_id.save()
+    return
+
+def save_project_id(project):
+    #create placeholder for BIL_instrument_ID
+    bil_prj_id = BIL_Project_ID(project_id = project)
+    bil_prj_id.save()
+    #grab the just created database ID and generate an mne id
+    saved_bil_prj_id = BIL_Project_ID.objects.get(project_id = project.id)
+    mne_id = Mne.project_num_to_mne(saved_bil_prj_id.id)
+    saved_bil_prj_id.bil_prj_id = mne_id
+    #final save
+    saved_bil_prj_id.save()
+    return
+
 def metadata_version_check(filename):
     version1 = False
     workbook=xlrd.open_workbook(filename)
@@ -2522,13 +2571,13 @@ def descriptive_metadata_upload(request):
         ingest_method = request.POST.get('ingest_method', False)
 
         if form.is_valid():
-            associated_collection = form.cleaned_data['associated_collection']
+            associated_submission = form.cleaned_data['associated_submission']
 
             # for production
-            datapath = associated_collection.data_path.replace("/lz/","/etc/")
+            #datapath = associated_collection.data_path.replace("/lz/","/etc/")
 
             # for development on vm
-            #datapath = '/Users/luketuite/shared_bil_dev' 
+            datapath = '/Users/luketuite/shared_bil_dev' 
 
             # for development locally
             # datapath = '/Users/ecp/Desktop/bil_metadata_uploads' 
@@ -2544,7 +2593,7 @@ def descriptive_metadata_upload(request):
 
             # using old metadata model for any old submissions (will eventually be deprecated)
             if version1 == True:
-                error = upload_descriptive_spreadsheet(filename, associated_collection, request)
+                error = upload_descriptive_spreadsheet(filename, associated_submission, request)
                 if error:
                     return redirect('ingest:descriptive_metadata_upload')
                 else:         
@@ -2559,7 +2608,7 @@ def descriptive_metadata_upload(request):
 
                 else:
                     saved = False
-                    collection = Collection.objects.get(name=associated_collection.name)
+                    collection = Collection.objects.get(name=associated_submission.name)
                     contributors = ingest_contributors_sheet(filename)
                     funders = ingest_funders_sheet(filename)
                     publications = ingest_publication_sheet(filename)
@@ -2575,27 +2624,47 @@ def descriptive_metadata_upload(request):
                         sheet = save_sheet_row(ingest_method, filename, collection)
                         saved = save_all_sheets_method_1(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications)
                         ingested_datasets = Dataset.objects.filter(sheet = sheet)
+                        ingested_specimens = Specimen.objects.filter(sheet = sheet)
+                        ingested_instruments = Instrument.objects.filter(sheet = sheet)
                         save_bil_ids(ingested_datasets)
+                        save_specimen_ids(ingested_specimens)
+                        save_instrument_ids(ingested_instruments)
                     elif ingest_method == 'ingest_2':
                         sheet = save_sheet_row(ingest_method, filename, collection)
                         saved = save_all_sheets_method_2(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications)
                         ingested_datasets = Dataset.objects.filter(sheet = sheet)
+                        ingested_specimens = Specimen.objects.filter(sheet = sheet)
+                        ingested_instruments = Instrument.objects.filter(sheet = sheet)
                         save_bil_ids(ingested_datasets)
+                        save_specimen_ids(ingested_specimens)
+                        save_instrument_ids(ingested_instruments)
                     elif ingest_method == 'ingest_3':
                         sheet = save_sheet_row(ingest_method, filename, collection)
                         saved = save_all_sheets_method_3(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications)
                         ingested_datasets = Dataset.objects.filter(sheet = sheet)
+                        ingested_specimens = Specimen.objects.filter(sheet = sheet)
+                        ingested_instruments = Instrument.objects.filter(sheet = sheet)
                         save_bil_ids(ingested_datasets)
+                        save_specimen_ids(ingested_specimens)
+                        save_instrument_ids(ingested_instruments)
                     elif ingest_method == 'ingest_4':
                         sheet = save_sheet_row(ingest_method, filename, collection)
                         saved = save_all_sheets_method_4(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications)
                         ingested_datasets = Dataset.objects.filter(sheet = sheet)
+                        ingested_specimens = Specimen.objects.filter(sheet = sheet)
+                        ingested_instruments = Instrument.objects.filter(sheet = sheet)
                         save_bil_ids(ingested_datasets)
+                        save_specimen_ids(ingested_specimens)
+                        save_instrument_ids(ingested_instruments)
                     elif ingest_method == 'ingest_5':
                         sheet = save_sheet_row(ingest_method, filename, collection)
                         saved = save_all_sheets_method_5(instruments, specimen_set, datasets, sheet, contributors, funders, publications, swcs)
                         ingested_datasets = Dataset.objects.filter(sheet = sheet)
+                        ingested_specimens = Specimen.objects.filter(sheet = sheet)
+                        ingested_instruments = Instrument.objects.filter(sheet = sheet)
                         save_bil_ids(ingested_datasets)
+                        save_specimen_ids(ingested_specimens)
+                        save_instrument_ids(ingested_instruments)
                     elif ingest_method != 'ingest_1' and ingest_method != 'ingest_2' and ingest_method != 'ingest_3' and ingest_method != 'ingest_4' and ingest_method != 'ingest_5':
                          saved = False
                          messages.error(request, 'You must choose a value from "Step 2 of 3: What does your data look like?"')                         
@@ -2619,14 +2688,14 @@ def descriptive_metadata_upload(request):
         user = request.user
         form = UploadForm()
         # Only let a user associate metadata with an unlocked collection that they own
-        form.fields['associated_collection'].queryset = Collection.objects.filter(
+        form.fields['associated_submission'].queryset = Collection.objects.filter(
             locked=False, user=request.user)
-        collections = form.fields['associated_collection'].queryset
+        collections = form.fields['associated_submission'].queryset
     collections = Collection.objects.filter(locked=False, user=request.user)
 
     return render( request, 'ingest/descriptive_metadata_upload.html',{'form': form, 'pi':pi, 'collections':collections})
 
-def upload_descriptive_spreadsheet(filename, associated_collection, request):
+def upload_descriptive_spreadsheet(filename, associated_submission, request):
     """ Helper used by image_metadata_upload and collection_detail."""
     workbook=xlrd.open_workbook(filename)
     worksheet = workbook.sheet_by_index(0)
@@ -2687,7 +2756,7 @@ def upload_descriptive_spreadsheet(filename, associated_collection, request):
         records = pe.iget_records(file_name=filename)
         for idx, record in enumerate(records):
             im = DescriptiveMetadata(
-                collection=associated_collection,
+                collection=associated_submission,
                 user=request.user)
             for k in record:
                 setattr(im, k, record[k])
@@ -2701,7 +2770,7 @@ def upload_descriptive_spreadsheet(filename, associated_collection, request):
         return error
 
 # This gets called in the descriptive_metadata_upload function but we've commented that out to use upload_all_metadata_sheets instead, but prob will harvest some code from here. don't remove yet.
-def upload_spreadsheet(spreadsheet_file, associated_collection, request):
+def upload_spreadsheet(spreadsheet_file, associated_submission, request):
     """ Helper used by metadata_upload and collection_detail."""
     fs = FileSystemStorage()
     filename = fs.save(spreadsheet_file.name, spreadsheet_file)
@@ -2733,7 +2802,7 @@ def upload_spreadsheet(spreadsheet_file, associated_collection, request):
             if record['age'] == '':
                 record['age'] = None
             im = ImageMetadata(
-                collection=associated_collection,
+                collection=associated_submission,
                 user=request.user)
             for k in record:
                 setattr(im, k, record[k])