diff --git a/ingest/forms.py b/ingest/forms.py index 6cef538..b01f6e5 100644 --- a/ingest/forms.py +++ b/ingest/forms.py @@ -4,7 +4,7 @@ class UploadForm(forms.Form): - associated_collection = forms.ModelChoiceField(queryset=Collection.objects.all()) + associated_submission = forms.ModelChoiceField(queryset=Collection.objects.all()) class DescriptiveMetadataForm(forms.ModelForm): diff --git a/ingest/migrations/0027_auto_20240205_1642.py b/ingest/migrations/0027_auto_20240205_1642.py new file mode 100644 index 0000000..f815f45 --- /dev/null +++ b/ingest/migrations/0027_auto_20240205_1642.py @@ -0,0 +1,46 @@ +# Generated by Django 3.2.18 on 2024-02-05 16:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ingest', '0026_datasetlinkage'), + ] + + operations = [ + migrations.AlterModelOptions( + name='bil_id', + options={'verbose_name': 'BIL ID', 'verbose_name_plural': 'BIL IDs'}, + ), + migrations.AlterModelOptions( + name='dataset', + options={'verbose_name': 'Dataset', 'verbose_name_plural': 'Datasets'}, + ), + migrations.CreateModel( + name='BIL_Specimen_ID', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('bil_spc_id', models.CharField(blank=True, max_length=256, null=True)), + ('specimen_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.specimen')), + ], + ), + migrations.CreateModel( + name='BIL_Project_ID', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('bil_prj_id', models.CharField(blank=True, max_length=256, null=True)), + ('project_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.project')), + ], + ), + migrations.CreateModel( + name='BIL_Instrument_ID', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('bil_ins_id', models.CharField(blank=True, max_length=256, null=True)), + ('instrument_id', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='ingest.instrument')), + ], + ), + ] diff --git a/ingest/mne.py b/ingest/mne.py index e354dce..a423495 100644 --- a/ingest/mne.py +++ b/ingest/mne.py @@ -171,6 +171,97 @@ def dataset_num_to_mne(num): rstring=rstring + '-' rstring=rstring+Mne.DATAA[element] return rstring + + @staticmethod + def specimen_num_to_mne(num): + + #----------------------------------- + #store 256 decoded elements in stack + #----------------------------------- + stack=[] + q=num + if q < 256 : + q,r = divmod(num,256) + stack.append(r) + else: + while q > 255 : + q,r = divmod(q,256) + stack.append(r) + stack.append(q) + #---------------------- + #Finally code the stack + #---------------------- + rstring="" + first=True + for element in reversed(stack): + if first: + first=False + else: + rstring=rstring + '-' + rstring=rstring+Mne.DATAA[element] + rstring = "spc_"+rstring + return rstring + + @staticmethod + def instrument_num_to_mne(num): + + #----------------------------------- + #store 256 decoded elements in stack + #----------------------------------- + stack=[] + q=num + if q < 256 : + q,r = divmod(num,256) + stack.append(r) + else: + while q > 255 : + q,r = divmod(q,256) + stack.append(r) + stack.append(q) + #---------------------- + #Finally code the stack + #---------------------- + rstring="" + first=True + for element in reversed(stack): + if first: + first=False + else: + rstring=rstring + '-' + rstring=rstring+Mne.DATAA[element] + rstring = "ins_" + rstring + return rstring + + @staticmethod + def project_num_to_mne(num): + + #----------------------------------- + #store 256 decoded elements in stack + #----------------------------------- + stack=[] + q=num + if q < 256 : + q,r = divmod(num,256) + stack.append(r) + else: + while q > 255 : + q,r = divmod(q,256) + stack.append(r) + stack.append(q) + #---------------------- + #Finally code the stack + #---------------------- + rstring="" + first=True + for element in reversed(stack): + if first: + first=False + else: + rstring=rstring + '-' + rstring=rstring+Mne.DATAA[element] + rstring = "prj_" + rstring + return rstring + def mne_to_num(mme): stack=[] diff --git a/ingest/models.py b/ingest/models.py index 0a6414c..a5f5541 100644 --- a/ingest/models.py +++ b/ingest/models.py @@ -508,4 +508,16 @@ class DatasetLinkage(models.Model): data_id_2 = models.CharField(max_length=256, blank=True, null=True) relationship = models.CharField(max_length=64, default="", choices=[('sequence data', 'Sequence Data'), ('neuron tracing', 'Neuron Tracing'), ('derived_data', 'Derived Data'), ('raw', 'Raw'), ('aligned', 'Aligned')]) description = models.TextField(blank=True, null=True) - linkage_date = models.DateField(null=True, blank=True) \ No newline at end of file + linkage_date = models.DateField(null=True, blank=True) + +class BIL_Specimen_ID(models.Model): + bil_spc_id = models.CharField(max_length=256, blank=True, null=True) + specimen_id = models.ForeignKey(Specimen, on_delete=models.SET_NULL, null = True, blank=True) + +class BIL_Instrument_ID(models.Model): + bil_ins_id = models.CharField(max_length=256, blank=True, null=True) + instrument_id = models.ForeignKey(Instrument, on_delete=models.SET_NULL, null = True, blank=True) + +class BIL_Project_ID(models.Model): + bil_prj_id = models.CharField(max_length=256, blank=True, null=True) + project_id = models.ForeignKey(Project, on_delete=models.SET_NULL, null = True, blank=True) \ No newline at end of file diff --git a/ingest/templates/ingest/descriptive_metadata_upload.html b/ingest/templates/ingest/descriptive_metadata_upload.html index 41094ac..4665d9d 100644 --- a/ingest/templates/ingest/descriptive_metadata_upload.html +++ b/ingest/templates/ingest/descriptive_metadata_upload.html @@ -26,7 +26,7 @@

Step 3 of 3: Upload metadata for associated submission

{% if collections %}
{% csrf_token %} diff --git a/ingest/views.py b/ingest/views.py index f6331ce..736fb3c 100644 --- a/ingest/views.py +++ b/ingest/views.py @@ -28,7 +28,7 @@ from .field_list import required_metadata from .filters import CollectionFilter from .forms import CollectionForm, ImageMetadataForm, DescriptiveMetadataForm, UploadForm, collection_send -from .models import UUID, Collection, ImageMetadata, DescriptiveMetadata, Project, ProjectPeople, People, Project, EventsLog, Contributor, Funder, Publication, Instrument, Dataset, Specimen, Image, Sheet, Consortium, ProjectConsortium, SWC, ProjectAssociation, BIL_ID, DatasetEventsLog +from .models import UUID, Collection, ImageMetadata, DescriptiveMetadata, Project, ProjectPeople, People, Project, EventsLog, Contributor, Funder, Publication, Instrument, Dataset, Specimen, Image, Sheet, Consortium, ProjectConsortium, SWC, ProjectAssociation, BIL_ID, DatasetEventsLog, BIL_Specimen_ID, BIL_Instrument_ID, BIL_Project_ID from .tables import CollectionTable, DescriptiveMetadataTable, CollectionRequestTable import uuid import datetime @@ -75,6 +75,7 @@ def index(request): project.funded_by = '' project.is_biccn = False project.save() + save_project_id(project) project_people = ProjectPeople() project_people.project_id = project project_people.people_id = people @@ -290,7 +291,7 @@ def create_project(request): # write project to the project table project = Project(funded_by=funded_by, name=name) project.save() - + save_project_id(project) proj_id = project.id for c in consortia_ids: @@ -2460,6 +2461,54 @@ def save_bil_ids(datasets): saved_bil_id.save() return +def save_specimen_ids(specimens): + """ + This function iterates through the provided list of specimens, generates and saves bil_specimen_IDs + using the bil_specimen_ID model. It also associates an MNE ID with each bil_specimen_ID and saves the updated + bil_specimen_ID object in the database. + """ + for specimen in specimens: + #create placeholder for BIL_specimen_ID + bil_spc_id = BIL_Specimen_ID(specimen_id = specimen) + bil_spc_id.save() + #grab the just created database ID and generate an mne id + saved_bil_spc_id = BIL_Specimen_ID.objects.get(specimen_id = specimen.id) + mne_id = Mne.specimen_num_to_mne(saved_bil_spc_id.id) + saved_bil_spc_id.bil_spc_id = mne_id + #final save + saved_bil_spc_id.save() + return + +def save_instrument_ids(instruments): + """ + This function iterates through the provided list of instruments, generates and saves bil_instrument_IDs + using the bil_instrument_ID model. It also associates an MNE ID with each bil_instrument_ID and saves the updated + bil_instrument_ID object in the database. + """ + for instrument in instruments: + #create placeholder for BIL_instrument_ID + bil_ins_id = BIL_Instrument_ID(instrument_id = instrument) + bil_ins_id.save() + #grab the just created database ID and generate an mne id + saved_bil_ins_id = BIL_Instrument_ID.objects.get(instrument_id = instrument.id) + mne_id = Mne.instrument_num_to_mne(saved_bil_ins_id.id) + saved_bil_ins_id.bil_ins_id = mne_id + #final save + saved_bil_ins_id.save() + return + +def save_project_id(project): + #create placeholder for BIL_instrument_ID + bil_prj_id = BIL_Project_ID(project_id = project) + bil_prj_id.save() + #grab the just created database ID and generate an mne id + saved_bil_prj_id = BIL_Project_ID.objects.get(project_id = project.id) + mne_id = Mne.project_num_to_mne(saved_bil_prj_id.id) + saved_bil_prj_id.bil_prj_id = mne_id + #final save + saved_bil_prj_id.save() + return + def metadata_version_check(filename): version1 = False workbook=xlrd.open_workbook(filename) @@ -2522,13 +2571,13 @@ def descriptive_metadata_upload(request): ingest_method = request.POST.get('ingest_method', False) if form.is_valid(): - associated_collection = form.cleaned_data['associated_collection'] + associated_submission = form.cleaned_data['associated_submission'] # for production - datapath = associated_collection.data_path.replace("/lz/","/etc/") + #datapath = associated_collection.data_path.replace("/lz/","/etc/") # for development on vm - #datapath = '/Users/luketuite/shared_bil_dev' + datapath = '/Users/luketuite/shared_bil_dev' # for development locally # datapath = '/Users/ecp/Desktop/bil_metadata_uploads' @@ -2544,7 +2593,7 @@ def descriptive_metadata_upload(request): # using old metadata model for any old submissions (will eventually be deprecated) if version1 == True: - error = upload_descriptive_spreadsheet(filename, associated_collection, request) + error = upload_descriptive_spreadsheet(filename, associated_submission, request) if error: return redirect('ingest:descriptive_metadata_upload') else: @@ -2559,7 +2608,7 @@ def descriptive_metadata_upload(request): else: saved = False - collection = Collection.objects.get(name=associated_collection.name) + collection = Collection.objects.get(name=associated_submission.name) contributors = ingest_contributors_sheet(filename) funders = ingest_funders_sheet(filename) publications = ingest_publication_sheet(filename) @@ -2575,27 +2624,47 @@ def descriptive_metadata_upload(request): sheet = save_sheet_row(ingest_method, filename, collection) saved = save_all_sheets_method_1(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications) ingested_datasets = Dataset.objects.filter(sheet = sheet) + ingested_specimens = Specimen.objects.filter(sheet = sheet) + ingested_instruments = Instrument.objects.filter(sheet = sheet) save_bil_ids(ingested_datasets) + save_specimen_ids(ingested_specimens) + save_instrument_ids(ingested_instruments) elif ingest_method == 'ingest_2': sheet = save_sheet_row(ingest_method, filename, collection) saved = save_all_sheets_method_2(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications) ingested_datasets = Dataset.objects.filter(sheet = sheet) + ingested_specimens = Specimen.objects.filter(sheet = sheet) + ingested_instruments = Instrument.objects.filter(sheet = sheet) save_bil_ids(ingested_datasets) + save_specimen_ids(ingested_specimens) + save_instrument_ids(ingested_instruments) elif ingest_method == 'ingest_3': sheet = save_sheet_row(ingest_method, filename, collection) saved = save_all_sheets_method_3(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications) ingested_datasets = Dataset.objects.filter(sheet = sheet) + ingested_specimens = Specimen.objects.filter(sheet = sheet) + ingested_instruments = Instrument.objects.filter(sheet = sheet) save_bil_ids(ingested_datasets) + save_specimen_ids(ingested_specimens) + save_instrument_ids(ingested_instruments) elif ingest_method == 'ingest_4': sheet = save_sheet_row(ingest_method, filename, collection) saved = save_all_sheets_method_4(instruments, specimen_set, images, datasets, sheet, contributors, funders, publications) ingested_datasets = Dataset.objects.filter(sheet = sheet) + ingested_specimens = Specimen.objects.filter(sheet = sheet) + ingested_instruments = Instrument.objects.filter(sheet = sheet) save_bil_ids(ingested_datasets) + save_specimen_ids(ingested_specimens) + save_instrument_ids(ingested_instruments) elif ingest_method == 'ingest_5': sheet = save_sheet_row(ingest_method, filename, collection) saved = save_all_sheets_method_5(instruments, specimen_set, datasets, sheet, contributors, funders, publications, swcs) ingested_datasets = Dataset.objects.filter(sheet = sheet) + ingested_specimens = Specimen.objects.filter(sheet = sheet) + ingested_instruments = Instrument.objects.filter(sheet = sheet) save_bil_ids(ingested_datasets) + save_specimen_ids(ingested_specimens) + save_instrument_ids(ingested_instruments) elif ingest_method != 'ingest_1' and ingest_method != 'ingest_2' and ingest_method != 'ingest_3' and ingest_method != 'ingest_4' and ingest_method != 'ingest_5': saved = False messages.error(request, 'You must choose a value from "Step 2 of 3: What does your data look like?"') @@ -2619,14 +2688,14 @@ def descriptive_metadata_upload(request): user = request.user form = UploadForm() # Only let a user associate metadata with an unlocked collection that they own - form.fields['associated_collection'].queryset = Collection.objects.filter( + form.fields['associated_submission'].queryset = Collection.objects.filter( locked=False, user=request.user) - collections = form.fields['associated_collection'].queryset + collections = form.fields['associated_submission'].queryset collections = Collection.objects.filter(locked=False, user=request.user) return render( request, 'ingest/descriptive_metadata_upload.html',{'form': form, 'pi':pi, 'collections':collections}) -def upload_descriptive_spreadsheet(filename, associated_collection, request): +def upload_descriptive_spreadsheet(filename, associated_submission, request): """ Helper used by image_metadata_upload and collection_detail.""" workbook=xlrd.open_workbook(filename) worksheet = workbook.sheet_by_index(0) @@ -2687,7 +2756,7 @@ def upload_descriptive_spreadsheet(filename, associated_collection, request): records = pe.iget_records(file_name=filename) for idx, record in enumerate(records): im = DescriptiveMetadata( - collection=associated_collection, + collection=associated_submission, user=request.user) for k in record: setattr(im, k, record[k]) @@ -2701,7 +2770,7 @@ def upload_descriptive_spreadsheet(filename, associated_collection, request): return error # This gets called in the descriptive_metadata_upload function but we've commented that out to use upload_all_metadata_sheets instead, but prob will harvest some code from here. don't remove yet. -def upload_spreadsheet(spreadsheet_file, associated_collection, request): +def upload_spreadsheet(spreadsheet_file, associated_submission, request): """ Helper used by metadata_upload and collection_detail.""" fs = FileSystemStorage() filename = fs.save(spreadsheet_file.name, spreadsheet_file) @@ -2733,7 +2802,7 @@ def upload_spreadsheet(spreadsheet_file, associated_collection, request): if record['age'] == '': record['age'] = None im = ImageMetadata( - collection=associated_collection, + collection=associated_submission, user=request.user) for k in record: setattr(im, k, record[k])