add support for pyifcb require_roi_files flag

ifcbdb/dashboard/accession.py

@@ -1,3 +1,4 @@
+from pprint import pprint
 import os
 import json
 import time
@@ -59,21 +60,27 @@ def scan(self):
         for dd in self.dataset.directories.filter(kind=DataDirectory.RAW).order_by('priority'):
             if not os.path.exists(dd.path):
                 continue # skip and continue searching
-            directory = ifcb.DataDirectory(dd.path)
+
+            directory = ifcb.DataDirectory(dd.path, require_roi_files=dd.require_roi_files)
             for b in directory:
+                # Bins do not have a require_roi_files property, but we're adding it here anyway to pass
+                #   along the associated directory's setting through to the next call
+                b.require_roi_files = dd.require_roi_files
                 yield b
     def sync_one(self, pid):
         bin = None
+        directory = None
         for dd in self.dataset.directories.filter(kind=DataDirectory.RAW).order_by('priority'):
             if not os.path.exists(dd.path):
                 continue # skip and continue searching
-            directory = ifcb.DataDirectory(dd.path)
+            directory = ifcb.DataDirectory(dd.path, require_roi_files=dd.require_roi_files)
             try:
                 bin = directory[pid]
             except KeyError:
                 continue
         if bin is None:
             return 'bin {} not found'.format(pid)
+
         # create instrument if necessary
         i = bin.pid.instrument
         version = bin.pid.schema_version
@@ -161,6 +168,7 @@ def sync(self, progress_callback=do_nothing, log_callback=do_nothing):
                     bad_bins += 1
                 else:
                     log_callback('{} not adding bin'.format(b.pid))
+
             with transaction.atomic():
                 for b in bins2save:
                     b.skip = False # unskip because we're ready to save
@@ -181,13 +189,13 @@ def sync(self, progress_callback=do_nothing, log_callback=do_nothing):
         progress_callback(prog)
         return prog
 
-    def add_bin(self, bin, b): # IFCB bin, Bin instance
+    def add_bin(self, bin, b, require_roi_files=True): # IFCB bin, Bin instance
         # qaqc checks
         qc_bad = check_bad(bin)
         if qc_bad:
             b.qc_bad = True
             return b, 'malformed raw data'
-        no_rois = check_no_rois(bin)
+        no_rois = require_roi_files and check_no_rois(bin)
         if no_rois:
             b.qc_bad = True
             return b, 'zero ROIs'
@@ -227,7 +235,7 @@ def add_bin(self, bin, b): # IFCB bin, Bin instance
             if latitude is not None and longitude is not None:
                 b.set_location(longitude, latitude, depth)
         #
-        b.qc_no_rois = check_no_rois(bin)
+        b.qc_no_rois = require_roi_files and check_no_rois(bin)
         # metrics
         try:
             b.temperature = bin.temperature

ifcbdb/dashboard/migrations/0039_datadirectory_require_roi_files.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.20 on 2025-04-17 20:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dashboard', '0038_remove_datadirectory_unique_path_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datadirectory',
+            name='require_roi_files',
+            field=models.BooleanField(default=True),
+        ),
+    ]

ifcbdb/dashboard/models.py

@@ -368,14 +368,15 @@ class DataDirectory(models.Model):
     blacklist = models.CharField(max_length=512, default='skip,bad') # comma separated list of directory names to skip
     # for product directories, the product version
     version = models.IntegerField(null=True, blank=True)
+    require_roi_files = models.BooleanField(default=True)
 
     def get_raw_directory(self):
         if self.kind != self.RAW:
             raise ValueError('not a raw directory')
         # return the underlying ifcb.DataDirectory
         whitelist = re.split(',', self.whitelist)
         blacklist = re.split(',', self.blacklist)
-        return ifcb.DataDirectory(self.path, whitelist=whitelist, blacklist=blacklist)
+        return ifcb.DataDirectory(self.path, whitelist=whitelist, blacklist=blacklist, require_roi_files=self.require_roi_files)
 
     def raw_destination(self, bin_id):
         # where to put an incoming bin with the given id

ifcbdb/secure/forms.py

@@ -128,7 +128,7 @@ def _match_folder_names(self, value):
 
     class Meta:
         model = DataDirectory
-        fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", ]
+        fields = ["id", "path", "kind", "priority", "whitelist", "blacklist", "version", "require_roi_files", ]
 
         widgets = {
             "path": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Path"}),
@@ -144,6 +144,7 @@ class Meta:
             "blacklist": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Blacklist"}),
             "version": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Version"}),
             "priority": forms.TextInput(attrs={"class": "form-control form-control-sm", "placeholder": "Priority"}),
+            "require_roi_files": forms.CheckboxInput(attrs={"class": "custom-control-input"})
         }
 
 

ifcbdb/secure/views.py

@@ -59,7 +59,7 @@ def dt_datasets(request):
 @login_required
 def dt_directories(request, dataset_id):
     directories = list(DataDirectory.objects.filter(dataset__id=dataset_id)
-                       .values_list("path", "kind", "priority", "whitelist", "blacklist", "id"))
+                       .values_list("path", "kind", "priority", "whitelist", "blacklist", "require_roi_files", "id"))
 
     return JsonResponse({
         "data": directories,

ifcbdb/templates/secure/directory-management.html

@@ -22,6 +22,7 @@
                     <th>Priority</th>
                     <th>Whitelist</th>
                     <th>Blacklist</th>
+                    <th>Require ROI Files</th>
                     <th></th>
                 </tr>
             </thead>
@@ -75,6 +76,9 @@ <h5 class="modal-title" id="directoryDeleteModalLabel">Confirm Removal</h5>
                 {}, // Priority
                 {}, // Whitelist
                 {}, // Blacklist
+                {
+                    render: function(data) { return data ? "Yes" : "No" }
+                }, // Require ROI Files
                 { // Edit
                     targets: -1,
                     render: function ( data, type, row ) {

ifcbdb/templates/secure/edit-directory.html

@@ -64,6 +64,15 @@
             {% if form.priority.errors %}<span class="text-danger">{{ form.priority.errors.as_text }}</span>{% endif %}
         </div>
     </div>
+    <div class="form-row">
+        <div class="form-group col">
+            <label for="id_require_roi_files">Require ROI Files</label>
+            <div class="col custom-switch">
+                {{ form.require_roi_files }}
+                <label class="custom-control-label" for="id_require_roi_files"></label>
+            </div>
+        </div>
+    </div>
     <div class="form-row">
         <a href="{% url 'secure:directory-management' dataset_id %}" class="btn btn-sm">Cancel</a>
         <button type="submit" class="btn btn-sm btn-mdb-color">Save</button>

requirements.txt

@@ -5,13 +5,13 @@ numba==0.59.1
 celery==5.3.6
 pymemcache==4.0.0
 redis==5.0.3
-scipy==1.12.0
-pandas==2.2.1
-h5py==3.10.0
-requests==2.32.0
-Pillow==10.3.0
+scipy==1.13.1
+pandas==2.2.3
+h5py==3.12.1
+requests==2.32.3
+Pillow==11.1.0
 rectpack==0.2.2
-scikit-image==0.22.0
-pysmb==1.2.9.1
-pyyaml==6.0.1
-git+https://github.com/joefutrelle/pyifcb@v1.1.1
+scikit-image==0.24.0
+pysmb==1.2.10
+pyyaml==6.0.2
+git+https://github.com/joefutrelle/pyifcb@v1.2.1