initial nabat models/api script (#122)

* initial nabat models/api script

* model updates and reset endpoints

* outlining tasks

* nabat retrieval logic

* NABat ingestion working

* rename acoustic_batch_initialize

* remove flower, add spectrogram viewer script, move spectrogram creation to tasks,  update NABat spectro generating process

* nabat integration

* deduplicating functions

* add species manual and auto ids to nabat retrieval

* non-compressed X-axis highlight, zooming legend fix, 2.5x default compressed

* cursor time compressed fix, axis length fix, edit column reorder in Recordings

* recording annotation details

* removing some commented out emits

* remove footer from details for inference annotations

* Ignore egg-info directories

* Add additional configuration settings

* Add new settings to admin page

* Use configuration to set `compressed` ref

* Move save button closer to settings

* Check config setting before running inference

* Use config setting for initial compressed scale

* reference the outputs.tags instead of outputs.results

* renames acoustic batch to recording (#128)

* renames acoustic batch to recording based on conversations

* add migrations

* Use slider for stretch setting

* recording based nabat spectrograms

* migrations modifications

* public apis, adding apiToken access control

* recording annotation endpoints

* nabat local annotaitons

* precition config, requesting annotations, updating species

* swapping to user_email from user_id

* pushing annotations back

* auth cleanup

* convert to mutation

* Update scripts/USGS/naBatSpecies.py

Co-authored-by: Michael Nagler <mike.nagler@kitware.com>

* addressing some of the comments

* enum for processing task types, processing task get_or_create

* client_id option, base environment file, updating instructions, NABAT_API_URL environment value

* SSL verification hack

* fixing get_or_create for processing tasks in celery tasks

* resolve lint warnings

* swap to logger.error in nabat recording creation

---------

Co-authored-by: naglepuff <mike.nagler@kitware.com>

Author: BryonLewis

DEPLOYMENT.md

@@ -45,9 +45,11 @@ to `.env` and change the default passwords for fields
    to collect the static files
 6. Run `docker compose -f docker-compose.prod.yml up` to start the server
    add `-d` for a silent version to run in the background
-7. Change the ID in the `./client/env.production` to a custom ID - this will
+7. **OPTIONAL** Change the ID in the `./client/env.production` to a custom ID - this will
    probably require a `docker compose -f docker-compose.prod.yml build` \
-   to build the app afterwards
+   to build the app afterwards.  This Id is used to indetify the application and
+   isn't required to be changed especially if the building of the client is done
+   outside of deployment.
 8. After creating the basic application log into the django admin `batdetectai.kitware.com/admin`
    and change the ApplicationId to the ID in the `./client.env.production`
 9. Test logging in/out and uploading data to the server.

bats_ai/api.py

@@ -7,10 +7,12 @@
     ConfigurationRouter,
     GRTSCellsRouter,
     GuanoMetadataRouter,
+    ProcessingTaskRouter,
     RecordingAnnotationRouter,
     RecordingRouter,
     SpeciesRouter,
 )
+from bats_ai.core.views.nabat import NABatConfigurationRouter, NABatRecordingRouter
 
 logger = logging.getLogger(__name__)
 
@@ -37,3 +39,6 @@ def global_auth(request):
 api.add_router('/guano/', GuanoMetadataRouter)
 api.add_router('/recording-annotation/', RecordingAnnotationRouter)
 api.add_router('/configuration/', ConfigurationRouter)
+api.add_router('/processing-task/', ProcessingTaskRouter)
+api.add_router('/nabat/recording/', NABatRecordingRouter)
+api.add_router('/nabat/configuration/', NABatConfigurationRouter)

bats_ai/core/admin/__init__.py

@@ -1,7 +1,15 @@
 from .annotations import AnnotationsAdmin
 from .compressed_spectrogram import CompressedSpectrogramAdmin
+from .configuration import ConfigurationAdmin
 from .grts_cells import GRTSCellsAdmin
 from .image import ImageAdmin
+from .nabat.admin import (
+    NABatCompressedSpectrogramAdmin,
+    NABatRecordingAdmin,
+    NABatRecordingAnnotationAdmin,
+    NABatSpectrogramAdmin,
+)
+from .processing_task import ProcessingTaskAdmin
 from .recording import RecordingAdmin
 from .recording_annotations import RecordingAnnotationAdmin
 from .species import SpeciesAdmin
@@ -18,4 +26,11 @@
     'GRTSCellsAdmin',
     'CompressedSpectrogramAdmin',
     'RecordingAnnotationAdmin',
+    'ProcessingTaskAdmin',
+    'ConfigurationAdmin',
+    # NABat Models
+    'NABatRecordingAnnotationAdmin',
+    'NABatCompressedSpectrogramAdmin',
+    'NABatSpectrogramAdmin',
+    'NABatRecordingAdmin',
 ]

bats_ai/core/admin/configuration.py

@@ -0,0 +1,24 @@
+from django.contrib import admin
+
+from bats_ai.core.models.configuration import Configuration
+
+
+@admin.register(Configuration)
+class ConfigurationAdmin(admin.ModelAdmin):
+    list_display = (
+        'display_pulse_annotations',
+        'display_sequence_annotations',
+        'run_inference_on_upload',
+        'spectrogram_x_stretch',
+        'spectrogram_view',
+    )
+
+    def has_add_permission(self, request):
+        # Allow add only if there is no Configuration instance
+        if Configuration.objects.exists():
+            return False
+        return super().has_add_permission(request)
+
+    def has_delete_permission(self, request, obj=None):
+        # Prevent deleting the Configuration through the admin
+        return False

bats_ai/core/admin/image.py

@@ -3,7 +3,7 @@
 from django.http import HttpRequest
 
 from bats_ai.core.models import Image
-from bats_ai.core.tasks import image_compute_checksum
+from bats_ai.tasks.tasks import image_compute_checksum
 
 
 @admin.register(Image)

bats_ai/core/admin/nabat/admin.py

@@ -0,0 +1,66 @@
+from django.contrib import admin
+
+from bats_ai.core.models.nabat import (
+    NABatCompressedSpectrogram,
+    NABatRecording,
+    NABatRecordingAnnotation,
+    NABatSpectrogram,
+)
+
+
+# Register models for the NaBat category
+@admin.register(NABatRecordingAnnotation)
+class NABatRecordingAnnotationAdmin(admin.ModelAdmin):
+    list_display = (
+        'nabat_recording',
+        'comments',
+        'model',
+        'confidence',
+        'additional_data',
+        'species_codes',
+    )
+    search_fields = ('nabat_recording_name', 'comments', 'model')
+    list_filter = ('nabat_recording',)
+
+    @admin.display(description='Species Codes')
+    def species_codes(self, obj):
+        # Assuming species have a `species_code` field
+        return ', '.join([species.species_code for species in obj.species.all()])
+
+
+@admin.register(NABatSpectrogram)
+class NABatSpectrogramAdmin(admin.ModelAdmin):
+    list_display = (
+        'nabat_recording',
+        'image_file',
+        'width',
+        'height',
+        'duration',
+        'frequency_min',
+        'frequency_max',
+        'colormap',
+    )
+    search_fields = ('nabat_recording__name', 'colormap')
+    list_filter = ('nabat_recording', 'colormap')
+
+
+@admin.register(NABatCompressedSpectrogram)
+class NABatCompressedSpectrogramAdmin(admin.ModelAdmin):
+    list_display = ('nabat_recording', 'spectrogram', 'length', 'cache_invalidated')
+    search_fields = ('nabat_recording__name', 'spectrogram__id')
+    list_filter = ('nabat_recording', 'cache_invalidated')
+
+
+@admin.register(NABatRecording)
+class NABatRecordingAdmin(admin.ModelAdmin):
+    list_display = (
+        'name',
+        'recording_id',
+        'equipment',
+        'comments',
+        'recording_location',
+        'grts_cell_id',
+        'grts_cell',
+    )
+    search_fields = ('name', 'recording_id', 'recording_location')
+    list_filter = ('name', 'recording_id', 'recording_location')

bats_ai/core/admin/processing_task.py

@@ -0,0 +1,28 @@
+from django.contrib import admin
+
+from bats_ai.core.models import ProcessingTask
+
+
+@admin.register(ProcessingTask)
+class ProcessingTaskAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'status', 'created', 'modified', 'celery_id', 'metadata', 'error')
+    list_filter = ('status', 'created', 'modified')
+    search_fields = ('name', 'celery_id', 'metadata', 'error')
+    ordering = ('-created',)
+    readonly_fields = ('created', 'modified')
+    fieldsets = (
+        (None, {'fields': ('name', 'status', 'celery_id', 'error')}),
+        (
+            'Metadata',
+            {
+                'classes': ('collapse',),
+                'fields': ('metadata',),
+            },
+        ),
+        (
+            'Timestamps',
+            {
+                'fields': ('created', 'modified'),
+            },
+        ),
+    )

bats_ai/core/admin/recording.py

@@ -5,7 +5,7 @@
 from django.utils.safestring import mark_safe
 
 from bats_ai.core.models import Recording
-from bats_ai.core.tasks import recording_compute_spectrogram
+from bats_ai.tasks.tasks import recording_compute_spectrogram
 
 
 @admin.register(Recording)

bats_ai/core/management/commands/makeclient.py

@@ -1,8 +1,10 @@
+import os
+
 from django.contrib.auth.models import User
 import djclick as click
 from oauth2_provider.models import Application
 
-CLIENT_ID = 'HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP'
+CLIENT_ID = os.environ.get('APPLICATION_CLIENT_ID', 'HSJWFZ2cIpWQOvNyCXyStV9hiOd7DfWeBOCzo4pP')
 
 
 # create django oauth toolkit appliction (client)
@@ -12,10 +14,23 @@
     required=True,
     help='superuser username for application creator',
 )
-@click.option('--uri', type=click.STRING, required=True, help='redirect uri for application')
+@click.option(
+    '--uri',
+    type=click.STRING,
+    default='http://localhost:3000/',
+    required=False,
+    help='redirect uri for application',
+)
+@click.option(
+    '--clientid',
+    type=click.STRING,
+    default=CLIENT_ID,
+    required=False,
+    help='clientID used in the application',
+)
 @click.command()
-def command(username, uri):
-    if Application.objects.filter(client_id=CLIENT_ID).exists():
+def command(username, uri, clientid):
+    if Application.objects.filter(client_id=clientid).exists():
         click.echo('The client already exists. You can administer it from the admin console.')
         return
 
@@ -29,7 +44,7 @@ def command(username, uri):
     if user:
         application = Application(
             name='batsai-client',
-            client_id=CLIENT_ID,
+            client_id=clientid,
             client_secret='',
             client_type='public',
             redirect_uris=uri,