diff --git a/api/serializers.py b/api/serializers.py index a4b9fb9..a9ca18f 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -19,7 +19,7 @@ class Meta: "audio_file", "start_time", "end_time", - "annotation", + "content", "timestamp", ) @@ -29,6 +29,6 @@ def create(self, validated_data): def update(self, instance, validated_data): instance.start_time = validated_data.get("start_time", instance.start_time) instance.end_time = validated_data.get("end_time", instance.end_time) - instance.annotation = validated_data.get("annotation", instance.annotation) + instance.content = validated_data.get("content", instance.content) instance.save() return instance diff --git a/docker-compose.yml b/docker-compose.yml index 5778e9a..4517532 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: interval: 10s timeout: 5s retries: 5 - entrypoint: ["./db-entrypoint.sh"] + entrypoint: ["/db-entrypoint.sh"] migrate: build: @@ -53,7 +53,6 @@ services: command: conda run --no-capture-output -n comedy-project-docker python manage.py migrate volumes: - .:/audio-annonation - - D:/anaconda-envs:/envs depends_on: db: condition: service_healthy diff --git a/pyproject.toml b/pyproject.toml index 9df25d6..284373a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,124 +1,4 @@ -[tool.poetry] -name = "audio-annonation" -version = "0.1.0" -description = "" -authors = ["Unnati Patel", "Christopher Keim"] -readme = "README.md" -packages = [ - { include = "api" }, - { include = "waveform_audio" }, -] - -[tool.poetry.dependencies] -python = ">=3.11, <3.12" - -# Web -requests = "^2.31.0" -urllib3 = "^2.0.2" -django = "^4.2.1" -djangorestframework = "^3.14.0" -crispy-tailwind = "^0.5.0" -whitenoise = "^6.6.0" -asgiref = "^3.6.0" -certifi = "^2023.5.7" -tzdata = "^2023.3" -python-dotenv = "^1.0.0" - -# Data Science -pandas = "^1.5.0" -numpy = "^1.24.3" -scipy = "^1.10.1" -scikit-learn = "^1.2.2" -matplotlib = "^3.6.0" -sqlparse = "^0.4.4" -audioread = "^3.0.0" -soundfile = "^0.12.1" -pooch = "^1.6.0" -soxr = "^0.3.5" -librosa = "^0.10.0.post2" - -# MLOps -threadpoolctl = "^3.1.0" -numba = "^0.57.0" -llvmlite = "^0.40.0" -cffi = "^1.15.1" - - - -[tool.poetry.group.dev] -optional = true -[tool.poetry.group.dev.dependencies] - -# DevOps -black = "^22.3.0" -pytest = "^7.4.0" -pytest-cov = "^4.1.0" -ruff = "^0.0.285" - -# PostgresSQL psycopg2 pre-compiled binary -psycopg2-binary = "^2.9.9" - - -[tool.poetry.group.prod] -optional = true -[tool.poetry.group.prod.dependencies] - -# PostgresSQL psycopg2 source build -psycopg2 = "^2.9.9" - - -[tool.ruff] -# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -select = ["E", "F"] -ignore = [] - -# Allow autofix for all enabled rules (when `--fix`) is provided. -fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] -unfixable = [] - -# Exclude a variety of commonly ignored directories. -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv", - "tests", -] - -# Same as Black. -line-length = 88 - -# Allow unused variables when underscore-prefixed. -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" - -# Assume Python 3.9 -target-version = "py39" - -[tool.ruff.mccabe] -# Unlike Flake8, default to a complexity level of 10. -max-complexity = 10 - [tool.pytest.ini_options] -# Configurations for pytest with coverage -addopts = "-vv --cov" - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +DJANGO_SETTINGS_MODULE = "waveform_audio.settings" +# -- recommended but optional: +python_files = ["test_*.py", "*_test.py", "testing/python/*.py"] diff --git a/waveform_audio/forms.py b/waveform_audio/forms.py index b89806e..ccac5aa 100644 --- a/waveform_audio/forms.py +++ b/waveform_audio/forms.py @@ -1,8 +1,12 @@ # import crispy_forms.helper as crispy_helper from django import forms -from .models import AudioFile +from django.utils.translation import gettext_lazy as _ + +from waveform_audio.models import AudioFile, Subtitle from crispy_forms.helper import FormHelper -from crispy_forms.layout import Submit # Layout, Row, Column +from crispy_forms.layout import Submit +from waveform_audio import utils +import mimetypes # url: https://stackoverflow.com/questions/24783275/django-form-with-choices-but-also-with-freetext-option?noredirect=1&lq=1 @@ -31,19 +35,30 @@ class Meta: model = AudioFile fields = ["file"] labels = { - "audio_file": "Select a file", + "file": "Select a file", } help_texts = { - "audio_file": "insert an audio file", + "file": "insert an audio file", } widgets = { - "audio_file": forms.FileInput(attrs={"accept": "audio/*"}), + "file": forms.FileInput(attrs={"accept": "audio/wav,audio/mp3,audio/mp4"}), } + def clean_file(self): + audio_file = self.cleaned_data.get("file") + if audio_file: + valid_mime_types = ["audio/mpeg", "audio/wav", "audio/mp3"] + mime_type, random_str = mimetypes.guess_type(audio_file.name) + if mime_type is None or mime_type not in valid_mime_types: + raise forms.ValidationError( + _("Invalid audio file type"), code="invalid_file_type" + ) + return audio_file + class SubtitleFileForm(forms.Form): subtitle_file = forms.FileField( - label="Select a file", + label="Select a subtitle file", help_text="insert a subtitle file", widget=forms.FileInput(attrs={"accept": ".srt"}), ) @@ -54,3 +69,26 @@ def helper(self): helper.form_method = "POST" helper.inputs.append(Submit("submit", "Submit")) return helper + + def clean_subtitle_file(self): + subtitle_file = self.cleaned_data.get("subtitle_file") + if subtitle_file: + try: + self.subtitle_texts = utils.process_subtitle_file(subtitle_file) + except Exception as e: + raise forms.ValidationError(f"Error processing subtitle file: {str(e)}") + return subtitle_file + + def save(self, audio_file_instance): + if hasattr(self, "subtitle_texts"): + subtitles = [ + Subtitle.objects.create( + audio_file=audio_file_instance, + start_time=subtitle["start_time"], + end_time=subtitle["end_time"], + content=subtitle["content"], + ) + for subtitle in self.subtitle_texts + ] + return subtitles + return [] diff --git a/waveform_audio/migrations/0001_initial.py b/waveform_audio/migrations/0001_initial.py index db03d58..abcc456 100644 --- a/waveform_audio/migrations/0001_initial.py +++ b/waveform_audio/migrations/0001_initial.py @@ -1,7 +1,7 @@ -# Generated by Django 4.2.1 on 2023-06-30 06:06 +# Generated by Django 5.1 on 2024-08-24 16:53 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): @@ -15,10 +15,13 @@ class Migration(migrations.Migration): name="AudioFile", fields=[ ("id", models.AutoField(primary_key=True, serialize=False)), - ("file", models.FileField(upload_to="audio/")), + ("file", models.FileField(unique=True, upload_to="audio/")), + ("file_hash", models.CharField(unique=True)), ("timestamp", models.DateTimeField(auto_now_add=True)), ], options={ + "verbose_name": "Audio File", + "verbose_name_plural": "Audio Files", "db_table": "audio_file", "ordering": ["timestamp"], }, @@ -29,34 +32,56 @@ class Migration(migrations.Migration): ("id", models.AutoField(primary_key=True, serialize=False)), ("start_time", models.TimeField()), ("end_time", models.TimeField()), + ("timestamp", models.DateTimeField(auto_now_add=True)), ( - "annotation", + "content", models.CharField( - choices=[ - ("speech", "Speech"), - ("music", "Music"), - ("noise", "Noise"), - ("laughter", "Laughter"), - ("other", "Other"), - ("unknown", "Unknown"), - ], - default="unknown", - max_length=10, + help_text="Annotation label of the audio segment", + max_length=255, ), ), - ("timestamp", models.DateTimeField(auto_now_add=True)), ( "audio_file", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - related_name="annotations", to="waveform_audio.audiofile", ), ), ], options={ + "verbose_name": "Audio Annotation", + "verbose_name_plural": "Audio Annotations", "db_table": "audio_annotation", "ordering": ["start_time", "end_time"], + "abstract": False, + "unique_together": { + ("audio_file", "start_time", "end_time", "content") + }, + }, + ), + migrations.CreateModel( + name="Subtitle", + fields=[ + ("id", models.AutoField(primary_key=True, serialize=False)), + ("start_time", models.TimeField()), + ("end_time", models.TimeField()), + ("timestamp", models.DateTimeField(auto_now_add=True)), + ("content", models.TextField(help_text="Subtitle content")), + ( + "audio_file", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="waveform_audio.audiofile", + ), + ), + ], + options={ + "verbose_name": "Subtitle", + "verbose_name_plural": "Subtitles", + "db_table": "subtitle", + "ordering": ["start_time", "end_time"], + "abstract": False, + "unique_together": {("audio_file", "start_time", "end_time")}, }, ), ] diff --git a/waveform_audio/migrations/0002_subtitle.py b/waveform_audio/migrations/0002_subtitle.py deleted file mode 100644 index 0b6ad75..0000000 --- a/waveform_audio/migrations/0002_subtitle.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 5.0 on 2024-07-13 10:22 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("waveform_audio", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="Subtitle", - fields=[ - ("id", models.AutoField(primary_key=True, serialize=False)), - ("start_time", models.TimeField()), - ("end_time", models.TimeField()), - ("text", models.TextField()), - ("timestamp", models.DateTimeField(auto_now_add=True)), - ( - "audio_file", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="subtitles", - to="waveform_audio.audiofile", - ), - ), - ], - options={ - "db_table": "subtitle", - "ordering": ["start_time", "end_time"], - }, - ), - ] diff --git a/waveform_audio/migrations/0003_rename_text_subtitle_content_and_more.py b/waveform_audio/migrations/0003_rename_text_subtitle_content_and_more.py deleted file mode 100644 index 45fe3ff..0000000 --- a/waveform_audio/migrations/0003_rename_text_subtitle_content_and_more.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 5.0 on 2024-07-13 13:50 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("waveform_audio", "0002_subtitle"), - ] - - operations = [ - migrations.RenameField( - model_name="subtitle", - old_name="text", - new_name="content", - ), - migrations.AlterUniqueTogether( - name="audiofile", - unique_together={("file",)}, - ), - migrations.AlterUniqueTogether( - name="subtitle", - unique_together={("audio_file", "start_time", "end_time")}, - ), - ] diff --git a/waveform_audio/migrations/0004_alter_subtitle_end_time_alter_subtitle_start_time.py b/waveform_audio/migrations/0004_alter_subtitle_end_time_alter_subtitle_start_time.py deleted file mode 100644 index 9611572..0000000 --- a/waveform_audio/migrations/0004_alter_subtitle_end_time_alter_subtitle_start_time.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 5.0 on 2024-07-13 17:55 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("waveform_audio", "0003_rename_text_subtitle_content_and_more"), - ] - - operations = [ - migrations.AlterField( - model_name="subtitle", - name="end_time", - field=models.DurationField(), - ), - migrations.AlterField( - model_name="subtitle", - name="start_time", - field=models.DurationField(), - ), - ] diff --git a/waveform_audio/models.py b/waveform_audio/models.py index 9e84018..6ddfd99 100644 --- a/waveform_audio/models.py +++ b/waveform_audio/models.py @@ -1,61 +1,130 @@ from django.db import models +from .utils import file_hash +from django.core.exceptions import ValidationError +import datetime as dt +from datetime import time +from django.utils.translation import gettext_lazy as _ +from django.core import checks +# Abstract model classes: +class TimeBoundLabelAbstract(models.Model): + + id = models.AutoField(primary_key=True) + start_time = models.TimeField() + end_time = models.TimeField() + audio_file = models.ForeignKey("AudioFile", on_delete=models.CASCADE) + timestamp = models.DateTimeField(auto_now_add=True) + content = None # This will be set by the subclass + + class Meta: + abstract = True + ordering = ["start_time", "end_time"] + + @staticmethod + def parse_time(time_value): + if isinstance(time_value, time): + return time_value + if isinstance(time_value, dt.timedelta): + time_value = str(time_value) + # check if the time value is in the format HH:MM:SS.%f, if not, add the missing parts with 0: + if "." not in time_value: + time_value += ".0" + + if isinstance(time_value, str): + try: + return dt.datetime.strptime(time_value, "%H:%M:%S.%f").time() + except ValueError: + try: + return dt.time.fromisoformat(time_value) + except ValueError: + raise ValidationError( + f"Invalid time format: {time_value}. Use HH:MM:SS.%f or ISO format or time delta." + ) + raise ValidationError( + f"Invalid time type: {type(time_value)}. Use string or time object." + ) + + def clean(self): + if self.start_time >= self.end_time: + raise ValidationError(_("End time must be after start time.")) + + def duration(self): + start_datetime = dt.datetime.combine(dt.datetime.min, self.start_time) + end_datetime = dt.datetime.combine(dt.datetime.min, self.end_time) + return end_datetime - start_datetime + + def save(self, *args, **kwargs): + self.start_time = self.parse_time(self.start_time) + self.end_time = self.parse_time(self.end_time) + self.full_clean() + super().save(*args, **kwargs) + + @classmethod + def check(cls, **kwargs): + errors = super().check(**kwargs) + if not any(f.name == "content" for f in cls._meta.fields): + errors.append( + checks.Error( + f"Subclasses of {cls.__name__} must define 'content' field", + hint="Add a 'content' field to your model.", + obj=cls, + id="models.E001", + ) + ) + return errors + + +# Models: class AudioFile(models.Model): id = models.AutoField(primary_key=True) - file = models.FileField(upload_to="audio/") + file = models.FileField(upload_to="audio/", unique=True) + file_hash = models.CharField(unique=True) timestamp = models.DateTimeField(auto_now_add=True) + # TODO: Add audio file details model def __str__(self): return str(self.file) + def save(self, *args, **kwargs): + if not self.pk: # Only on creation + self.file_hash = file_hash(self.file) + super().save(*args, **kwargs) + class Meta: ordering = ["timestamp"] db_table = "audio_file" - # make sure the file is unique: - unique_together = ["file"] - + verbose_name = "Audio File" + verbose_name_plural = "Audio Files" -class AudioAnnotation(models.Model): - class AnnotationLabel(models.TextChoices): - SPEECH = "speech" - MUSIC = "music" - NOISE = "noise" - LAUGHTER = "laughter" - OTHER = "other" - UNKNOWN = "unknown" - id = models.AutoField(primary_key=True) - audio_file = models.ForeignKey( - AudioFile, on_delete=models.CASCADE, related_name="annotations" - ) - start_time = models.TimeField() - end_time = models.TimeField() +class AudioAnnotation(TimeBoundLabelAbstract): - annotation = models.CharField( - max_length=10, choices=AnnotationLabel.choices, default=AnnotationLabel.UNKNOWN + content = models.CharField( + max_length=255, help_text=_("Annotation label of the audio segment") ) - # user = models.ForeignKey('auth.User', on_delete=models.CASCADE) - timestamp = models.DateTimeField(auto_now_add=True) - class Meta: - ordering = ["start_time", "end_time"] + class Meta(TimeBoundLabelAbstract.Meta): db_table = "audio_annotation" + # TODO: Add a Annotator model and field + # make sure the annotation is unique: + unique_together = ["audio_file", "start_time", "end_time", "content"] + verbose_name = "Audio Annotation" + verbose_name_plural = "Audio Annotations" + def __str__(self): + return ( + f"{self.audio_file} - {self.content} ({self.start_time} to {self.end_time})" + ) -class Subtitle(models.Model): - id = models.AutoField(primary_key=True) - audio_file = models.ForeignKey( - AudioFile, on_delete=models.CASCADE, related_name="subtitles" - ) - start_time = models.DurationField() - end_time = models.DurationField() - content = models.TextField() - timestamp = models.DateTimeField(auto_now_add=True) +class Subtitle(TimeBoundLabelAbstract): - class Meta: - ordering = ["start_time", "end_time"] + content = models.TextField(help_text=_("Subtitle content")) + # language = models.CharField(max_length=255, default="en") + class Meta(TimeBoundLabelAbstract.Meta): db_table = "subtitle" # make sure the subtitle is unique: unique_together = ["audio_file", "start_time", "end_time"] + verbose_name = "Subtitle" + verbose_name_plural = "Subtitles" diff --git a/waveform_audio/templates/index.html b/waveform_audio/templates/index.html index 443af63..7876c2b 100644 --- a/waveform_audio/templates/index.html +++ b/waveform_audio/templates/index.html @@ -14,7 +14,7 @@

Choose an audio file to annotate:

- My Image + My Image