Allow recording and saving audio

backend/FwHeadless/LexboxFwDataMediaAdapter.cs

@@ -4,7 +4,9 @@
 using LexCore.Exceptions;
 using Microsoft.Extensions.Options;
 using MiniLcm;
+using MiniLcm.Media;
 using SIL.LCModel;
+using MediaFile = LexCore.Entities.MediaFile;
 
 namespace FwHeadless;
 

backend/FwLite/FwDataMiniLcmBridge.Tests/MediaFileTests.cs

@@ -1,6 +1,7 @@
 using FwDataMiniLcmBridge.Api;
 using FwDataMiniLcmBridge.Media;
 using FwDataMiniLcmBridge.Tests.Fixtures;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using SIL.LCModel.Infrastructure;
 

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -9,6 +9,7 @@
 using Microsoft.Extensions.Options;
 using MiniLcm;
 using MiniLcm.Exceptions;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using MiniLcm.SyncHelpers;
 using MiniLcm.Validators;
@@ -148,7 +149,7 @@
         }
     }
 
     public async Task<WritingSystem> CreateWritingSystem(WritingSystem writingSystem)
     {
         var type = writingSystem.Type;
         var exitingWs = type == WritingSystemType.Analysis ? Cache.ServiceLocator.WritingSystems.AnalysisWritingSystems : Cache.ServiceLocator.WritingSystems.VernacularWritingSystems;
@@ -235,7 +236,7 @@
             ? FromLcmPartOfSpeech(partOfSpeech) : null);
     }
 
     public async Task<PartOfSpeech> CreatePartOfSpeech(PartOfSpeech partOfSpeech)
     {
         IPartOfSpeech? lcmPartOfSpeech = null;
         if (partOfSpeech.Id == default) partOfSpeech.Id = Guid.NewGuid();
@@ -437,7 +438,7 @@
         return new ComplexFormType() { Id = t.Guid, Name = FromLcmMultiString(t.Name) };
     }
 
     public async Task<ComplexFormType> CreateComplexFormType(ComplexFormType complexFormType)
     {
         if (complexFormType.Id == default) complexFormType.Id = Guid.NewGuid();
         UndoableUnitOfWorkHelper.DoUsingNewOrCurrentUOW("Create complex form type",
@@ -817,7 +818,7 @@
     {
         if (string.IsNullOrEmpty(query)) return null;
         return entry => entry.CitationForm.SearchValue(query) ||
-                        entry.LexemeFormOA.Form.SearchValue(query) ||
+                        entry.LexemeFormOA?.Form.SearchValue(query) == true ||
                         entry.AllSenses.Any(s => s.Gloss.SearchValue(query));
     }
 
@@ -1295,7 +1296,7 @@
         return Task.FromResult(lcmSense is null ? null : FromLexSense(lcmSense));
     }
 
     public async Task<Sense> CreateSense(Guid entryId, Sense sense, BetweenPosition? between = null)
     {
         if (sense.Id == default) sense.Id = Guid.NewGuid();
         if (!EntriesRepository.TryGetObject(entryId, out var lexEntry))
@@ -1440,7 +1441,7 @@
         return CmTranslationFactory.Create(parent, freeTranslationType);
     }
 
     public async Task<ExampleSentence> CreateExampleSentence(Guid entryId, Guid senseId, ExampleSentence exampleSentence, BetweenPosition? between = null)
     {
         if (exampleSentence.Id == default) exampleSentence.Id = Guid.NewGuid();
         if (!SenseRepository.TryGetObject(senseId, out var lexSense))
@@ -1540,4 +1541,49 @@
         if (!File.Exists(fullPath)) return Task.FromResult(new ReadFileResponse(ReadFileResult.NotFound));
         return Task.FromResult(new ReadFileResponse(File.OpenRead(fullPath), Path.GetFileName(fullPath)));
     }
+
+    public async Task<UploadFileResponse> SaveFile(Stream stream, LcmFileMetadata metadata)
+    {
+        var pathRelativeToRoot = Path.Combine(TypeToLinkedFolder(metadata.MimeType), Path.GetFileName(metadata.Filename));
+        var fullPath = Path.Combine(Cache.LangProject.LinkedFilesRootDir, pathRelativeToRoot);
+        if (File.Exists(fullPath)) return new UploadFileResponse(UploadFileResult.AlreadyExists);
+        var directory = Path.GetDirectoryName(fullPath);
+        if (directory is not null)
+        {
+            try
+            {
+                Directory.CreateDirectory(directory);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to create directory {Directory} for file {Filename}", directory, metadata.Filename);
+                return new UploadFileResponse($"Failed to create directory: {ex.Message}");
+            }
+        }
+
+        try
+        {
+            await using var fileStream = File.OpenWrite(fullPath);
+            await stream.CopyToAsync(fileStream);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to save file {Filename} to {Path}", metadata.Filename, fullPath);
+            return new UploadFileResponse($"Failed to save file: {ex.Message}");
+        }
+
+        var mediaUri = mediaAdapter.MediaUriFromPath(pathRelativeToRoot, Cache);
+        return new UploadFileResponse(mediaUri, false);
+    }
+
+    private string TypeToLinkedFolder(string mimeType)
+    {
+        return mimeType switch
+        {
+            { } s when s.StartsWith("audio/") => AudioVisualFolder,
+            {} s when s.StartsWith("video/") => AudioVisualFolder,
+            { } s when s.StartsWith("image/") => "Pictures",
+            _ => "Others"
+        };
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateDictionaryProxy.cs

@@ -1,6 +1,7 @@
 using System.Collections;
 using System.Diagnostics.CodeAnalysis;
 using MiniLcm;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using SIL.LCModel.Core.KernelInterfaces;
 using SIL.LCModel.Core.Text;

backend/FwLite/FwDataMiniLcmBridge/LexEntryFilterMapProvider.cs

@@ -28,7 +28,8 @@ public class LexEntryFilterMapProvider : EntryFilterMapProvider<ILexEntry>
     public override Expression<Func<ILexEntry, string, object?>> EntrySensesGloss => (entry, ws) => entry.AllSenses.Select(s => s.PickText(s.Gloss, ws));
     public override Expression<Func<ILexEntry, string, object?>> EntrySensesDefinition => (entry, ws) => entry.AllSenses.Select(s => s.PickText(s.Definition, ws));
     public override Expression<Func<ILexEntry, string, object?>> EntryNote => (entry, ws) => entry.PickText(entry.Comment, ws);
-    public override Expression<Func<ILexEntry, string, object?>> EntryLexemeForm => (entry, ws) => entry.PickText(entry.LexemeFormOA.Form, ws);
+    public override Expression<Func<ILexEntry, string, object?>> EntryLexemeForm => (entry, ws) =>
+        entry.LexemeFormOA == null ? null : entry.PickText(entry.LexemeFormOA.Form, ws);
     public override Expression<Func<ILexEntry, string, object?>> EntryCitationForm => (entry, ws) => entry.PickText(entry.CitationForm, ws);
     public override Expression<Func<ILexEntry, string, object?>> EntryLiteralMeaning => (entry, ws) => entry.PickText(entry.LiteralMeaning, ws);
     public override Expression<Func<ILexEntry, object?>> EntryComplexFormTypes => e => EmptyToNull(e.ComplexFormEntryRefs.SelectMany(r => r.ComplexEntryTypesRS));

backend/FwLite/FwDataMiniLcmBridge/Media/IMediaAdapter.cs

@@ -1,4 +1,5 @@
 using MiniLcm;
+using MiniLcm.Media;
 using SIL.LCModel;
 
 namespace FwDataMiniLcmBridge.Media;

backend/FwLite/FwDataMiniLcmBridge/Media/LocalMediaAdapter.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Caching.Memory;
 using MiniLcm;
 using MiniLcm.Exceptions;
+using MiniLcm.Media;
 using SIL.LCModel;
 using UUIDNext;
 
@@ -22,14 +23,22 @@ private Dictionary<Guid, string> Paths(LcmCache cache)
                 return Directory
                     .EnumerateFiles(cache.LangProject.LinkedFilesRootDir, "*", SearchOption.AllDirectories)
                     .Select(file => Path.GetRelativePath(cache.LangProject.LinkedFilesRootDir, file))
-                    .ToDictionary(file => MediaUriFromPath(file, cache).FileId, file => file);
+                    .ToDictionary(file => PathToUri(file).FileId, file => file);
             }) ?? throw new Exception("Failed to get paths");
     }
 
     //path is expected to be relative to the LinkedFilesRootDir
     public MediaUri MediaUriFromPath(string path, LcmCache cache)
     {
         if (!File.Exists(Path.Combine(cache.LangProject.LinkedFilesRootDir, path))) return MediaUri.NotFound;
+        var uri = PathToUri(path);
+        //this may be a new file, so we need to add it to the cache
+        Paths(cache)[uri.FileId] = path;
+        return uri;
+    }
+
+    private static MediaUri PathToUri(string path)
+    {
         return new MediaUri(NewGuidV5(path), LocalMediaAuthority);
     }
 

backend/FwLite/FwLiteShared/Services/MiniLcmJsInvokable.cs

@@ -1,7 +1,9 @@
 using FwLiteShared.Sync;
 using LcmCrdt;
+using Microsoft.Extensions.Logging;
 using Microsoft.JSInterop;
 using MiniLcm;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using MiniLcm.Validators;
 using Reinforced.Typings.Attributes;
@@ -12,6 +14,7 @@ public class MiniLcmJsInvokable(
     IMiniLcmApi api,
     BackgroundSyncService backgroundSyncService,
     IProjectIdentifier project,
+    ILogger<MiniLcmJsInvokable> logger,
     MiniLcmApiNotifyWrapperFactory notificationWrapperFactory,
     MiniLcmApiValidationWrapperFactory validationWrapperFactory) : IDisposable
 {
@@ -343,6 +346,24 @@ public record ReadFileResponseJs(
         string? FileName,
         ReadFileResult Result,
         string? ErrorMessage);
+    public const int TenMbFileLimit = 10 * 1024 * 1024;
+
+    [JSInvokable]
+    public async Task<UploadFileResponse> SaveFile(IJSStreamReference streamReference, LcmFileMetadata metadata)
+    {
+        if (streamReference.Length > TenMbFileLimit) return new(UploadFileResult.TooBig);
+        await using var stream = await streamReference.OpenReadStreamAsync(TenMbFileLimit);
+        var result =  await _wrappedApi.SaveFile(stream, metadata);
+        try
+        {
+            await streamReference.DisposeAsync();
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Error disposing stream reference");
+        }
+        return result;
+    }
 
     public void Dispose()
     {

backend/FwLite/FwLiteShared/Sync/SyncService.cs

@@ -4,6 +4,7 @@
 using FwLiteShared.Projects;
 using LexCore.Sync;
 using LcmCrdt;
+using LcmCrdt.MediaServer;
 using LcmCrdt.RemoteSync;
 using LcmCrdt.Utils;
 using Microsoft.EntityFrameworkCore;
@@ -26,6 +27,7 @@ public class SyncService(
     ProjectEventBus changeEventBus,
     LexboxProjectService lexboxProjectService,
     IMiniLcmApi lexboxApi,
+    LcmMediaService lcmMediaService,
     IOptions<AuthConfig> authOptions,
     ILogger<SyncService> logger,
     IDbContextFactory<LcmCrdtDbContext> dbContextFactory)
@@ -80,6 +82,8 @@ public async Task<SyncResults> ExecuteSync(bool skipNotifications = false)
             UpdateSyncStatus(SyncStatus.Offline);
             return new SyncResults([], [], false);
         }
+
+        await UploadPendingMedia();
         var syncDate = DateTimeOffset.UtcNow;//create sync date first to ensure it's consistent and not based on how long it takes to sync
         var syncResults = await dataModel.SyncWith(remoteModel);
         if (!syncResults.IsSynced)
@@ -139,6 +143,18 @@ public async Task<ProjectSyncStatus> GetSyncStatus()
         return new PendingCommits(localChanges.Value, remoteChanges);
     }
 
+    public async Task UploadPendingMedia()
+    {
+        try
+        {
+            await lcmMediaService.UploadPendingResources();
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Failed to upload pending media");
+        }
+    }
+
     private void UpdateSyncStatus(SyncStatus status)
     {
         changeEventBus.PublishEvent(currentProjectService.Project, new SyncEvent(status));

backend/FwLite/FwLiteShared/TypeGen/ReinforcedFwLiteTypingConfig.cs

@@ -24,6 +24,8 @@
 using System.Runtime.CompilerServices;
 using FwLiteShared.AppUpdate;
 using FwLiteShared.Sync;
+using MiniLcm.Media;
+using MediaFile = MiniLcm.Media.MediaFile;
 
 namespace FwLiteShared.TypeGen;
 
@@ -53,6 +55,7 @@ public static void Configure(ConfigurationBuilder builder)
             exportBuilder => exportBuilder.WithName("DotNet.DotNetObject").Imports([
                 new() { From = "@microsoft/dotnet-js-interop", Target = "type {DotNet}" }
             ]));
+        builder.Substitute(typeof(IJSStreamReference), new RtSimpleTypeName("Blob | ArrayBuffer | Uint8Array"));
         builder.Substitute(typeof(DotNetStreamReference), new RtSimpleTypeName("{stream: () => Promise<ReadableStream>, arrayBuffer: () => Promise<ArrayBuffer>}"));
         builder.ExportAsInterface<IAsyncDisposable>();
 
@@ -63,6 +66,7 @@ public static void Configure(ConfigurationBuilder builder)
     private static void ConfigureMiniLcmTypes(ConfigurationBuilder builder)
     {
         builder.Substitute(typeof(WritingSystemId), new RtSimpleTypeName("string"));
+        builder.Substitute(typeof(MediaUri), new RtSimpleTypeName("string"));
         builder.ExportAsThirdParty<MultiString>().WithName("IMultiString").Imports([
             new() { From = "$lib/dotnet-types/i-multi-string", Target = "type {IMultiString}" }
         ]);
@@ -78,6 +82,9 @@ private static void ConfigureMiniLcmTypes(ConfigurationBuilder builder)
                 typeof(IObjectWithId),
                 typeof(RichString),
                 typeof(RichTextObjectData),
+
+                typeof(MediaFile),
+                typeof(LcmFileMetadata)
             ],
             exportBuilder => exportBuilder.WithPublicNonStaticProperties(exportBuilder =>
         {
@@ -100,6 +107,7 @@ private static void ConfigureMiniLcmTypes(ConfigurationBuilder builder)
         ]);
         builder.ExportAsEnum<WritingSystemType>();
         builder.ExportAsEnum<ReadFileResult>().UseString();
+        builder.ExportAsEnum<UploadFileResult>().UseString();
         builder.ExportAsInterface<MiniLcmJsInvokable>()
             .FlattenHierarchy()
             .WithPublicProperties()
@@ -111,7 +119,8 @@ private static void ConfigureMiniLcmTypes(ConfigurationBuilder builder)
                 typeof(SortOptions),
                 typeof(ExemplarOptions),
                 typeof(EntryFilter),
-                typeof(MiniLcmJsInvokable.ReadFileResponseJs)
+                typeof(MiniLcmJsInvokable.ReadFileResponseJs),
+                typeof(UploadFileResponse)
             ],
             exportBuilder => exportBuilder.WithPublicNonStaticProperties(propExportBuilder =>
         {

backend/FwLite/LcmCrdt/CrdtMiniLcmApi.cs

@@ -17,6 +17,7 @@
 using MiniLcm.SyncHelpers;
 using SIL.Harmony.Core;
 using MiniLcm.Culture;
+using MiniLcm.Media;
 using SystemTextJsonPatch;
 
 namespace LcmCrdt;
@@ -699,6 +700,20 @@ public async Task<ReadFileResponse> GetFileStream(MediaUri mediaUri)
         return await lcmMediaService.GetFileStream(mediaUri.FileId);
     }
 
+    public async Task<UploadFileResponse> SaveFile(Stream stream, LcmFileMetadata metadata)
+    {
+        try
+        {
+            var result = await lcmMediaService.SaveFile(stream, metadata);
+            return new UploadFileResponse(new MediaUri(result.Id, ProjectData.ServerId ?? "lexbox.org"), result.Remote);
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Failed to save file {Filename}", metadata.Filename);
+            return new UploadFileResponse(e.Message);
+        }
+    }
+
     public void Dispose()
     {
     }

backend/FwLite/LcmCrdt/MediaServer/IMediaServerClient.cs

@@ -7,4 +7,14 @@ public interface IMediaServerClient
 {
     [Get("/api/media/{fileId}")]
     Task<HttpResponseMessage> DownloadFile(Guid fileId);
+
+    [Post("/api/media")]
+    [Multipart]
+    Task<MediaUploadFileResponse> UploadFile(MultipartItem file,
+        [Query] Guid projectId,
+        string fileId,//using a string because Refit doesn't handle a Guid properly
+        string? author = null,
+        string? filename = null);
 }
+
+public record MediaUploadFileResponse(Guid Guid);