From 55f1573c4e575b5a5a130b236bdc18b69b666eaf Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Mon, 5 Jan 2026 12:04:34 +1300 Subject: [PATCH 1/2] SF-3666 Set the scripture ranges for all build endpoints --- .../Services/MachineApiService.cs | 270 ++++++++++-------- .../Services/MachineApiServiceTests.cs | 18 +- 2 files changed, 161 insertions(+), 127 deletions(-) diff --git a/src/SIL.XForge.Scripture/Services/MachineApiService.cs b/src/SIL.XForge.Scripture/Services/MachineApiService.cs index 51e5cd872a2..dcac640a63b 100644 --- a/src/SIL.XForge.Scripture/Services/MachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineApiService.cs @@ -840,19 +840,11 @@ await eventMetricService.SaveEventMetricAsync( CancellationToken cancellationToken ) { - ServalBuildDto? buildDto = null; - // Ensure that the user has permission - SFProject project = await EnsureProjectPermissionAsync( - curUserId, - sfProjectId, - isServalAdmin, - cancellationToken - ); + await EnsureProjectPermissionAsync(curUserId, sfProjectId, isServalAdmin, cancellationToken); // Execute on Serval, if it is enabled string translationEngineId = await GetTranslationIdAsync(sfProjectId, preTranslate); - try { TranslationBuild translationBuild = await translationEnginesClient.GetBuildAsync( @@ -861,21 +853,13 @@ CancellationToken cancellationToken minRevision, cancellationToken ); - buildDto = CreateDto(translationBuild); + return await MapDtoAsync(sfProjectId, translationBuild, preTranslate); } catch (ServalApiException e) { ProcessServalApiException(e); + return null; } - - // Make sure the DTO conforms to the machine-api V2 URLs - if (buildDto is not null) - { - buildDto = UpdateDto(buildDto, project.TranslateConfig.DraftConfig); - buildDto = UpdateDto(buildDto, sfProjectId); - } - - return buildDto; } public async Task GetRawBuildAsync( @@ -931,9 +915,6 @@ public virtual async Task> GetBuildsAsync( CancellationToken cancellationToken ) { - // Set up the list of builds to be returned - List builds = []; - // Ensure that the user has permission await EnsureProjectPermissionAsync(curUserId, sfProjectId, isServalAdmin, cancellationToken); @@ -970,72 +951,13 @@ CancellationToken cancellationToken ); } - // Return the builds as DTOs - foreach (TranslationBuild translationBuild in translationBuilds) - { - ServalBuildDto buildDto = CreateDto(translationBuild); - - // See if we have event metrics for downloading the pre-translation USFM to Scripture Forge - EventMetric eventMetric = eventMetrics.Results.FirstOrDefault(e => - e.Result == translationBuild.Id && e.EventType == nameof(RetrievePreTranslationStatusAsync) - ); - if (eventMetric is not null) - { - buildDto.AdditionalInfo!.DateGenerated = new DateTimeOffset(eventMetric.TimeStamp, TimeSpan.Zero); - } - - // If we have event metrics for sending the build to Serval, add the scripture ranges to the DTO - eventMetric = eventMetrics.Results.FirstOrDefault(e => - e.Result == translationBuild.Id && e.EventType == nameof(MachineProjectService.BuildProjectAsync) - ); - if (eventMetric is not null) - { - buildDto = UpdateDto(buildDto, eventMetric); - } - else if (preTranslate) - { - // Fallback for builds previous to the event metric being recorded: - // - As there is no event metric, get the translation scripture range from the pre-translation corpus - // - We cannot accurately determine the source projects, so do not record the training scripture ranges. - - // Get the translation scripture range - PretranslateCorpus translationCorpus = translationBuild.Pretranslate?.FirstOrDefault(); - if (translationCorpus is not null) - { -#pragma warning disable CS0612 // Type or member is obsolete - string scriptureRange = - translationCorpus.SourceFilters?.FirstOrDefault()?.ScriptureRange - ?? translationCorpus.ScriptureRange; -#pragma warning restore CS0612 // Type or member is obsolete - if (!string.IsNullOrWhiteSpace(scriptureRange)) - { - buildDto.AdditionalInfo!.TranslationScriptureRanges.Add( - new ProjectScriptureRange { ProjectId = sfProjectId, ScriptureRange = scriptureRange } - ); - } - } - - // Get the training scripture range - TrainingCorpus trainingCorpus = translationBuild.TrainOn?.FirstOrDefault(); - if (trainingCorpus is not null) - { -#pragma warning disable CS0612 // Type or member is obsolete - string scriptureRange = - trainingCorpus.SourceFilters?.FirstOrDefault()?.ScriptureRange ?? trainingCorpus.ScriptureRange; -#pragma warning restore CS0612 // Type or member is obsolete - if (!string.IsNullOrWhiteSpace(scriptureRange)) - { - // We do not accurately know the training, project, so leave it blank - buildDto.AdditionalInfo!.TrainingScriptureRanges.Add( - new ProjectScriptureRange { ProjectId = string.Empty, ScriptureRange = scriptureRange } - ); - } - } - } - - // Make sure the DTO conforms to the machine-api URLs - builds.Add(UpdateDto(buildDto, sfProjectId)); - } + // Map the builds to DTOs + List builds = + [ + .. translationBuilds.Select(translationBuild => + MapDto(sfProjectId, translationBuild, preTranslate, eventMetrics) + ), + ]; // See if any builds are queued at our end ServalBuildDto? queuedState = await GetQueuedStateAsync( @@ -1069,15 +991,8 @@ CancellationToken cancellationToken CancellationToken cancellationToken ) { - ServalBuildDto? buildDto = null; - // Ensure that the user has permission - SFProject project = await EnsureProjectPermissionAsync( - curUserId, - sfProjectId, - isServalAdmin, - cancellationToken - ); + await EnsureProjectPermissionAsync(curUserId, sfProjectId, isServalAdmin, cancellationToken); // Get the translation engine string translationEngineId = await GetTranslationIdAsync(sfProjectId, preTranslate: true); @@ -1094,7 +1009,7 @@ CancellationToken cancellationToken .MaxBy(b => b.DateFinished); if (lastCompletedTranslationBuild is not null) { - buildDto = CreateDto(lastCompletedTranslationBuild); + return await MapDtoAsync(sfProjectId, lastCompletedTranslationBuild, preTranslate: true); } } catch (ServalApiException e) @@ -1102,13 +1017,8 @@ CancellationToken cancellationToken ProcessServalApiException(e); } - // Make sure the DTO conforms to the machine-api V2 URLs - if (buildDto is not null) - { - buildDto = UpdateDto(buildDto, sfProjectId); - } - - return buildDto; + // No completed build found + return null; } /// @@ -1132,8 +1042,6 @@ CancellationToken cancellationToken CancellationToken cancellationToken ) { - ServalBuildDto? buildDto = null; - // Ensure that the user has permission await EnsureProjectPermissionAsync(curUserId, sfProjectId, isServalAdmin, cancellationToken); @@ -1166,9 +1074,7 @@ static DateTimeOffset GetSortTimestamp(TranslationBuild b) if (lastBuild is not null) { - buildDto = CreateDto(lastBuild); - // Make sure the DTO conforms to the machine-api V2 URLs - buildDto = UpdateDto(buildDto, sfProjectId); + return await MapDtoAsync(sfProjectId, lastBuild, preTranslate: true); } } catch (ServalApiException e) @@ -1176,7 +1082,7 @@ static DateTimeOffset GetSortTimestamp(TranslationBuild b) ProcessServalApiException(e); } - return buildDto; + return null; } public async Task GetCurrentBuildAsync( @@ -2496,6 +2402,85 @@ .. builds.Where(b => ]; } + /// + /// Maps a translation build to a Serval build DTO, including the scripture ranges from the event metrics. + /// + /// The scripture forge project identifier. + /// The translation build from Serval. + /// If true, return NMT builds only; otherwise, return SMT builds. + /// (Optional) The event metrics for the project's builds or this specific build. + /// The Serval build DTO. + private static ServalBuildDto MapDto( + string sfProjectId, + TranslationBuild translationBuild, + bool preTranslate, + QueryResults eventMetrics + ) + { + // Create the initial DTO + ServalBuildDto buildDto = CreateDto(translationBuild); + + // See if we have event metrics for downloading the pre-translation USFM to Scripture Forge + EventMetric? eventMetric = eventMetrics.Results.FirstOrDefault(e => + e.Result == translationBuild.Id && e.EventType == nameof(RetrievePreTranslationStatusAsync) + ); + if (eventMetric is not null) + { + buildDto.AdditionalInfo!.DateGenerated = new DateTimeOffset(eventMetric.TimeStamp, TimeSpan.Zero); + } + + // If we have event metrics for sending the build to Serval, add the scripture ranges to the DTO + eventMetric = eventMetrics.Results.FirstOrDefault(e => + e.Result == translationBuild.Id && e.EventType == nameof(MachineProjectService.BuildProjectAsync) + ); + if (eventMetric is not null) + { + buildDto = UpdateDto(buildDto, eventMetric); + } + else if (preTranslate) + { + // Fallback for builds previous to the event metric being recorded: + // - As there is no event metric, get the translation scripture range from the pre-translation corpus + // - We cannot accurately determine the source projects, so do not record the training scripture ranges. + + // Get the translation scripture range + PretranslateCorpus translationCorpus = translationBuild.Pretranslate?.FirstOrDefault(); + if (translationCorpus is not null) + { +#pragma warning disable CS0612 // Type or member is obsolete + string scriptureRange = + translationCorpus.SourceFilters?.FirstOrDefault()?.ScriptureRange + ?? translationCorpus.ScriptureRange; +#pragma warning restore CS0612 // Type or member is obsolete + if (!string.IsNullOrWhiteSpace(scriptureRange)) + { + buildDto.AdditionalInfo!.TranslationScriptureRanges.Add( + new ProjectScriptureRange { ProjectId = sfProjectId, ScriptureRange = scriptureRange } + ); + } + } + + // Get the training scripture range + TrainingCorpus trainingCorpus = translationBuild.TrainOn?.FirstOrDefault(); + if (trainingCorpus is not null) + { +#pragma warning disable CS0612 // Type or member is obsolete + string scriptureRange = + trainingCorpus.SourceFilters?.FirstOrDefault()?.ScriptureRange ?? trainingCorpus.ScriptureRange; +#pragma warning restore CS0612 // Type or member is obsolete + if (!string.IsNullOrWhiteSpace(scriptureRange)) + { + // We do not accurately know the training, project, so leave it blank + buildDto.AdditionalInfo!.TrainingScriptureRanges.Add( + new ProjectScriptureRange { ProjectId = string.Empty, ScriptureRange = scriptureRange } + ); + } + } + } + + return UpdateDto(buildDto, sfProjectId); + } + /// /// This method maps Serval API exceptions to the exceptions that Machine.js understands. /// @@ -2566,27 +2551,42 @@ private static ServalBuildDto UpdateDto(ServalBuildDto buildDto, string sfProjec return buildDto; } + /// + /// Updates the Serval build DTO with the scripture ranges and training files for the latest build. + /// + /// The Serval build DTO. + /// The draft configuration from the project document. + /// The updated Serval build DTO. + /// This should only be called for queued or current builds. private static ServalBuildDto UpdateDto(ServalBuildDto buildDto, DraftConfig draftConfig) { + ArgumentNullException.ThrowIfNull(buildDto.AdditionalInfo); + // Add the training scripture ranges - buildDto.AdditionalInfo.TrainingScriptureRanges.Clear(); - foreach (ProjectScriptureRange scriptureRange in draftConfig.LastSelectedTrainingScriptureRanges) + if (buildDto.AdditionalInfo.TrainingScriptureRanges.Count == 0) { - buildDto.AdditionalInfo.TrainingScriptureRanges.Add(scriptureRange); + foreach (ProjectScriptureRange scriptureRange in draftConfig.LastSelectedTrainingScriptureRanges) + { + buildDto.AdditionalInfo.TrainingScriptureRanges.Add(scriptureRange); + } } // Add the translation scripture ranges - buildDto.AdditionalInfo.TranslationScriptureRanges.Clear(); - foreach (ProjectScriptureRange scriptureRange in draftConfig.LastSelectedTranslationScriptureRanges) + if (buildDto.AdditionalInfo.TranslationScriptureRanges.Count == 0) { - buildDto.AdditionalInfo.TranslationScriptureRanges.Add(scriptureRange); + foreach (ProjectScriptureRange scriptureRange in draftConfig.LastSelectedTranslationScriptureRanges) + { + buildDto.AdditionalInfo.TranslationScriptureRanges.Add(scriptureRange); + } } // Add training data files - buildDto.AdditionalInfo.TrainingDataFileIds.Clear(); - foreach (string trainingFileDataId in draftConfig.LastSelectedTrainingDataFiles) + if (buildDto.AdditionalInfo.TrainingDataFileIds.Count == 0) { - buildDto.AdditionalInfo.TrainingDataFileIds.Add(trainingFileDataId); + foreach (string trainingFileDataId in draftConfig.LastSelectedTrainingDataFiles) + { + buildDto.AdditionalInfo.TrainingDataFileIds.Add(trainingFileDataId); + } } return buildDto; @@ -2721,4 +2721,36 @@ private async Task GetTranslationIdAsync( return GetTranslationEngineId(projectSecret, preTranslate, returnEmptyStringIfMissing); } + + /// + /// Maps a translation build to a Serval build DTO, including the scripture ranges from the event metrics. + /// + /// The scripture forge project identifier. + /// The translation build from Serval. + /// If true, return NMT builds only; otherwise, return SMT builds. + /// The Serval build DTO. + private async Task MapDtoAsync( + string sfProjectId, + TranslationBuild translationBuild, + bool preTranslate + ) + { + // Get the event metrics for build configurations, if we are pre-translating + QueryResults eventMetrics = QueryResults.Empty; + if (preTranslate) + { + eventMetrics = await eventMetricService.GetEventMetricsAsync( + sfProjectId, + scopes: [EventScope.Drafting], + eventTypes: + [ + nameof(MachineProjectService.BuildProjectAsync), + nameof(RetrievePreTranslationStatusAsync), + nameof(StartPreTranslationBuildAsync), + ] + ); + } + + return MapDto(sfProjectId, translationBuild, preTranslate, eventMetrics); + } } diff --git a/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs b/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs index e060d23790c..0d3fe2141a2 100644 --- a/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs +++ b/test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs @@ -970,6 +970,7 @@ public async Task GetBuildAsync_ServalAdminDoesNotNeedPermission() { // Set up test environment var env = new TestEnvironment(); + env.SetupEventMetrics("GEN", "EXO", DateTime.UtcNow); env.ConfigureTranslationBuild(); // SUT @@ -991,6 +992,7 @@ public async Task GetBuildAsync_Success() { // Set up test environment var env = new TestEnvironment(); + env.SetupEventMetrics("GEN", "EXO", DateTime.UtcNow); TranslationBuild translationBuild = env.ConfigureTranslationBuild(); // SUT @@ -999,7 +1001,7 @@ public async Task GetBuildAsync_Success() Project01, ServalBuildId01, minRevision: null, - preTranslate: false, + preTranslate: true, isServalAdmin: false, CancellationToken.None ); @@ -1007,11 +1009,11 @@ public async Task GetBuildAsync_Success() TestEnvironment.AssertCoreBuildProperties(translationBuild, actual); Assert.NotNull(actual.AdditionalInfo); Assert.AreEqual( - new ProjectScriptureRange { ScriptureRange = "GEN" }, + new ProjectScriptureRange { ProjectId = Project03, ScriptureRange = "GEN" }, actual.AdditionalInfo.TranslationScriptureRanges.Single() ); Assert.AreEqual( - new ProjectScriptureRange { ScriptureRange = "EXO" }, + new ProjectScriptureRange { ProjectId = Project02, ScriptureRange = "EXO" }, actual.AdditionalInfo.TrainingScriptureRanges.Single() ); } @@ -1034,7 +1036,6 @@ public async Task GetBuildAsync_IncludesAdditionalInfo() const string corpusId2 = "corpusId2"; const string corpusId3 = "corpusId3"; const string corpusId4 = "corpusId4"; - const string parallelCorpusId1 = ParallelCorpusId01; const string parallelCorpusId2 = "parallelCorpusId2"; const int step = 123; @@ -1055,7 +1056,7 @@ public async Task GetBuildAsync_IncludesAdditionalInfo() [ new PretranslateCorpus { - ParallelCorpus = new ResourceLink { Id = parallelCorpusId1, Url = "https://example.com" }, + ParallelCorpus = new ResourceLink { Id = ParallelCorpusId01, Url = "https://example.com" }, }, new PretranslateCorpus { @@ -1096,7 +1097,7 @@ public async Task GetBuildAsync_IncludesAdditionalInfo() [ new ParallelCorpusAnalysis { - ParallelCorpusRef = parallelCorpusId1, + ParallelCorpusRef = ParallelCorpusId01, SourceQuoteConvention = "standard_english", TargetQuoteConvention = "standard_english", }, @@ -1134,9 +1135,8 @@ public async Task GetBuildAsync_IncludesAdditionalInfo() Assert.AreEqual(corpusId3, actual.AdditionalInfo.CorporaIds.ElementAt(2)); Assert.AreEqual(corpusId4, actual.AdditionalInfo.CorporaIds.ElementAt(3)); Assert.IsNotNull(actual.AdditionalInfo.ParallelCorporaIds); - Assert.AreEqual(parallelCorpusId1, actual.AdditionalInfo.ParallelCorporaIds!.ElementAt(0)); + Assert.AreEqual(ParallelCorpusId01, actual.AdditionalInfo.ParallelCorporaIds!.ElementAt(0)); Assert.AreEqual(parallelCorpusId2, actual.AdditionalInfo.ParallelCorporaIds.ElementAt(1)); - Assert.AreEqual(TrainingDataId01, actual.AdditionalInfo.TrainingDataFileIds.Single()); Assert.AreEqual(actual.AdditionalInfo.QuotationDenormalization, QuotationAnalysis.Successful); } @@ -2043,6 +2043,7 @@ public async Task GetLastCompletedPreTranslationBuildAsync_NoRetrievePreTranslat { // Set up test environment var env = new TestEnvironment(); + env.SetupEventMetrics("GEN", "EXO", DateTime.UtcNow); const string scriptureRange = "GEN"; await env.Projects.UpdateAsync( Project01, @@ -2176,6 +2177,7 @@ public async Task GetLastPreTranslationBuildAsync_LatestByDateFinished_Success() { var env = new TestEnvironment(); DateTimeOffset now = DateTimeOffset.UtcNow; + env.SetupEventMetrics("GEN", "EXO", now.DateTime); TranslationBuild completedEarlier = new TranslationBuild { Url = "https://example.com", From 2059e8255124f75bfe5e14afe42d06c35ea52b38 Mon Sep 17 00:00:00 2001 From: Peter Chapman Date: Mon, 5 Jan 2026 13:46:48 +1300 Subject: [PATCH 2/2] SF-3666 Download all chapters for a draft when the target is shorter --- .../Services/MachineApiService.cs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/SIL.XForge.Scripture/Services/MachineApiService.cs b/src/SIL.XForge.Scripture/Services/MachineApiService.cs index dcac640a63b..90a3b4c6812 100644 --- a/src/SIL.XForge.Scripture/Services/MachineApiService.cs +++ b/src/SIL.XForge.Scripture/Services/MachineApiService.cs @@ -1516,6 +1516,13 @@ CancellationToken cancellationToken // If the user is a serval admin, get the highest ranked user on the project string userId = isServalAdmin ? GetHighestRankedUserId(project) : curUserId; + // Retrieve the user secret + Attempt attempt = await userSecrets.TryGetAsync(userId, cancellationToken); + if (!attempt.TryResult(out UserSecret userSecret)) + { + throw new DataNotFoundException("The user does not exist."); + } + // Connect to the realtime server await using IConnection connection = await realtimeService.ConnectAsync(userId); string id = TextDocument.GetDocId(sfProjectId, bookNum, chapterNum, TextDocument.Draft); @@ -1536,10 +1543,18 @@ CancellationToken cancellationToken // Retrieve the chapters for this book from the realtime server, if the chapter is zero if (chapterNum == 0) { + // Get the draft project versification so we can get the number of chapters in the book + ScrVers versification = + paratextService.GetParatextSettings(userSecret, project.ParatextId)?.Versification + ?? VerseRef.defaultVersification; + // Just in case the versification is incorrect, use the last chapter in Mongo if it is larger + int lastChapterInMongo = + project.Texts.SingleOrDefault(t => t.BookNum == bookNum)?.Chapters.Max(c => c.Number) ?? 0; + int lastChapter = Math.Max(versification.GetLastChapter(bookNum), lastChapterInMongo); List content = []; - foreach (Chapter chapter in project.Texts.SingleOrDefault(t => t.BookNum == bookNum)?.Chapters ?? []) + for (int chapter = 1; chapter <= lastChapter; chapter++) { - id = TextDocument.GetDocId(sfProjectId, bookNum, chapter.Number, TextDocument.Draft); + id = TextDocument.GetDocId(sfProjectId, bookNum, chapter, TextDocument.Draft); textDocument = await connection.FetchAsync(id); if (textDocument.IsLoaded) { @@ -1589,13 +1604,6 @@ CancellationToken cancellationToken } } - // Retrieve the user secret - Attempt attempt = await userSecrets.TryGetAsync(userId, cancellationToken); - if (!attempt.TryResult(out UserSecret userSecret)) - { - throw new DataNotFoundException("The user does not exist."); - } - DraftUsfmConfig config = draftUsfmConfig ?? project.TranslateConfig.DraftConfig.UsfmConfig ?? new DraftUsfmConfig();