SF-3651 Allow serval admins to download training data

Author: RaymondLuong3

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html

@@ -83,7 +83,7 @@ <h2>Downloads</h2>
   <em>any edits made in Scripture Forge</em> since the last sync will not be included
 </app-notice>
 <div class="table-container">
-  <table mat-table [dataSource]="rows">
+  <table mat-table [dataSource]="rows" class="draft-sources-table">
     @for (column of columnsToDisplay | slice: 0 : columnsToDisplay.length - 1; track column) {
       <ng-container [matColumnDef]="column">
         <th mat-header-cell *matHeaderCellDef>{{ headingsToDisplay[column] }}</th>
@@ -109,6 +109,26 @@ <h2>Downloads</h2>
     <tr mat-header-row *matHeaderRowDef="columnsToDisplay"></tr>
     <tr mat-row *matRowDef="let myRowData; columns: columnsToDisplay"></tr>
   </table>
+  <h3>Training Files</h3>
+  <table mat-table [dataSource]="trainingDataFiles" class="training-data-table">
+    <ng-container matColumnDef="title">
+      <th mat-header-cell *matHeaderCellDef>File Name</th>
+      <td mat-cell *matCellDef="let file">{{ file.title }}</td>
+    </ng-container>
+
+    <ng-container matColumnDef="download">
+      <th mat-header-cell *matHeaderCellDef></th>
+      <td mat-cell *matCellDef="let file" class="data-table-end">
+        <button mat-flat-button color="primary" (click)="downloadTrainingData(file.dataId)" [disabled]="!isOnline">
+          <mat-icon>download</mat-icon>
+          Download
+        </button>
+      </td>
+    </ng-container>
+
+    <tr mat-header-row *matHeaderRowDef="['title', 'download']"></tr>
+    <tr mat-row *matRowDef="let row; columns: ['title', 'download']"></tr>
+  </table>
 </div>
 
 @if (rawLastCompletedBuild?.executionData?.warnings?.length > 0) {

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss

@@ -18,3 +18,10 @@
   display: flex;
   align-items: center;
 }
+
+.data-table-end {
+  display: flex;
+  justify-content: flex-end;
+  align-items: center;
+  height: inherit;
+}

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.spec.ts

@@ -7,22 +7,28 @@ import { ActivatedRoute } from '@angular/router';
 import { saveAs } from 'file-saver';
 import { SystemRole } from 'realtime-server/lib/esm/common/models/system-role';
 import { createTestProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-test-data';
+import { TrainingData } from 'realtime-server/lib/esm/scriptureforge/models/training-data';
 import { DraftConfig } from 'realtime-server/lib/esm/scriptureforge/models/translate-config';
 import { BehaviorSubject, of, throwError } from 'rxjs';
-import { anything, mock, verify, when } from 'ts-mockito';
+import { anything, instance, mock, verify, when } from 'ts-mockito';
 import { ActivatedProjectService } from 'xforge-common/activated-project.service';
 import { AuthService } from 'xforge-common/auth.service';
+import { FileService } from 'xforge-common/file.service';
 import { I18nService } from 'xforge-common/i18n.service';
 import { NoticeService } from 'xforge-common/notice.service';
 import { OnlineStatusService } from 'xforge-common/online-status.service';
 import { provideTestOnlineStatus } from 'xforge-common/test-online-status-providers';
 import { TestOnlineStatusService } from 'xforge-common/test-online-status.service';
 import { configureTestingModule } from 'xforge-common/test-utils';
+import { FileType } from '../../xforge-common/models/file-offline-data';
+import { RealtimeQuery } from '../../xforge-common/models/realtime-query';
 import { SFProjectProfileDoc } from '../core/models/sf-project-profile-doc';
+import { TrainingDataDoc } from '../core/models/training-data-doc';
 import { SFProjectService } from '../core/sf-project.service';
 import { BuildDto } from '../machine-api/build-dto';
 import { DraftZipProgress } from '../translate/draft-generation/draft-generation';
 import { DraftGenerationService } from '../translate/draft-generation/draft-generation.service';
+import { TrainingDataService } from '../translate/draft-generation/training-data/training-data.service';
 import { ServalAdministrationService } from './serval-administration.service';
 import { ServalProjectComponent } from './serval-project.component';
 
@@ -36,23 +42,27 @@ const mockActivatedProjectService = mock(ActivatedProjectService);
 const mockActivatedRoute = mock(ActivatedRoute);
 const mockAuthService = mock(AuthService);
 const mockDraftGenerationService = mock(DraftGenerationService);
+const mockFileService = mock(FileService);
 const mockedI18nService = mock(I18nService);
 const mockNoticeService = mock(NoticeService);
 const mockSFProjectService = mock(SFProjectService);
 const mockServalAdministrationService = mock(ServalAdministrationService);
+const mockTrainingDataService = mock(TrainingDataService);
 
-describe('ServalProjectComponent', () => {
+fdescribe('ServalProjectComponent', () => {
   configureTestingModule(() => ({
     providers: [
       provideTestOnlineStatus(),
       { provide: ActivatedProjectService, useMock: mockActivatedProjectService },
       { provide: ActivatedRoute, useMock: mockActivatedRoute },
       { provide: AuthService, useMock: mockAuthService },
       { provide: DraftGenerationService, useMock: mockDraftGenerationService },
+      { provide: FileService, useMock: mockFileService },
       { provide: I18nService, useMock: mockedI18nService },
       { provide: NoticeService, useMock: mockNoticeService },
       { provide: OnlineStatusService, useClass: TestOnlineStatusService },
       { provide: ServalAdministrationService, useMock: mockServalAdministrationService },
+      { provide: TrainingDataService, useMock: mockTrainingDataService },
       { provide: SFProjectService, useMock: mockSFProjectService },
       provideNoopAnimations()
     ]
@@ -116,33 +126,33 @@ describe('ServalProjectComponent', () => {
     it('should disable the download button when offline', fakeAsync(() => {
       const env = new TestEnvironment();
       env.onlineStatus = false;
-      expect(env.firstDownloadButton.innerText).toContain('Download');
-      expect(env.firstDownloadButton.disabled).toBe(true);
+      expect(env.firstSourceDownloadButton.innerText).toContain('Download');
+      expect(env.firstSourceDownloadButton.disabled).toBe(true);
     }));
 
     it('should display a notice if the project cannot be downloaded', fakeAsync(() => {
       const env = new TestEnvironment();
       when(mockServalAdministrationService.downloadProject(anything())).thenReturn(
         throwError(() => new HttpErrorResponse({ status: 404 }))
       );
-      expect(env.firstDownloadButton.innerText).toContain('Download');
-      expect(env.firstDownloadButton.disabled).toBe(false);
-      env.clickElement(env.firstDownloadButton);
+      expect(env.firstSourceDownloadButton.innerText).toContain('Download');
+      expect(env.firstSourceDownloadButton.disabled).toBe(false);
+      env.clickElement(env.firstSourceDownloadButton);
       verify(mockNoticeService.showError(anything())).once();
     }));
 
     it('should have a download button', fakeAsync(() => {
       const env = new TestEnvironment();
-      expect(env.downloadButtons.length).toBe(4);
-      expect(env.firstDownloadButton.innerText).toContain('Download');
-      expect(env.firstDownloadButton.disabled).toBe(false);
+      expect(env.sourceDownloadButtons.length).toBe(4);
+      expect(env.firstSourceDownloadButton.innerText).toContain('Download');
+      expect(env.firstSourceDownloadButton.disabled).toBe(false);
     }));
 
     it('should allow clicking of the button to download', fakeAsync(() => {
       const env = new TestEnvironment();
-      expect(env.firstDownloadButton.innerText).toContain('Download');
-      expect(env.firstDownloadButton.disabled).toBe(false);
-      env.clickElement(env.firstDownloadButton);
+      expect(env.firstSourceDownloadButton.innerText).toContain('Download');
+      expect(env.firstSourceDownloadButton.disabled).toBe(false);
+      env.clickElement(env.firstSourceDownloadButton);
       expect(saveAs).toHaveBeenCalled();
     }));
   });
@@ -186,6 +196,33 @@ describe('ServalProjectComponent', () => {
     }));
   });
 
+  describe('download training data', () => {
+    it('should show training data saved on a project', fakeAsync(() => {
+      const env = new TestEnvironment();
+      tick();
+      env.fixture.detectChanges();
+      expect(env.component.trainingDataFiles).toBeDefined();
+      expect(env.component.trainingDataFiles.length).toBe(1);
+      expect(env.component.trainingDataFiles[0].dataId).toBe('dataId01');
+      expect(env.component.trainingDataFiles[0].fileUrl).toBe('file-url');
+    }));
+
+    it('should disable the download button when offline', fakeAsync(() => {
+      const env = new TestEnvironment();
+      env.onlineStatus = false;
+      expect(env.downloadTrainingDataButton.disabled).toBe(true);
+    }));
+
+    it('can download the training data', fakeAsync(() => {
+      const env = new TestEnvironment();
+      tick();
+      env.fixture.detectChanges();
+      expect(env.downloadTrainingDataButton).not.toBeNull();
+      env.clickElement(env.downloadTrainingDataButton);
+      verify(mockFileService.onlineDownloadFile(FileType.TrainingData, 'file-url', 'training-data-01.csv')).once();
+    }));
+  });
+
   describe('get last completed build', () => {
     it('does not get last completed build if project does not have draft books', fakeAsync(() => {
       const env = new TestEnvironment({ preTranslate: false });
@@ -360,6 +397,19 @@ describe('ServalProjectComponent', () => {
       when(mockDraftGenerationService.getBuildProgress(anything())).thenReturn(of({ additionalInfo: {} } as BuildDto));
       when(mockSFProjectService.hasDraft(anything())).thenReturn(args.preTranslate);
       when(mockSFProjectService.onlineSetServalConfig(this.mockProjectId, anything())).thenResolve();
+      const trainingData: TrainingDataDoc[] = [
+        {
+          id: 'training01',
+          data: {
+            fileUrl: 'file-url',
+            dataId: 'dataId01',
+            title: 'training-data-01.csv'
+          } as TrainingData
+        } as TrainingDataDoc
+      ];
+      const mockQuery: RealtimeQuery<TrainingDataDoc> = mock(RealtimeQuery<TrainingDataDoc>);
+      when(mockQuery.docs).thenReturn(trainingData);
+      when(mockTrainingDataService.queryTrainingDataAsync(anything(), anything())).thenResolve(instance(mockQuery));
 
       spyOn(saveAs, 'saveAs').and.stub();
 
@@ -380,16 +430,20 @@ describe('ServalProjectComponent', () => {
       return this.fixture.nativeElement.querySelector('#view-event-log');
     }
 
-    get firstDownloadButton(): HTMLInputElement {
-      return this.fixture.nativeElement.querySelector('td button');
+    get firstSourceDownloadButton(): HTMLInputElement {
+      return this.fixture.nativeElement.querySelector('.draft-sources-table td button');
     }
 
-    get downloadButtons(): NodeListOf<HTMLButtonElement> {
-      return this.fixture.nativeElement.querySelectorAll('td button');
+    get sourceDownloadButtons(): NodeListOf<HTMLButtonElement> {
+      return this.fixture.nativeElement.querySelectorAll('.draft-sources-table td button');
     }
 
     get downloadDraftButton(): HTMLInputElement {
-      return this.fixture.nativeElement.querySelector('#download-draft');
+      return this.fixture.nativeElement.querySelector('#download-draft')!;
+    }
+
+    get downloadTrainingDataButton(): HTMLInputElement {
+      return this.fixture.nativeElement.querySelector('.training-data-table td button');
     }
 
     get saveServalConfigButton(): HTMLInputElement {

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts

@@ -23,17 +23,32 @@ import {
 import { Router } from '@angular/router';
 import { saveAs } from 'file-saver';
 import { SFProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project';
+import { TrainingData } from 'realtime-server/lib/esm/scriptureforge/models/training-data';
 import { DraftConfig, TranslateSource } from 'realtime-server/lib/esm/scriptureforge/models/translate-config';
-import { catchError, firstValueFrom, lastValueFrom, Observable, of, Subscription, switchMap, throwError } from 'rxjs';
+import {
+  catchError,
+  filter,
+  firstValueFrom,
+  lastValueFrom,
+  Observable,
+  of,
+  Subscription,
+  switchMap,
+  throwError
+} from 'rxjs';
 import { ActivatedProjectService } from 'xforge-common/activated-project.service';
 import { DataLoadingComponent } from 'xforge-common/data-loading-component';
+import { FileService } from 'xforge-common/file.service';
 import { I18nService } from 'xforge-common/i18n.service';
 import { ElementState } from 'xforge-common/models/element-state';
+import { FileType } from 'xforge-common/models/file-offline-data';
+import { RealtimeQuery } from 'xforge-common/models/realtime-query';
 import { NoticeService } from 'xforge-common/notice.service';
 import { OnlineStatusService } from 'xforge-common/online-status.service';
 import { RouterLinkDirective } from 'xforge-common/router-link.directive';
 import { filterNullish, quietTakeUntilDestroyed } from 'xforge-common/util/rxjs-util';
 import { WriteStatusComponent } from 'xforge-common/write-status/write-status.component';
+import { TrainingDataDoc } from '../core/models/training-data-doc';
 import { ParatextService } from '../core/paratext.service';
 import { SFProjectService } from '../core/sf-project.service';
 import { BuildDto } from '../machine-api/build-dto';
@@ -46,6 +61,7 @@ import { DraftZipProgress } from '../translate/draft-generation/draft-generation
 import { DraftGenerationService } from '../translate/draft-generation/draft-generation.service';
 import { DraftInformationComponent } from '../translate/draft-generation/draft-information/draft-information.component';
 import { DraftSourcesAsTranslateSourceArrays, projectToDraftSources } from '../translate/draft-generation/draft-utils';
+import { TrainingDataService } from '../translate/draft-generation/training-data/training-data.service';
 import { ServalAdministrationService } from './serval-administration.service';
 interface Row {
   id: string;
@@ -133,14 +149,19 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn
   lastCompletedBuild: BuildDto | undefined;
   rawLastCompletedBuild: any;
   zipSubscription: Subscription | undefined;
+  trainingDataFiles: TrainingData[] = [];
+
+  private trainingDataQuery?: RealtimeQuery<TrainingDataDoc>;
 
   constructor(
     private readonly activatedProjectService: ActivatedProjectService,
     private readonly draftGenerationService: DraftGenerationService,
     private readonly i18n: I18nService,
     noticeService: NoticeService,
+    private readonly trainingDataService: TrainingDataService,
     private readonly onlineStatusService: OnlineStatusService,
     private readonly projectService: SFProjectService,
+    private readonly fileService: FileService,
     private readonly router: Router,
     private readonly servalAdministrationService: ServalAdministrationService,
     private destroyRef: DestroyRef
@@ -254,6 +275,17 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn
           this.rawLastCompletedBuild = await firstValueFrom(this.draftGenerationService.getRawBuild(build.id));
         }
       });
+
+    this.activatedProjectService.projectId$
+      .pipe(
+        filter(p => p != null),
+        quietTakeUntilDestroyed(this.destroyRef)
+      )
+      .subscribe(async projectId => {
+        this.trainingDataQuery?.dispose();
+        this.trainingDataQuery = await this.trainingDataService.queryTrainingDataAsync(projectId, this.destroyRef);
+        this.trainingDataFiles = this.trainingDataQuery.docs.map(doc => doc.data).filter(d => d != null);
+      });
   }
 
   async downloadDraft(): Promise<void> {
@@ -299,6 +331,13 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn
     this.loadingFinished();
   }
 
+  downloadTrainingData(dataId: string): void {
+    const trainingData: TrainingData | undefined = this.trainingDataFiles.find(t => t.dataId === dataId);
+    if (trainingData == null) return this.noticeService.show('File not found');
+
+    this.fileService.onlineDownloadFile(FileType.TrainingData, trainingData.fileUrl, trainingData.title);
+  }
+
   onUpdatePreTranslate(newValue: boolean): Promise<void> {
     return this.projectService.onlineSetPreTranslate(this.activatedProjectService.projectId!, newValue);
   }

src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.spec.ts

@@ -1,5 +1,6 @@
 import { HttpTestingController, RequestMatch } from '@angular/common/http/testing';
 import { fakeAsync, TestBed, tick } from '@angular/core/testing';
+import { saveAs } from 'file-saver';
 import { ProjectData } from 'realtime-server/lib/esm/common/models/project-data';
 import { obj, PathItem } from 'realtime-server/lib/esm/common/utils/obj-path';
 import { anything, mock, verify, when } from 'ts-mockito';
@@ -225,6 +226,29 @@ describe('FileService', () => {
     verify(mockedDialogService.message(anything(), anything())).once();
     expect(env.doc!.data!.audioUrl).toBeUndefined();
   }));
+
+  it('should download the file', fakeAsync(() => {
+    const env = new TestEnvironment();
+    const filename: string = 'training-data.csv';
+    const source: string = '/path/to/training-data.csv';
+
+    // Spy on the saveAs function
+    spyOn(saveAs, 'saveAs').and.stub();
+
+    env.service.onlineDownloadFile(FileType.TrainingData, source, filename);
+
+    // Verify the HTTP request was made with the correct URL
+    const expectedUrl: string = formatFileSource(FileType.TrainingData, source);
+    const req = env.httpMock.expectOne(expectedUrl);
+    expect(req.request.method).toBe('GET');
+
+    // Respond with a blob
+    const testBlob: Blob = new Blob();
+    req.flush(testBlob);
+    tick();
+    expect(saveAs).toHaveBeenCalled();
+    env.httpMock.verify();
+  }));
 });
 
 interface ChildData {

src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.ts

@@ -1,5 +1,6 @@
 import { HttpClient, HttpHeaders, HttpResponse } from '@angular/common/http';
 import { DestroyRef, Injectable } from '@angular/core';
+import { saveAs } from 'file-saver';
 import { lastValueFrom, Observable, Subject } from 'rxjs';
 import { quietTakeUntilDestroyed } from 'xforge-common/util/rxjs-util';
 import { environment } from '../environments/environment';
@@ -219,6 +220,15 @@ export class FileService {
     }
   }
 
+  onlineDownloadFile(fileType: FileType, source: string, filename: string): void {
+    const url: string = formatFileSource(fileType, source);
+    void this.onlineRequestFile(url).then(blob => {
+      if (blob != null) {
+        saveAs(blob, filename);
+      }
+    });
+  }
+
   private convertToPascalCase(input: string): string {
     return input
       .split('-')