diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html index 9c2d925dfc4..5eb1dde6659 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.html @@ -83,7 +83,7 @@

Downloads

any edits made in Scripture Forge since the last sync will not be included
- +
@for (column of columnsToDisplay | slice: 0 : columnsToDisplay.length - 1; track column) { @@ -109,6 +109,26 @@

Downloads

{{ headingsToDisplay[column] }}
+

Training Files

+ + + + + + + + + + + + + +
File Name{{ file.title }} + +
@if (rawLastCompletedBuild?.executionData?.warnings?.length > 0) { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss index 3bde17cd0d4..afc69769457 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.scss @@ -18,3 +18,10 @@ display: flex; align-items: center; } + +.data-table-end { + display: flex; + justify-content: flex-end; + align-items: center; + height: inherit; +} diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.spec.ts index bc4b584ed81..c5f3810e547 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.spec.ts @@ -7,22 +7,28 @@ import { ActivatedRoute } from '@angular/router'; import { saveAs } from 'file-saver'; import { SystemRole } from 'realtime-server/lib/esm/common/models/system-role'; import { createTestProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project-test-data'; +import { TrainingData } from 'realtime-server/lib/esm/scriptureforge/models/training-data'; import { DraftConfig } from 'realtime-server/lib/esm/scriptureforge/models/translate-config'; import { BehaviorSubject, of, throwError } from 'rxjs'; -import { anything, mock, verify, when } from 'ts-mockito'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; import { ActivatedProjectService } from 'xforge-common/activated-project.service'; import { AuthService } from 'xforge-common/auth.service'; +import { FileService } from 'xforge-common/file.service'; import { I18nService } from 'xforge-common/i18n.service'; +import { FileType } from 'xforge-common/models/file-offline-data'; +import { RealtimeQuery } from 'xforge-common/models/realtime-query'; import { NoticeService } from 'xforge-common/notice.service'; import { OnlineStatusService } from 'xforge-common/online-status.service'; import { provideTestOnlineStatus } from 'xforge-common/test-online-status-providers'; import { TestOnlineStatusService } from 'xforge-common/test-online-status.service'; import { configureTestingModule } from 'xforge-common/test-utils'; import { SFProjectProfileDoc } from '../core/models/sf-project-profile-doc'; +import { TrainingDataDoc } from '../core/models/training-data-doc'; import { SFProjectService } from '../core/sf-project.service'; import { BuildDto } from '../machine-api/build-dto'; import { DraftZipProgress } from '../translate/draft-generation/draft-generation'; import { DraftGenerationService } from '../translate/draft-generation/draft-generation.service'; +import { TrainingDataService } from '../translate/draft-generation/training-data/training-data.service'; import { ServalAdministrationService } from './serval-administration.service'; import { ServalProjectComponent } from './serval-project.component'; @@ -36,10 +42,12 @@ const mockActivatedProjectService = mock(ActivatedProjectService); const mockActivatedRoute = mock(ActivatedRoute); const mockAuthService = mock(AuthService); const mockDraftGenerationService = mock(DraftGenerationService); +const mockFileService = mock(FileService); const mockedI18nService = mock(I18nService); const mockNoticeService = mock(NoticeService); const mockSFProjectService = mock(SFProjectService); const mockServalAdministrationService = mock(ServalAdministrationService); +const mockTrainingDataService = mock(TrainingDataService); describe('ServalProjectComponent', () => { configureTestingModule(() => ({ @@ -49,10 +57,12 @@ describe('ServalProjectComponent', () => { { provide: ActivatedRoute, useMock: mockActivatedRoute }, { provide: AuthService, useMock: mockAuthService }, { provide: DraftGenerationService, useMock: mockDraftGenerationService }, + { provide: FileService, useMock: mockFileService }, { provide: I18nService, useMock: mockedI18nService }, { provide: NoticeService, useMock: mockNoticeService }, { provide: OnlineStatusService, useClass: TestOnlineStatusService }, { provide: ServalAdministrationService, useMock: mockServalAdministrationService }, + { provide: TrainingDataService, useMock: mockTrainingDataService }, { provide: SFProjectService, useMock: mockSFProjectService }, provideNoopAnimations() ] @@ -116,8 +126,8 @@ describe('ServalProjectComponent', () => { it('should disable the download button when offline', fakeAsync(() => { const env = new TestEnvironment(); env.onlineStatus = false; - expect(env.firstDownloadButton.innerText).toContain('Download'); - expect(env.firstDownloadButton.disabled).toBe(true); + expect(env.firstSourceDownloadButton.innerText).toContain('Download'); + expect(env.firstSourceDownloadButton.disabled).toBe(true); })); it('should display a notice if the project cannot be downloaded', fakeAsync(() => { @@ -125,24 +135,24 @@ describe('ServalProjectComponent', () => { when(mockServalAdministrationService.downloadProject(anything())).thenReturn( throwError(() => new HttpErrorResponse({ status: 404 })) ); - expect(env.firstDownloadButton.innerText).toContain('Download'); - expect(env.firstDownloadButton.disabled).toBe(false); - env.clickElement(env.firstDownloadButton); + expect(env.firstSourceDownloadButton.innerText).toContain('Download'); + expect(env.firstSourceDownloadButton.disabled).toBe(false); + env.clickElement(env.firstSourceDownloadButton); verify(mockNoticeService.showError(anything())).once(); })); it('should have a download button', fakeAsync(() => { const env = new TestEnvironment(); - expect(env.downloadButtons.length).toBe(4); - expect(env.firstDownloadButton.innerText).toContain('Download'); - expect(env.firstDownloadButton.disabled).toBe(false); + expect(env.sourceDownloadButtons.length).toBe(4); + expect(env.firstSourceDownloadButton.innerText).toContain('Download'); + expect(env.firstSourceDownloadButton.disabled).toBe(false); })); it('should allow clicking of the button to download', fakeAsync(() => { const env = new TestEnvironment(); - expect(env.firstDownloadButton.innerText).toContain('Download'); - expect(env.firstDownloadButton.disabled).toBe(false); - env.clickElement(env.firstDownloadButton); + expect(env.firstSourceDownloadButton.innerText).toContain('Download'); + expect(env.firstSourceDownloadButton.disabled).toBe(false); + env.clickElement(env.firstSourceDownloadButton); expect(saveAs).toHaveBeenCalled(); })); }); @@ -186,6 +196,33 @@ describe('ServalProjectComponent', () => { })); }); + describe('download training data', () => { + it('should show training data saved on a project', fakeAsync(() => { + const env = new TestEnvironment(); + tick(); + env.fixture.detectChanges(); + expect(env.component.trainingDataFiles).toBeDefined(); + expect(env.component.trainingDataFiles.length).toBe(1); + expect(env.component.trainingDataFiles[0].dataId).toBe('dataId01'); + expect(env.component.trainingDataFiles[0].fileUrl).toBe('file-url'); + })); + + it('should disable the download button when offline', fakeAsync(() => { + const env = new TestEnvironment(); + env.onlineStatus = false; + expect(env.downloadTrainingDataButton.disabled).toBe(true); + })); + + it('can download the training data', fakeAsync(() => { + const env = new TestEnvironment(); + tick(); + env.fixture.detectChanges(); + expect(env.downloadTrainingDataButton).not.toBeNull(); + env.clickElement(env.downloadTrainingDataButton); + verify(mockFileService.onlineDownloadFile(FileType.TrainingData, 'file-url', 'training-data-01.csv')).once(); + })); + }); + describe('get last completed build', () => { it('does not get last completed build if project does not have draft books', fakeAsync(() => { const env = new TestEnvironment({ preTranslate: false }); @@ -360,6 +397,19 @@ describe('ServalProjectComponent', () => { when(mockDraftGenerationService.getBuildProgress(anything())).thenReturn(of({ additionalInfo: {} } as BuildDto)); when(mockSFProjectService.hasDraft(anything())).thenReturn(args.preTranslate); when(mockSFProjectService.onlineSetServalConfig(this.mockProjectId, anything())).thenResolve(); + const trainingData: TrainingDataDoc[] = [ + { + id: 'training01', + data: { + fileUrl: 'file-url', + dataId: 'dataId01', + title: 'training-data-01.csv' + } as TrainingData + } as TrainingDataDoc + ]; + const mockQuery: RealtimeQuery = mock(RealtimeQuery); + when(mockQuery.docs).thenReturn(trainingData); + when(mockTrainingDataService.queryTrainingDataAsync(anything(), anything())).thenResolve(instance(mockQuery)); spyOn(saveAs, 'saveAs').and.stub(); @@ -380,16 +430,20 @@ describe('ServalProjectComponent', () => { return this.fixture.nativeElement.querySelector('#view-event-log'); } - get firstDownloadButton(): HTMLInputElement { - return this.fixture.nativeElement.querySelector('td button'); + get firstSourceDownloadButton(): HTMLInputElement { + return this.fixture.nativeElement.querySelector('.draft-sources-table td button'); } - get downloadButtons(): NodeListOf { - return this.fixture.nativeElement.querySelectorAll('td button'); + get sourceDownloadButtons(): NodeListOf { + return this.fixture.nativeElement.querySelectorAll('.draft-sources-table td button'); } get downloadDraftButton(): HTMLInputElement { - return this.fixture.nativeElement.querySelector('#download-draft'); + return this.fixture.nativeElement.querySelector('#download-draft')!; + } + + get downloadTrainingDataButton(): HTMLInputElement { + return this.fixture.nativeElement.querySelector('.training-data-table td button'); } get saveServalConfigButton(): HTMLInputElement { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts index ba94a623077..9225b1e338d 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-project.component.ts @@ -23,17 +23,32 @@ import { import { Router } from '@angular/router'; import { saveAs } from 'file-saver'; import { SFProjectProfile } from 'realtime-server/lib/esm/scriptureforge/models/sf-project'; +import { TrainingData } from 'realtime-server/lib/esm/scriptureforge/models/training-data'; import { DraftConfig, TranslateSource } from 'realtime-server/lib/esm/scriptureforge/models/translate-config'; -import { catchError, firstValueFrom, lastValueFrom, Observable, of, Subscription, switchMap, throwError } from 'rxjs'; +import { + catchError, + filter, + firstValueFrom, + lastValueFrom, + Observable, + of, + Subscription, + switchMap, + throwError +} from 'rxjs'; import { ActivatedProjectService } from 'xforge-common/activated-project.service'; import { DataLoadingComponent } from 'xforge-common/data-loading-component'; +import { FileService } from 'xforge-common/file.service'; import { I18nService } from 'xforge-common/i18n.service'; import { ElementState } from 'xforge-common/models/element-state'; +import { FileType } from 'xforge-common/models/file-offline-data'; +import { RealtimeQuery } from 'xforge-common/models/realtime-query'; import { NoticeService } from 'xforge-common/notice.service'; import { OnlineStatusService } from 'xforge-common/online-status.service'; import { RouterLinkDirective } from 'xforge-common/router-link.directive'; import { filterNullish, quietTakeUntilDestroyed } from 'xforge-common/util/rxjs-util'; import { WriteStatusComponent } from 'xforge-common/write-status/write-status.component'; +import { TrainingDataDoc } from '../core/models/training-data-doc'; import { ParatextService } from '../core/paratext.service'; import { SFProjectService } from '../core/sf-project.service'; import { BuildDto } from '../machine-api/build-dto'; @@ -46,6 +61,7 @@ import { DraftZipProgress } from '../translate/draft-generation/draft-generation import { DraftGenerationService } from '../translate/draft-generation/draft-generation.service'; import { DraftInformationComponent } from '../translate/draft-generation/draft-information/draft-information.component'; import { DraftSourcesAsTranslateSourceArrays, projectToDraftSources } from '../translate/draft-generation/draft-utils'; +import { TrainingDataService } from '../translate/draft-generation/training-data/training-data.service'; import { ServalAdministrationService } from './serval-administration.service'; interface Row { id: string; @@ -133,14 +149,19 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn lastCompletedBuild: BuildDto | undefined; rawLastCompletedBuild: any; zipSubscription: Subscription | undefined; + trainingDataFiles: TrainingData[] = []; + + private trainingDataQuery?: RealtimeQuery; constructor( private readonly activatedProjectService: ActivatedProjectService, private readonly draftGenerationService: DraftGenerationService, private readonly i18n: I18nService, noticeService: NoticeService, + private readonly trainingDataService: TrainingDataService, private readonly onlineStatusService: OnlineStatusService, private readonly projectService: SFProjectService, + private readonly fileService: FileService, private readonly router: Router, private readonly servalAdministrationService: ServalAdministrationService, private destroyRef: DestroyRef @@ -254,6 +275,17 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn this.rawLastCompletedBuild = await firstValueFrom(this.draftGenerationService.getRawBuild(build.id)); } }); + + this.activatedProjectService.projectId$ + .pipe( + filter(p => p != null), + quietTakeUntilDestroyed(this.destroyRef) + ) + .subscribe(async projectId => { + this.trainingDataQuery?.dispose(); + this.trainingDataQuery = await this.trainingDataService.queryTrainingDataAsync(projectId, this.destroyRef); + this.trainingDataFiles = this.trainingDataQuery.docs.map(doc => doc.data).filter(d => d != null); + }); } async downloadDraft(): Promise { @@ -299,6 +331,13 @@ export class ServalProjectComponent extends DataLoadingComponent implements OnIn this.loadingFinished(); } + downloadTrainingData(dataId: string): void { + const trainingData: TrainingData | undefined = this.trainingDataFiles.find(t => t.dataId === dataId); + if (trainingData == null) return this.noticeService.show('File not found'); + + this.fileService.onlineDownloadFile(FileType.TrainingData, trainingData.fileUrl, trainingData.title); + } + onUpdatePreTranslate(newValue: boolean): Promise { return this.projectService.onlineSetPreTranslate(this.activatedProjectService.projectId!, newValue); } diff --git a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.spec.ts b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.spec.ts index 2a2dd507578..460b67e7ff9 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.spec.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.spec.ts @@ -1,5 +1,6 @@ import { HttpTestingController, RequestMatch } from '@angular/common/http/testing'; import { fakeAsync, TestBed, tick } from '@angular/core/testing'; +import { saveAs } from 'file-saver'; import { ProjectData } from 'realtime-server/lib/esm/common/models/project-data'; import { obj, PathItem } from 'realtime-server/lib/esm/common/utils/obj-path'; import { anything, mock, verify, when } from 'ts-mockito'; @@ -225,6 +226,52 @@ describe('FileService', () => { verify(mockedDialogService.message(anything(), anything())).once(); expect(env.doc!.data!.audioUrl).toBeUndefined(); })); + + it('should download the file', fakeAsync(() => { + const env = new TestEnvironment(); + const filename: string = 'training-data.csv'; + const source: string = '/path/to/training-data.csv'; + + // Spy on the saveAs function + spyOn(saveAs, 'saveAs').and.stub(); + + env.service.onlineDownloadFile(FileType.TrainingData, source, filename); + + // Verify the HTTP request was made with the correct URL + const expectedUrl: string = formatFileSource(FileType.TrainingData, source); + const req = env.httpMock.expectOne(expectedUrl); + expect(req.request.method).toBe('GET'); + + // Respond with a blob + const testBlob: Blob = new Blob(); + req.flush(testBlob); + tick(); + expect(saveAs).toHaveBeenCalled(); + env.httpMock.verify(); + })); + + it('detects csv MIME type and adds the file extension', fakeAsync(() => { + const env = new TestEnvironment(); + const filename: string = 'training-data.xlsx'; + const source: string = '/path/to/training-data.xlsx'; + + // Spy on the saveAs function + const saveAsSpy = spyOn(saveAs, 'saveAs').and.stub(); + + env.service.onlineDownloadFile(FileType.TrainingData, source, filename); + + // Verify the HTTP request was made with the correct URL + const expectedUrl: string = formatFileSource(FileType.TrainingData, source); + const req = env.httpMock.expectOne(expectedUrl); + expect(req.request.method).toBe('GET'); + + // Respond with a blob + const testBlob: Blob = new Blob([], { type: 'text/csv' }); + req.flush(testBlob); + tick(); + expect(saveAsSpy).toHaveBeenCalledWith(testBlob, 'training-data.csv'); + env.httpMock.verify(); + })); }); interface ChildData { diff --git a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.ts b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.ts index 252afcd776b..34d2b35772f 100644 --- a/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.ts +++ b/src/SIL.XForge.Scripture/ClientApp/src/xforge-common/file.service.ts @@ -1,5 +1,6 @@ import { HttpClient, HttpHeaders, HttpResponse } from '@angular/common/http'; import { DestroyRef, Injectable } from '@angular/core'; +import { saveAs } from 'file-saver'; import { lastValueFrom, Observable, Subject } from 'rxjs'; import { quietTakeUntilDestroyed } from 'xforge-common/util/rxjs-util'; import { environment } from '../environments/environment'; @@ -219,6 +220,19 @@ export class FileService { } } + onlineDownloadFile(fileType: FileType, source: string, filename: string): void { + const url: string = formatFileSource(fileType, source); + void this.onlineRequestFile(url).then(blob => { + if (blob != null) { + if (blob.type === 'text/csv') { + // use the .csv extension explicitly if the MIME type is csv + filename = filename.split('.')[0] + '.csv'; + } + saveAs(blob, filename); + } + }); + } + private convertToPascalCase(input: string): string { return input .split('-')