Merge pull request #424 from GSA/lc/1286-update-snapshots

1286 - Update, create new snapshots

Author: luke-at-flexion

libs/database/src/websites/websites.service.integration.spec.ts

@@ -81,6 +81,7 @@ describe('AnalysisService', () => {
     firstCoreResult.sitemapXmlScanStatus = 'complete';
     firstCoreResult.targetUrlBaseDomain = 'complete';
     firstCoreResult.finalUrlMIMEType = 'text/html';
+    firstCoreResult.filter = false;
 
     const secondCoreResult = new CoreResult();
     secondCoreResult.website = secondWebsite;
@@ -91,6 +92,7 @@ describe('AnalysisService', () => {
     secondCoreResult.sitemapXmlScanStatus = 'complete';
     secondCoreResult.targetUrlBaseDomain = 'complete';
     secondCoreResult.finalUrlMIMEType = 'text/html';
+    secondCoreResult.filter = false;
 
     const thirdCoreResult = new CoreResult();
     thirdCoreResult.website = thirdWebsite;
@@ -101,6 +103,7 @@ describe('AnalysisService', () => {
     thirdCoreResult.sitemapXmlScanStatus = 'complete';
     thirdCoreResult.targetUrlBaseDomain = 'complete';
     thirdCoreResult.finalUrlMIMEType = 'application/json';
+    thirdCoreResult.filter = false;
 
     const fourthCoreResult = new CoreResult();
     fourthCoreResult.website = fourthWebsite;
@@ -111,6 +114,7 @@ describe('AnalysisService', () => {
     fourthCoreResult.sitemapXmlScanStatus = 'complete';
     fourthCoreResult.targetUrlBaseDomain = 'complete';
     fourthCoreResult.finalUrlMIMEType = 'application/html';
+    fourthCoreResult.filter = false;
 
     await websiteRepository.insert(firstWebsite);
     await coreResultRepository.insert(firstCoreResult);

libs/database/src/websites/websites.service.ts

@@ -30,10 +30,30 @@ export class WebsiteService {
     const queryBuilder = this.website
       .createQueryBuilder('website')
       .innerJoinAndSelect('website.coreResult', 'coreResult')
+      .andWhere('coreResult.filter = :isFilter', { isFilter: false })
       .andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
       .andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
         mimeTypes: [
-          'application/xhtml+xml',
+          'image/jpeg',
+          'application/xml',
+          'application/json',
+          'text/xml',
+        ],
+      });
+
+    return await queryBuilder.getMany();
+  }
+
+  async findUniqueSnapshotResults(): Promise<Website[]> {
+    const queryBuilder = this.website
+      .createQueryBuilder('website')
+      .innerJoinAndSelect('website.coreResult', 'coreResult')
+      .andWhere('coreResult.filter = :isFilter', { isFilter: false })
+      .andWhere('coreResult.targetUrlRedirects = :isRedirect', { isRedirect: false })
+      .andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
+      .andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
+        mimeTypes: [
+          'image/jpeg',
           'application/xml',
           'application/json',
           'text/xml',

libs/snapshot/src/config/snapshot.config.ts

@@ -5,12 +5,14 @@ export default () => {
   if (process.env.NODE_ENV === 'production') {
     return {
       fileNameLive: 'weekly-snapshot',
+      fileNameUnique: 'weekly-snapshot-unique',
       fileNameAll: 'weekly-snapshot-all',
       fileNameAccessibility: 'weekly-snapshot-accessibility-details',
     };
   } else {
     return {
       fileNameLive: `weekly-snapshot-${process.env.NODE_ENV}`,
+      fileNameUnique: `weekly-snapshot-unique-${process.env.NODE_ENV}`,
       fileNameAll: `weekly-snapshot-all-${process.env.NODE_ENV}`,
       fileNameAccessibility: `weekly-snapshot-accessibility-details-${process.env.NODE_ENV}`,
     };

libs/snapshot/src/snapshot.service.spec.ts

@@ -43,6 +43,10 @@ describe('SnapshotService', () => {
                 return 'weekly-snapshot';
               }
 
+              if (key === 'fileNameUnique') {
+                return 'weekly-snapshot-unique';
+              }
+
               if (key === 'fileNameAll') {
                 return 'weekly-snapshot-all';
               }
@@ -80,14 +84,15 @@ describe('SnapshotService', () => {
     website.url = 'supremecourt.gov';
 
     mockWebsiteService.findLiveSnapshotResults.mockResolvedValue([website]);
+    mockWebsiteService.findUniqueSnapshotResults.mockResolvedValue([website]);
     mockWebsiteService.findAllSnapshotResults.mockResolvedValue([website]);
 
     await service.weeklySnapshot();
 
     copyDate.setDate(copyDate.getDate() - 7);
     const expectedDate = copyDate.toISOString();
 
-    expect(mockStorageService.copy).toBeCalledTimes(4);
+    expect(mockStorageService.copy).toBeCalledTimes(6);
 
     expect(mockStorageService.copy).toBeCalledWith(
       'weekly-snapshot.json',
@@ -99,6 +104,16 @@ describe('SnapshotService', () => {
       `archive/csv/weekly-snapshot-${expectedDate}.csv`,
     );
 
+    expect(mockStorageService.copy).toBeCalledWith(
+      'weekly-snapshot-unique.json',
+      `archive/json/weekly-snapshot-unique-${expectedDate}.json`,
+    );
+
+    expect(mockStorageService.copy).toBeCalledWith(
+      'weekly-snapshot-unique.csv',
+      `archive/csv/weekly-snapshot-unique-${expectedDate}.csv`,
+    );
+
     expect(mockStorageService.copy).toBeCalledWith(
       'weekly-snapshot-all.json',
       `archive/json/weekly-snapshot-all-${expectedDate}.json`,

libs/snapshot/src/snapshot.service.ts

@@ -20,6 +20,7 @@ export class SnapshotService {
   ) {}
 
   private fileNameLive = this.configService.get<string>('fileNameLive');
+  private fileNameUnique = this.configService.get<string>('fileNameUnique');
   private fileNameAll = this.configService.get<string>('fileNameAll');
   private fileNameAccessibility = this.configService.get<string>(
     'fileNameAccessibility',
@@ -42,6 +43,7 @@ export class SnapshotService {
     const priorDate = date.toISOString();
 
     await this.liveSnapshot(priorDate, CoreResult.snapshotColumnOrder);
+    await this.uniqueSnapshot(priorDate, CoreResult.snapshotColumnOrder);
     await this.allSnapshot(priorDate, CoreResult.snapshotColumnOrder);
   }
 
@@ -69,6 +71,30 @@ export class SnapshotService {
     liveSnapshot = null;
   }
 
+  async uniqueSnapshot(date: string, columns: string[]): Promise<void> {
+    let uniqueWebsites = await this.websiteService.findUniqueSnapshotResults();
+    this.logger.log(
+      `Total number of unique websites retrieved for snapshot: ${uniqueWebsites.length}`,
+    );
+
+    let uniqueSnapshot = new Snapshot(
+      this.storageService,
+      [new JsonSerializer(columns), new CsvSerializer(columns)],
+      uniqueWebsites,
+      date,
+      this.fileNameUnique,
+    );
+
+    await uniqueSnapshot.archiveExisting();
+    this.logger.log('Unique snapshot archived.');
+
+    await uniqueSnapshot.saveNew();
+    this.logger.log('Unique snapshot saved.');
+
+    uniqueWebsites = null;
+    uniqueSnapshot = null;
+  }
+
   async allSnapshot(date: string, columns: string[]) {
     let allWebsites = await this.websiteService.findAllSnapshotResults();
     this.logger.log(