Merge pull request #508 from GSA/1745-add-new-snapshots

1745: Add New Snapshot

Author: akuny

.github/workflows/create-snapshot.yml

@@ -12,11 +12,18 @@ import { SecurityDataController } from './security-data.controller';
 
 import pino from 'pino';
 import { getRootLogger } from '../../../libs/logging/src';
-import { logCount } from "../../../libs/logging/src/metric-utils";
+import { logCount } from '../../../libs/logging/src/metric-utils';
 
-function createCommandLogger(commandName: string, options = undefined): pino.Logger {
+function createCommandLogger(
+  commandName: string,
+  options = undefined,
+): pino.Logger {
   const appLogger = getRootLogger();
-  return appLogger.child({ cliCommand: commandName, sseContext: 'Cli.Main', cliOptions: options });
+  return appLogger.child({
+    cliCommand: commandName,
+    sseContext: 'Cli.Main',
+    cliOptions: options,
+  });
 }
 
 async function bootstrap() {
@@ -32,19 +39,24 @@ function printMemoryUsage(logger: pino.Logger) {
   const used = process.memoryUsage();
   for (const key in used) {
     const valueMb = Math.round((used[key] / 1024 / 1024) * 100) / 100;
-    logCount(logger, {
+    logCount(
+      logger,
+      {
         metricUnit: 'megabytes',
       },
       `scanner.core.memory.used.${key}.mb`,
       `Memory used: ${key}: ${valueMb} MB`,
-      valueMb
+      valueMb,
     );
   }
 }
 
 async function ingest(cmdObj) {
   const nestApp = await bootstrap();
-  const logger = createCommandLogger('ingest', { limit: cmdObj.limit, federalSubdomainsUrl: cmdObj.federalSubdomainsUrl });
+  const logger = createCommandLogger('ingest', {
+    limit: cmdObj.limit,
+    federalSubdomainsUrl: cmdObj.federalSubdomainsUrl,
+  });
   const controller = nestApp.get(IngestController);
   logger.info('ingesting target urls');
 
@@ -82,8 +94,14 @@ async function checkQueueStatus() {
   logger.info('checking queue status');
 
   const queueStatus = await controller.getQueueStatus();
-  logger.info({ jobsRemaining: queueStatus.count  }, `${queueStatus.count} jobs remaining in queue`);
-  logger.info({ activeJobs: queueStatus.activeCount  }, `${queueStatus.activeCount} active jobs in queue`);
+  logger.info(
+    { jobsRemaining: queueStatus.count },
+    `${queueStatus.count} jobs remaining in queue`,
+  );
+  logger.info(
+    { activeJobs: queueStatus.activeCount },
+    `${queueStatus.activeCount} active jobs in queue`,
+  );
   printMemoryUsage(logger);
   await nestApp.close();
 }
@@ -101,7 +119,9 @@ async function enqueueSite(cmdObj) {
 
 async function enqueueLimitedScans(cmdObj) {
   const nestApp = await bootstrap();
-  const logger = createCommandLogger('enqueue-limited-scans', { limit: cmdObj.limit });
+  const logger = createCommandLogger('enqueue-limited-scans', {
+    limit: cmdObj.limit,
+  });
   const controller = nestApp.get(QueueController);
   logger.info('enqueueing limited scan jobs');
 
@@ -110,17 +130,6 @@ async function enqueueLimitedScans(cmdObj) {
   await nestApp.close();
 }
 
-async function createSnapshot() {
-  const nestApp = await bootstrap();
-  const logger = createCommandLogger('create-snapshot');
-  const controller = nestApp.get(SnapshotController);
-  logger.info('creating snapshot');
-
-  await controller.weeklySnapshot();
-  printMemoryUsage(logger);
-  await nestApp.close();
-}
-
 async function createDailySnapshot() {
   const nestApp = await bootstrap();
   const logger = createCommandLogger('create-daily-snapshot');
@@ -145,11 +154,15 @@ async function createAccessibilityResultsSnapshot() {
 
 async function scanSite(cmdObj) {
   const nestApp = await bootstrap();
-  const logger = createCommandLogger('scan-site', { url: cmdObj.url, page: cmdObj.page, scan: cmdObj.scan });
+  const logger = createCommandLogger('scan-site', {
+    url: cmdObj.url,
+    page: cmdObj.page,
+    scan: cmdObj.scan,
+  });
   const controller = nestApp.get(ScanController);
 
   logger.info(
-    `Scanning site: ${cmdObj.url}, page: ${cmdObj.page ?? 'ALL'}, scan: ${cmdObj.scan ?? 'ALL'}`
+    `Scanning site: ${cmdObj.url}, page: ${cmdObj.page ?? 'ALL'}, scan: ${cmdObj.scan ?? 'ALL'}`,
   );
 
   await controller.scanSite(cmdObj.url, cmdObj.page, cmdObj.scan);
@@ -232,10 +245,7 @@ async function main() {
     .description(
       'enqueue-site add 1 site from the Website database table to the redis queue',
     )
-    .option(
-      '--url <string>',
-      'queue up one specific site by URL'
-    )
+    .option('--url <string>', 'queue up one specific site by URL')
     .action(enqueueSite);
 
   // queue-status
@@ -245,7 +255,7 @@ async function main() {
       'queue-status retrieves the active and remaining jobs in the redis queue',
     )
     .action(checkQueueStatus);
-  
+
   // queue-limited-scans
   program
     .command('enqueue-limited-scans')
@@ -259,14 +269,6 @@ async function main() {
     )
     .action((cmdObj) => enqueueLimitedScans(cmdObj));
 
-  // create-snapshot
-  program
-    .command('create-snapshot')
-    .description(
-      'create-snapshot writes a CSV and JSON of the current scans to S3',
-    )
-    .action(createSnapshot);
-
   // create-daily-snapshot
   program
     .command('create-daily-snapshot')
@@ -287,7 +289,7 @@ async function main() {
   program
     .command('scan-site')
     .description(
-      'scan-site scans a given URL, which MUST exist in the website table'
+      'scan-site scans a given URL, which MUST exist in the website table',
     )
     .option('--url <string>', 'URL to scan')
     .option('--page <string>', 'Page to scan (optional)')
@@ -321,4 +323,4 @@ async function main() {
   await program.parseAsync(process.argv);
 }
 
-main();
+main();

apps/cli/src/main.ts

@@ -23,7 +23,7 @@ describe('SnapshotController', () => {
   });
 
   it('creates snapshot', () => {
-    snapshotController.weeklySnapshot();
-    expect(mockSnapshotService.weeklySnapshot).toHaveBeenCalled();
+    snapshotController.dailySnapshot();
+    expect(mockSnapshotService.dailySnapshot).toHaveBeenCalled();
   });
 });

apps/cli/src/snapshot.controller.spec.ts

@@ -6,10 +6,6 @@ import { SnapshotService } from '@app/snapshot';
 export class SnapshotController {
   constructor(private readonly snapshotService: SnapshotService) {}
 
-  async weeklySnapshot() {
-    await this.snapshotService.weeklySnapshot();
-  }
-
   async dailySnapshot() {
     await this.snapshotService.dailySnapshot();
   }

apps/cli/src/snapshot.controller.ts

@@ -31,6 +31,27 @@ export class WebsiteService {
   }
 
   async findLiveSnapshotResults(): Promise<Website[]> {
+    const queryBuilder = this.website
+      .createQueryBuilder('website')
+      .innerJoinAndSelect('website.coreResult', 'coreResult')
+      .andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
+      .andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
+        mimeTypes: [
+          'image/jpeg',
+          'application/xml',
+          'application/json',
+          'text/xml',
+        ],
+      })
+      .orderBy({
+        'coreResult.targetUrlBaseDomain': 'ASC',
+        'website.url': 'ASC',
+      });
+
+    return await queryBuilder.getMany();
+  }
+
+  async findLiveFilteredSnapshotResults(): Promise<Website[]> {
     const queryBuilder = this.website
       .createQueryBuilder('website')
       .innerJoinAndSelect('website.coreResult', 'coreResult')
@@ -52,12 +73,14 @@ export class WebsiteService {
     return await queryBuilder.getMany();
   }
 
-  async findUniqueSnapshotResults(): Promise<Website[]> {
+  async findLiveFilteredUniqueSnapshotResults(): Promise<Website[]> {
     const queryBuilder = this.website
       .createQueryBuilder('website')
       .innerJoinAndSelect('website.coreResult', 'coreResult')
       .andWhere('coreResult.filter = :isFilter', { isFilter: false })
-      .andWhere('coreResult.targetUrlRedirects = :isRedirect', { isRedirect: false })
+      .andWhere('coreResult.targetUrlRedirects = :isRedirect', {
+        isRedirect: false,
+      })
       .andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
       .andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
         mimeTypes: [
@@ -110,7 +133,6 @@ export class WebsiteService {
   }
 
   async findWebsitesWithPrimaryTimedOutCoreResults(): Promise<Website[]> {
-
     const queryBuilder = this.website
       .createQueryBuilder('website')
       .innerJoinAndSelect('website.coreResult', 'coreResult')

libs/database/src/websites/websites.service.ts

@@ -4,23 +4,20 @@
 export default () => {
   if (process.env.NODE_ENV === 'production') {
     return {
-      fileNameLive: 'weekly-snapshot',
-      fileNameUnique: 'weekly-snapshot-unique',
-      fileNameAll: 'weekly-snapshot-all',
-      fileNameAccessibility: 'weekly-snapshot-accessibility-details',
-      fileNameDailyLive: 'site-scanning-live-filtered-latest',
-      fileNameDailyUnique: 'site-scanning-live-filtered-unique-latest',
+      fileNameDailyLive: 'site-scanning-live-latest',
+      fileNameDailyLiveFiltered: 'site-scanning-live-filtered-latest',
+      fileNameDailyLiveFilteredUnique:
+        'site-scanning-live-filtered-unique-latest',
       fileNameDailyAll: 'site-scanning-latest',
+      fileNameAccessibility: 'weekly-snapshot-accessibility-details',
     };
   } else {
     return {
-      fileNameLive: `weekly-snapshot-${process.env.NODE_ENV}`,
-      fileNameUnique: `weekly-snapshot-unique-${process.env.NODE_ENV}`,
-      fileNameAll: `weekly-snapshot-all-${process.env.NODE_ENV}`,
-      fileNameAccessibility: `weekly-snapshot-accessibility-details-${process.env.NODE_ENV}`,
-      fileNameDailyLive: `site-scanning-live-filtered-latest-${process.env.NODE_ENV}`,
-      fileNameDailyUnique: `site-scanning-live-filtered-unique-latest-${process.env.NODE_ENV}`,
+      fileNameDailyLive: `site-scanning-live-latest-${process.env.NODE_ENV}`,
+      fileNameDailyLiveFiltered: `site-scanning-live-filtered-latest-${process.env.NODE_ENV}`,
+      fileNameDailyLiveFilteredUnique: `site-scanning-live-filtered-unique-latest-${process.env.NODE_ENV}`,
       fileNameDailyAll: `site-scanning-latest-${process.env.NODE_ENV}`,
+      fileNameAccessibility: `weekly-snapshot-accessibility-details-${process.env.NODE_ENV}`,
     };
   }
 };