Merge pull request #516 from GSA/1784-filter-logic-update

akuny · web-flow · commit aac762128d8e · 2026-03-17T13:06:43.000-04:00
1784: Filter Logic Update
diff --git a/entities/core-result.entity.ts b/entities/core-result.entity.ts
@@ -657,4 +657,14 @@ export class CoreResult {
     'www_status_code',
     'www_title',
   ];
+
+  static filteredMediaTypes = [
+    'application/javascript',
+    'application/json',
+    'application/xml',
+    'application/xhtml+xml',
+    'image/jpeg',
+    'text/javascript',
+    'text/xml',
+  ];
 }
diff --git a/libs/database/src/core-results/core-result.module.ts b/libs/database/src/core-results/core-result.module.ts
@@ -2,9 +2,10 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { CoreResult } from 'entities/core-result.entity';
 import { CoreResultService } from './core-result.service';
+import { WebsiteModule } from '../websites/website.module';
 
 @Module({
-  imports: [TypeOrmModule.forFeature([CoreResult])],
+  imports: [TypeOrmModule.forFeature([CoreResult]), WebsiteModule],
   providers: [CoreResultService],
   exports: [TypeOrmModule, CoreResultService],
 })
diff --git a/libs/database/src/core-results/core-result.service.spec.ts b/libs/database/src/core-results/core-result.service.spec.ts
@@ -5,22 +5,29 @@ import { Website } from 'entities/website.entity';
 import { mock, mockReset } from 'jest-mock-extended';
 import { Repository } from 'typeorm';
 import { CoreResultService } from './core-result.service';
+import { WebsiteService } from '@app/database/websites/websites.service';
 import { Logger } from '@nestjs/common';
 import { ScanStatus } from 'entities/scan-status';
 
 describe('CoreResultService', () => {
   let service: CoreResultService;
   let mockRepository: any;
+  let mockWebsiteService: any;
 
   beforeEach(async () => {
     mockRepository = mock<Repository<CoreResult>>();
+    mockWebsiteService = mock<WebsiteService>();
     const module: TestingModule = await Test.createTestingModule({
       providers: [
         CoreResultService,
         {
           provide: getRepositoryToken(CoreResult),
           useValue: mockRepository,
         },
+        {
+          provide: WebsiteService,
+          useValue: mockWebsiteService,
+        },
       ],
     }).compile();
 
@@ -29,6 +36,7 @@ describe('CoreResultService', () => {
 
   afterEach(async () => {
     mockReset(mockRepository);
+    mockReset(mockWebsiteService);
   });
 
   it('should be defined', () => {
@@ -59,17 +67,6 @@ describe('CoreResultService', () => {
     expect(result).toStrictEqual(coreResult);
   });
 
-  it('should create a CoreResult', async () => {
-    const coreResult = new CoreResult();
-    const website = new Website();
-    website.id = 1;
-    coreResult.id = 1;
-    coreResult.website = website;
-
-    await service.create(coreResult);
-    expect(mockRepository.insert).toHaveBeenCalledWith(coreResult);
-  });
-
   it('should create a CoreResult from CoreResultPages', async () => {
     const websiteId = 1;
     const scanStatus: ScanStatus = ScanStatus['Completed'];
@@ -285,5 +282,201 @@ describe('CoreResultService', () => {
       websiteUrl,
     );
     expect(mockRepository.insert).toHaveBeenCalled();
+    expect(mockWebsiteService.setFilter).not.toHaveBeenCalled();
+    expect(mockRepository.insert).toHaveBeenCalledWith(
+      expect.objectContaining({ filter: false }),
+    );
+  });
+
+  it('should call setFilter when finalUrlMIMEType is a filtered type', async () => {
+    const websiteId = 1;
+    const scanStatus: ScanStatus = ScanStatus['Completed'];
+    const websiteUrl = 'https://example.gov';
+
+    const pages = {
+      base: {
+        targetUrlBaseDomain: 'example.gov',
+      },
+      notFound: {
+        status: scanStatus,
+        result: {
+          notFoundScan: { targetUrl404Test: false },
+        },
+      },
+      primary: {
+        status: scanStatus,
+        result: {
+          urlScan: {
+            targetUrlRedirects: null,
+            finalUrl: null,
+            finalUrlIsLive: null,
+            finalUrlBaseDomain: null,
+            finalUrlWebsite: null,
+            finalUrlTopLevelDomain: null,
+            finalUrlMIMEType: 'application/json',
+            finalUrlSameDomain: null,
+            finalUrlStatusCode: null,
+            finalUrlSameWebsite: null,
+            finalUrlPageHash: null,
+          },
+          dapScan: {
+            dapDetected: null,
+            dapParameters: null,
+            dapVersion: '',
+            gaTagIds: '',
+          },
+          seoScan: {
+            ogTitleFinalUrl: null,
+            ogDescriptionFinalUrl: null,
+            ogArticlePublishedFinalUrl: null,
+            ogArticleModifiedFinalUrl: null,
+            mainElementFinalUrl: null,
+            canonicalLink: null,
+            pageTitle: null,
+            metaDescriptionContent: null,
+            metaKeywordsContent: null,
+            ogImageContent: null,
+            ogTypeContent: null,
+            ogUrlContent: null,
+            htmlLangContent: null,
+            hrefLangContent: null,
+            dcDateContent: null,
+            dcDateCreatedContent: null,
+            dctermsCreatedContent: null,
+            revisedContent: null,
+            lastModifiedContent: null,
+            dateContent: null,
+          },
+          thirdPartyScan: {
+            thirdPartyServiceDomains: null,
+            thirdPartyServiceCount: null,
+            cookieDomains: null,
+            thirdPartyServiceUrls: null,
+          },
+          cookieScan: { domains: '' },
+          uswdsScan: {
+            usaClasses: null,
+            usaClassesUsed: null,
+            uswdsString: null,
+            uswdsInlineCss: null,
+            uswdsUsFlag: null,
+            uswdsStringInCss: null,
+            uswdsUsFlagInCss: null,
+            uswdsPublicSansFont: null,
+            uswdsSemanticVersion: null,
+            uswdsVersion: null,
+            uswdsCount: null,
+            heresHowYouKnowBanner: null,
+          },
+          loginScan: { loginDetected: null, loginProvider: null },
+          cmsScan: { cms: null },
+          requiredLinksScan: {
+            requiredLinksUrl: null,
+            requiredLinksText: null,
+          },
+          searchScan: { searchDetected: null, searchgov: null },
+          mobileScan: { viewportMetaTag: false },
+          toolingScan: { tooling: null },
+        },
+      },
+      robotsTxt: {
+        status: scanStatus,
+        result: {
+          robotsTxtScan: {
+            robotsTxtFinalUrl: null,
+            robotsTxtStatusCode: null,
+            robotsTxtFinalUrlLive: null,
+            robotsTxtDetected: null,
+            robotsTxtFinalUrlMimeType: null,
+            robotsTxtTargetUrlRedirects: null,
+            robotsTxtFinalUrlSize: null,
+            robotsTxtCrawlDelay: null,
+            robotsTxtSitemapLocations: null,
+          },
+        },
+      },
+      sitemapXml: {
+        status: scanStatus,
+        result: {
+          sitemapXmlScan: {
+            sitemapXmlDetected: null,
+            sitemapXmlStatusCode: null,
+            sitemapXmlFinalUrl: null,
+            sitemapXmlFinalUrlLive: null,
+            sitemapTargetUrlRedirects: null,
+            sitemapXmlFinalUrlFilesize: null,
+            sitemapXmlFinalUrlMimeType: null,
+            sitemapXmlLastMod: null,
+            sitemapXmlPageHash: null,
+            sitemapXmlCount: null,
+            sitemapXmlPdfCount: null,
+          },
+        },
+      },
+      dns: {
+        status: scanStatus,
+        result: { dnsScan: { ipv6: true, dnsHostname: null } },
+      },
+      accessibility: {
+        status: scanStatus,
+        result: {
+          accessibilityScan: {
+            accessibilityResults: '',
+            accessibilityResultsList: '',
+          },
+        },
+      },
+      performance: {
+        status: scanStatus,
+        result: {
+          performanceScan: {
+            largestContentfulPaint: null,
+            cumulativeLayoutShift: null,
+          },
+        },
+      },
+      security: {
+        status: scanStatus,
+        result: { securityScan: { httpsEnforced: null, hsts: null } },
+      },
+      clientRedirect: {
+        status: scanStatus,
+        result: {
+          clientRedirectScan: {
+            hasClientRedirect: null,
+            usesJsRedirect: null,
+            usesMetaRefresh: null,
+          },
+        },
+      },
+      www: {
+        status: scanStatus,
+        result: {
+          wwwScan: {
+            wwwFinalUrl: null,
+            wwwStatusCode: null,
+            wwwTitle: null,
+            wwwSame: null,
+          },
+        },
+      },
+    };
+    const logger = mock<Logger>();
+
+    await service.createFromCoreResultPages(
+      websiteId,
+      pages,
+      logger,
+      false,
+      1,
+      1,
+      websiteUrl,
+    );
+
+    expect(mockRepository.insert).toHaveBeenCalled();
+    expect(mockWebsiteService.setFilter).toHaveBeenCalledWith(websiteId, true);
+    expect(mockRepository.insert).toHaveBeenCalledWith(
+      expect.objectContaining({ filter: true }),
+    );
   });
 });
diff --git a/libs/database/src/core-results/core-result.service.ts b/libs/database/src/core-results/core-result.service.ts
@@ -5,21 +5,22 @@ import { CoreResult } from 'entities/core-result.entity';
 import { Website } from 'entities/website.entity';
 import { ScanStatus } from 'entities/scan-status';
 import { CoreResultPages } from 'entities/core-result.entity';
-import e from 'express';
+import { WebsiteService } from '@app/database/websites/websites.service';
 
 @Injectable()
 export class CoreResultService {
   constructor(
     @InjectRepository(CoreResult)
     private coreResultRepository: Repository<CoreResult>,
+    private websiteService: WebsiteService,
   ) {}
 
   async findAll(): Promise<CoreResult[]> {
     const results = await this.coreResultRepository.find();
     return results;
   }
 
-  createFromCoreResultPages(
+  async createFromCoreResultPages(
     websiteId: number,
     pages: CoreResultPages,
     logger: Logger,
@@ -48,10 +49,6 @@ export class CoreResultService {
     this.updateSecurityScanResults(coreResult, pages, logger);
     this.updateWwwScanResults(coreResult, pages, logger);
 
-    return this.create(coreResult);
-  }
-
-  async create(coreResult: CoreResult) {
     const exists = await this.coreResultRepository.findOne({
       where: {
         website: {
@@ -60,6 +57,14 @@ export class CoreResultService {
       },
     });
 
+    if (
+      coreResult.finalUrlMIMEType &&
+      CoreResult.filteredMediaTypes.includes(coreResult.finalUrlMIMEType)
+    ) {
+      coreResult.filter = true;
+      await this.websiteService.setFilter(websiteId, true);
+    }
+
     if (exists) {
       await this.coreResultRepository.update(exists.id, coreResult);
     } else {
diff --git a/libs/database/src/websites/websites.service.ts b/libs/database/src/websites/websites.service.ts
