Merge pull request #513 from GSA/1780-tooling-scan-prototype

akuny · web-flow · commit 01a049c012d8 · 2026-03-10T13:28:19.000-04:00
1780 tooling scan prototype
diff --git a/apt.yml b/apt.yml
@@ -1,41 +1,17 @@
 ---
 packages:
-  - gconf-service
-  - libasound2
+  - libnss3
+  - libnspr4
   - libatk1.0-0
-  - libc6
-  - libcairo2
+  - libatk-bridge2.0-0
   - libcups2
-  - libdbus-1-3
-  - libexpat1
-  - libfontconfig1
-  - libgcc1
-  - libgconf-2-4
-  - libgdk-pixbuf2.0-0
-  - libglib2.0-0
-  - libgtk-3-0
-  - libnspr4
-  - libpango-1.0-0
-  - libpangocairo-1.0-0
-  - libstdc++6
-  - libx11-6
-  - libx11-xcb1
-  - libxcb1
   - libxcomposite1
-  - libxcursor1
   - libxdamage1
-  - libxext6
-  - libxfixes3
-  - libxi6
   - libxrandr2
-  - libxrender1
+  - libgbm-dev
+  - libpango-1.0-0
+  - libcairo2
+  - libasound2
   - libxss1
   - libxtst6
-  - ca-certificates
   - fonts-liberation
-  - libappindicator1
-  - libnss3
-  - lsb-release
-  - xdg-utils
-  - wget
-  - libgbm-dev
diff --git a/entities/core-result.entity.ts b/entities/core-result.entity.ts
@@ -538,6 +538,18 @@ export class CoreResult {
   @Exclude()
   dateContent?: string;
 
+  @Column({ nullable: true })
+  @Expose({ name: 'tooling' })
+  @Exclude()
+  @Transform(({ value }: { value: string }) => {
+    if (value) {
+      return value.split(',');
+    } else {
+      return null;
+    }
+  })
+  tooling?: string;
+
   static getColumnNames(): string[] {
     // return class-transformer version of column names
     return Object.keys(classToPlain(new CoreResult()));
diff --git a/entities/scan-data.entity.ts b/entities/scan-data.entity.ts
@@ -157,3 +157,7 @@ export type WwwScan = {
   wwwTitle: string;
   wwwSame: boolean;
 };
+
+export type ToolingScan = {
+  tooling: string;
+};
diff --git a/entities/scan-page.entity.ts b/entities/scan-page.entity.ts
@@ -27,6 +27,7 @@ export type PrimaryScans = {
   requiredLinksScan: ScanData.RequiredLinksScan;
   searchScan: ScanData.SearchScan;
   mobileScan: ScanData.MobileScan;
+  toolingScan: ScanData.ToolingScan;
 };
 export type PrimaryScan = PageScan<PrimaryScans>;
 
diff --git a/libs/core-scanner/src/core-scanner.service.ts b/libs/core-scanner/src/core-scanner.service.ts
@@ -3,17 +3,14 @@ import { HttpService } from '@nestjs/axios';
 import { InjectPinoLogger, PinoLogger } from 'nestjs-pino';
 import { Logger } from 'pino';
 import { Browser } from 'puppeteer';
-
 import { BrowserService } from '@app/browser';
 import { SecurityDataService } from '@app/security-data';
-
 import {
   AnyFailureStatus,
   parseBrowserError,
   ScanStatus,
 } from 'entities/scan-status';
 import { Scanner } from 'libs/scanner.interface';
-
 import { CoreInputDto } from './core.input.dto';
 import * as pages from './pages';
 import { Page } from './pages';
diff --git a/libs/core-scanner/src/pages/primary.ts b/libs/core-scanner/src/pages/primary.ts
@@ -1,10 +1,8 @@
 import { Logger } from 'pino';
 import { Page } from 'puppeteer';
-
 import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { buildUrlScanResult } from '@app/core-scanner/scans/url-scan';
 import { PrimaryScans } from 'entities/scan-page.entity';
-
 import { buildDapResult } from '../scans/dap';
 import { buildSeoResult } from '../scans/seo';
 import { buildThirdPartyResult } from '../scans/third-party';
@@ -20,9 +18,8 @@ import { buildRequiredLinksResult } from '../scans/required-links';
 import { buildCookieResult } from '../scans/cookies';
 import { buildSearchResult } from '../scans/search';
 import { buildMobileResult } from '../scans/mobile';
-
+import { buildToolingResult } from '../scans/tooling';
 import { logCount, logTimer } from '../../../logging/src/metric-utils';
-
 import { createRequestHandlers } from '../util';
 
 export const createPrimaryScanner = (logger: Logger, input: CoreInputDto) => {
@@ -117,6 +114,13 @@ const primaryScan = async (
     'MobileScan',
     url,
   );
+  const wrappedToolingResult = runScan(
+    input,
+    pageLogger,
+    buildToolingResult,
+    'ToolingScan',
+    url,
+  );
 
   const [
     urlScan,
@@ -130,6 +134,7 @@ const primaryScan = async (
     requiredLinksScan,
     searchScan,
     mobileScan,
+    toolingScan,
   ] = await promiseAll([
     buildUrlScanResult(input, page, response, pageLogger),
     wrappedDapResult(getOutboundRequests(), page),
@@ -142,6 +147,7 @@ const primaryScan = async (
     wrappedRequiredLinksResult(page),
     wrappedSearchResult(page),
     wrappedMobileResult(page),
+    wrappedToolingResult(page),
   ]);
 
   return {
@@ -156,6 +162,7 @@ const primaryScan = async (
     requiredLinksScan,
     searchScan,
     mobileScan,
+    toolingScan,
   };
 
   function runScan<T>(
diff --git a/libs/core-scanner/src/scans/tooling.spec.ts b/libs/core-scanner/src/scans/tooling.spec.ts
@@ -0,0 +1,76 @@
+import { buildToolingResult } from './tooling';
+import { browserInstance, newTestPage } from '../test-helper';
+import pino from 'pino';
+
+const mockLogger = pino();
+
+describe('tooling scan', () => {
+  it('detects Bootstrap via link href', async () => {
+    await newTestPage(async ({ page }) => {
+      await page.setContent(`
+        <html>
+          <head>
+            <link rel="stylesheet" href="/css/bootstrap.min.css">
+          </head>
+          <body></body>
+        </html>
+      `);
+      expect(await buildToolingResult(mockLogger, page)).toEqual({
+        tooling: 'bootstrap',
+      });
+    });
+  });
+
+  it('detects Bootstrap via characteristic classes', async () => {
+    await newTestPage(async ({ page }) => {
+      await page.setContent(`
+        <html>
+          <head></head>
+          <body>
+            <div class="container-fluid"></div>
+          </body>
+        </html>
+      `);
+      expect(await buildToolingResult(mockLogger, page)).toEqual({
+        tooling: 'bootstrap',
+      });
+    });
+  });
+
+  it('detects multiple libraries', async () => {
+    await newTestPage(async ({ page }) => {
+      await page.setContent(`
+        <html>
+          <head>
+            <link rel="stylesheet" href="/css/bootstrap.min.css">
+            <link rel="stylesheet" href="/css/animate.min.css">
+          </head>
+          <body></body>
+        </html>
+      `);
+      expect(await buildToolingResult(mockLogger, page)).toEqual({
+        tooling: 'bootstrap,animate.css',
+      });
+    });
+  });
+
+  it('returns null when no libraries are detected', async () => {
+    await newTestPage(async ({ page }) => {
+      await page.setContent(`
+        <html>
+          <head></head>
+          <body><p>Nothing here</p></body>
+        </html>
+      `);
+      expect(await buildToolingResult(mockLogger, page)).toEqual({
+        tooling: null,
+      });
+    });
+  });
+
+  afterAll(async () => {
+    if (browserInstance) {
+      await browserInstance.close();
+    }
+  });
+});
diff --git a/libs/core-scanner/src/scans/tooling.ts b/libs/core-scanner/src/scans/tooling.ts
@@ -0,0 +1,108 @@
+import { Logger } from 'pino';
+import { Page } from 'puppeteer';
+import { ToolingScan } from 'entities/scan-data.entity';
+
+export const buildToolingResult = async (
+  logger: Logger,
+  page: Page,
+): Promise<ToolingScan> => {
+  const cssLibraryResults = await getCssLibraryResults(page);
+
+  return {
+    tooling: cssLibraryResults.length ? cssLibraryResults : null,
+  };
+};
+
+async function getCssLibraryResults(page: Page): Promise<string> {
+  const result = await page.evaluate(() => {
+    const detected: string[] = [];
+
+    const linkHrefs = Array.from(
+      document.querySelectorAll('link[rel="stylesheet"]'),
+    ).map((el) => el.getAttribute('href')?.toLowerCase() ?? '');
+
+    const scriptSrcs = Array.from(document.querySelectorAll('script')).map(
+      (el) => el.getAttribute('src')?.toLowerCase() ?? '',
+    );
+
+    // Bootstrap
+    if (
+      linkHrefs.some((href) => href.includes('bootstrap')) ||
+      document.querySelector('.container-fluid, .navbar-toggler, .btn-primary')
+    ) {
+      detected.push('bootstrap');
+    }
+
+    // Tailwind
+    if (
+      linkHrefs.some((href) => href.includes('tailwind')) ||
+      scriptSrcs.some((src) => src.includes('tailwind'))
+    ) {
+      detected.push('tailwind');
+    }
+
+    // Foundation
+    if (
+      linkHrefs.some((href) => href.includes('foundation')) ||
+      document.querySelector('.top-bar, .callout, .orbit-container')
+    ) {
+      detected.push('foundation');
+    }
+
+    // Animate.css
+    if (
+      linkHrefs.some((href) => href.includes('animate')) ||
+      document.querySelector('.animate__animated, .animated')
+    ) {
+      detected.push('animate.css');
+    }
+
+    // Bulma
+    if (
+      linkHrefs.some((href) => href.includes('bulma')) ||
+      document.querySelector('.hero-body, .navbar-burger, .is-primary')
+    ) {
+      detected.push('bulma');
+    }
+
+    // Materialize
+    if (
+      linkHrefs.some((href) => href.includes('materialize')) ||
+      document.querySelector(
+        '.materialize-red, .waves-effect, .collection-item',
+      )
+    ) {
+      detected.push('materialize');
+    }
+
+    // Semantic UI
+    if (
+      linkHrefs.some((href) => href.includes('semantic')) ||
+      document.querySelector('.ui.button, .ui.menu, .ui.container')
+    ) {
+      detected.push('semantic-ui');
+    }
+
+    // UIkit
+    if (
+      linkHrefs.some((href) => href.includes('uikit')) ||
+      document.querySelector('[uk-grid], [uk-slider], [uk-navbar]')
+    ) {
+      detected.push('uikit');
+    }
+
+    // Material Design Lite
+    if (
+      linkHrefs.some(
+        (href) => href.includes('material.min') || href.includes('mdl'),
+      ) ||
+      document.querySelector('.mdl-button, .mdl-layout, .mdl-grid')
+    ) {
+      detected.push('material-design-lite');
+    }
+
+    return detected;
+  });
+
+  return result.join(',');
+}
diff --git a/libs/database/src/core-results/core-result.service.spec.ts b/libs/database/src/core-results/core-result.service.spec.ts
@@ -7,7 +7,6 @@ import { Repository } from 'typeorm';
 import { CoreResultService } from './core-result.service';
 import { Logger } from '@nestjs/common';
 import { ScanStatus } from 'entities/scan-status';
-import { filter } from 'lodash';
 
 describe('CoreResultService', () => {
   let service: CoreResultService;
@@ -177,6 +176,9 @@ describe('CoreResultService', () => {
           mobileScan: {
             viewportMetaTag: false,
           },
+          toolingScan: {
+            tooling: null,
+          },
         },
       },
       robotsTxt: {
diff --git a/libs/database/src/core-results/core-result.service.ts b/libs/database/src/core-results/core-result.service.ts
@@ -1,7 +1,6 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
-
 import { CoreResult } from 'entities/core-result.entity';
 import { Website } from 'entities/website.entity';
 import { ScanStatus } from 'entities/scan-status';
@@ -215,6 +214,9 @@ export class CoreResultService {
 
       // Mobile scan
       coreResult.viewportMetaTag = result.mobileScan.viewportMetaTag;
+
+      // Tooling scan
+      coreResult.tooling = result.toolingScan.tooling;
     } else {
       logger.error({
         msg: pages.primary.error,
@@ -268,6 +270,7 @@ export class CoreResultService {
       coreResult.ogUrlContent = null;
       coreResult.htmlLangContent = null;
       coreResult.hrefLangContent = null;
+      coreResult.tooling = null;
     }
   }
 
diff --git a/libs/datetime/src/datetime.service.spec.ts b/libs/datetime/src/datetime.service.spec.ts
