store discovery errors

Author: sushobhit-lt

src/lib/httpClient.ts

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import FormData from 'form-data';
 import axios, { AxiosInstance, AxiosRequestConfig } from 'axios';
-import { Env, Snapshot, ProcessedSnapshot, Git, Build, Context } from '../types.js';
+import { Env, Snapshot, ProcessedSnapshot, Git, Build, Context, DiscoveryErrors } from '../types.js';
 import constants from './constants.js';
 import type { Logger } from 'winston'
 import pkgJSON from './../../package.json'
@@ -91,6 +91,9 @@ export default class httpClient {
         if (config && config.data && !config.data.name) {
             log.debug(config.data);
         }
+        if (config && config.data && config.data.snapshotUuid) {
+            log.debug(config.data);
+        }
         return this.axiosInstance.request(config)
             .then(resp => {
                 if (resp) {
@@ -222,7 +225,7 @@ export default class httpClient {
     }
 
 
-    uploadSnapshot(ctx: Context, snapshot: ProcessedSnapshot) {
+    uploadSnapshot(ctx: Context, snapshot: ProcessedSnapshot,  discoveryErrors: DiscoveryErrors) {
         // Use capsBuildId if provided, otherwise fallback to ctx.build.id
         return this.request({
             url: `/builds/${ctx.build.id}/snapshot`,
@@ -233,12 +236,13 @@ export default class httpClient {
                 test: {
                     type: ctx.testType,
                     source: 'cli'
-                }
+                },
+                discoveryErrors: discoveryErrors,
             }
         }, ctx.log)
     }
 
-    processSnapshot(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string) {
+    processSnapshot(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string,  discoveryErrors: DiscoveryErrors) {
         return this.request({
             url: `/build/${ctx.build.id}/snapshot`,
             method: 'POST',
@@ -252,11 +256,12 @@ export default class httpClient {
                     source: 'cli'
                 },
                 async: false,
+                discoveryErrors: discoveryErrors,
             }
         }, ctx.log)
     }
 
-    processSnapshotCaps(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string, capsBuildId: string, capsProjectToken: string) {
+    processSnapshotCaps(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string, capsBuildId: string, capsProjectToken: string, discoveryErrors: DiscoveryErrors) {
         return this.request({
             url: `/build/${capsBuildId}/snapshot`,
             method: 'POST',
@@ -273,11 +278,12 @@ export default class httpClient {
                     source: 'cli'
                 },
                 async: false,
+                discoveryErrors: discoveryErrors,
             }
         }, ctx.log)
     }
 
-    uploadSnapshotForCaps(ctx: Context, snapshot: ProcessedSnapshot, capsBuildId: string, capsProjectToken: string) {
+    uploadSnapshotForCaps(ctx: Context, snapshot: ProcessedSnapshot, capsBuildId: string, capsProjectToken: string, discoveryErrors: DiscoveryErrors) {
         // Use capsBuildId if provided, otherwise fallback to ctx.build.id
         const buildId = capsBuildId !== '' ? capsBuildId : ctx.build.id;
 
@@ -293,7 +299,8 @@ export default class httpClient {
                 test: {
                     type: ctx.testType,
                     source: 'cli'
-                }
+                },
+                discoveryErrors: discoveryErrors,
             }
         }, ctx.log);
     }

src/lib/processSnapshot.ts

@@ -1,4 +1,4 @@
-import { Snapshot, Context, ProcessedSnapshot } from "../types.js";
+import { Snapshot, Context, DiscoveryErrors } from "../types.js";
 import { scrollToBottomAndBackToTop, getRenderViewports, getRenderViewportsForOptions } from "./utils.js"
 import { chromium, Locator } from "@playwright/test"
 import constants from "./constants.js";
@@ -16,7 +16,17 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
     updateLogContext({ task: 'discovery' });
     ctx.log.debug(`Processing snapshot ${snapshot.name} ${snapshot.url}`);
     const isHeadless = process.env.HEADLESS?.toLowerCase() === 'false' ? false : true;
+    let discoveryErrors: DiscoveryErrors = {
+        name: "",
+        url: "",
+        timestamp: "",
+        snapshotUUID: "",
+        browsers: {
+          chrome: {}
+        }
+      };
 
+    let globalViewport = ""
     let launchOptions: Record<string, any> = {
         headless: isHeadless,
         args: constants.LAUNCH_ARGS
@@ -84,7 +94,7 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
                 ...constants.REQUEST_HEADERS
             }
         }
-
+        
         try {
             // abort audio/video media requests
             if (/\.(mp3|mp4|wav|ogg|webm)$/i.test(request.url())) {
@@ -141,10 +151,23 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
                 ctx.log.debug(`Handling request ${requestUrl}\n - skipping already cached resource`);
             } else if (body.length > MAX_RESOURCE_SIZE) {
                 ctx.log.debug(`Handling request ${requestUrl}\n - skipping resource larger than 15MB`);
-            } else if (!ALLOWED_STATUSES.includes(response.status())) {
-                ctx.log.debug(`Handling request ${requestUrl}\n - skipping disallowed status [${response.status()}]`);
             } else if (!ALLOWED_RESOURCES.includes(request.resourceType())) {
                 ctx.log.debug(`Handling request ${requestUrl}\n - skipping disallowed resource type [${request.resourceType()}]`);
+            }  else if (!ALLOWED_STATUSES.includes(response.status())) {
+                ctx.log.debug(`${globalViewport} Handling request ${requestUrl}\n - skipping disallowed status [${response.status()}]`);
+                let data = {
+                    statusCode: `${response.status()}`,
+                    url: requestUrl,
+                    resourceType: request.resourceType(),
+                } 
+                if (!discoveryErrors.browsers.chrome) {
+                    discoveryErrors.browsers.chrome = {};
+                }
+                if (!discoveryErrors.browsers.chrome[globalViewport]) {
+                    discoveryErrors.browsers.chrome[globalViewport] = [];
+                }
+                discoveryErrors.browsers.chrome[globalViewport]?.push(data);
+
             } else {
                 ctx.log.debug(`Handling request ${requestUrl}\n - content-type ${response.headers()['content-type']}`);
 
@@ -304,10 +327,14 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
 
         await page.setViewportSize({ width: viewport.width, height: viewport.height || MIN_VIEWPORT_HEIGHT });
         ctx.log.debug(`Page resized to ${viewport.width}x${viewport.height || MIN_VIEWPORT_HEIGHT}`);
+        globalViewport = viewportString;
+        ctx.log.debug(`globalViewport : ${globalViewport}`);
 
         // navigate to snapshot url once
         if (!navigated) {
             try {
+                discoveryErrors.url = snapshot.url;
+                discoveryErrors.name = snapshot.name;
                 // domcontentloaded event is more reliable than load event
                 await page.goto(snapshot.url, { waitUntil: "domcontentloaded", timeout: ctx.config.waitForDiscovery });
                 // adding extra timeout since domcontentloaded event is fired pretty quickly
@@ -399,6 +426,8 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
         ctx.log.debug(`Processed options: ${JSON.stringify(processedOptions)}`);
     }
 
+    discoveryErrors.timestamp = new Date().toISOString();
+
     return {
         processedSnapshot: {
             name: snapshot.name,
@@ -407,6 +436,7 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
             resources: cache,
             options: processedOptions
         },
-        warnings: [...optionWarnings, ...snapshot.dom.warnings]
+        warnings: [...optionWarnings, ...snapshot.dom.warnings],
+        discoveryErrors: discoveryErrors
     }
 }

src/lib/snapshotQueue.ts

@@ -313,7 +313,7 @@ export default class Queue {
                     }
 
                     // Process and upload snapshot
-                    let { processedSnapshot, warnings } = await processSnapshot(snapshot, this.ctx);
+                    let { processedSnapshot, warnings, discoveryErrors } = await processSnapshot(snapshot, this.ctx);
 
                     if (useCapsBuildId) {
                         if (useKafkaFlowCaps) {
@@ -322,9 +322,9 @@ export default class Queue {
                             const uploadUrl = presignedResponse.data.url;
 
                             await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken)
-                            await this.ctx.client.processSnapshotCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken);
+                            await this.ctx.client.processSnapshotCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken, discoveryErrors);
                         } else {
-                            await this.ctx.client.uploadSnapshotForCaps(this.ctx, processedSnapshot, capsBuildId, capsProjectToken);
+                            await this.ctx.client.uploadSnapshotForCaps(this.ctx, processedSnapshot, capsBuildId, capsProjectToken, discoveryErrors);
                         }
 
                         // Increment snapshot count for the specific buildId
@@ -358,9 +358,9 @@ export default class Queue {
                             const uploadUrl = presignedResponse.data.url;
 
                             await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot)
-                            await this.ctx.client.processSnapshot(this.ctx, processedSnapshot, snapshotUuid);
+                            await this.ctx.client.processSnapshot(this.ctx, processedSnapshot, snapshotUuid, discoveryErrors);
                         } else {
-                            await this.ctx.client.uploadSnapshot(this.ctx, processedSnapshot);
+                            await this.ctx.client.uploadSnapshot(this.ctx, processedSnapshot,  discoveryErrors);
                         }
                         this.ctx.totalSnapshots++;
                     }

src/types.ts

@@ -186,4 +186,23 @@ export interface basicAuth {
 export interface FigmaWebConfig {
     autoDetectViewports: Array<string>;
     configs: Array<{ figma_file_token: string, figma_ids: Array<string>, screenshot_names:Array<string> }>;
-}
+}
+
+
+export interface ViewportErrors {
+    statusCode: "aborted" | "404" | string;
+    url: string;
+    resourceType: string;
+  }
+  
+export interface DiscoveryErrors {
+    name: string;
+    url: string;
+    timestamp: string;
+    snapshotUUID: string;
+    browsers: {
+      [browserName: string]: {
+        [viewport: string]: ViewportErrors[];
+      };
+    };
+  }