Merge pull request #400 from Nick-1234531/DOT-6676

changes for Retry to fetch resources and accessToken fix for SSE connection

Author: parthlambdatest

src/lib/processSnapshot.ts

@@ -9,8 +9,9 @@ const globalCache = new NodeCache({ stdTTL: 3600, checkperiod: 600 });
 const MAX_RESOURCE_SIZE = 15 * (1024 ** 2); // 15MB
 var ALLOWED_RESOURCES = ['document', 'stylesheet', 'image', 'media', 'font', 'other'];
 const ALLOWED_STATUSES = [200, 201];
-const REQUEST_TIMEOUT = 1800000;
+const REQUEST_TIMEOUT = 180000;
 const MIN_VIEWPORT_HEIGHT = 1080;
+const MAX_WAIT_FOR_REQUEST_CALL = 30000;
 
 const normalizeSameSite = (value) => {
     if (!value) return 'Lax';
@@ -312,6 +313,8 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
         }
     }
 
+    const pendingRequests = new Set<string>();
+
     // Use route to intercept network requests and discover resources
     await page.route('**/*', async (route, request) => {
         const requestUrl = request.url()
@@ -370,8 +373,14 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
                 body = globalCache.get(requestUrl).body;
             } else {
                 ctx.log.debug(`Resource not found in cache or global cache ${requestUrl} fetching from server`);
+                if(ctx.build.checkPendingRequests){
+                    pendingRequests.add(requestUrl);
+                }
                 response = await page.request.fetch(request, requestOptions);
                 body = await response.body();
+                if(ctx.build.checkPendingRequests){
+                    pendingRequests.delete(requestUrl);
+                }
             }
 
             // handle response
@@ -399,15 +408,26 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
 
                 let responseOfRetry, bodyOfRetry
                 ctx.log.debug(`Resource had a disallowed status ${requestUrl} fetching from server again`);
+                if(ctx.build.checkPendingRequests){
+                    pendingRequests.add(requestUrl);
+                }
                 responseOfRetry = await page.request.fetch(request, requestOptions);
                 bodyOfRetry = await responseOfRetry.body();
-
+                if(ctx.build.checkPendingRequests){
+                    pendingRequests.delete(requestUrl);
+                }
                 if (responseOfRetry && responseOfRetry.status() && ALLOWED_STATUSES.includes(responseOfRetry.status())) {
                     ctx.log.debug(`Handling request after retry ${requestUrl}\n - content-type ${responseOfRetry.headers()['content-type']}`);
                     cache[requestUrl] = {
                         body: bodyOfRetry.toString('base64'),
                         type: responseOfRetry.headers()['content-type']
                     }
+                    if (ctx.config.useGlobalCache) {
+                        globalCache.set(requestUrl, {
+                            body: bodyOfRetry.toString('base64'),
+                            type: responseOfRetry.headers()['content-type']
+                        });
+                    }
                     route.fulfill({
                         status: responseOfRetry.status(),
                         headers: responseOfRetry.headers(),
@@ -458,6 +478,7 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
                 }
             }
 
+
             // Continue the request with the fetched response
             route.fulfill({
                 status: response.status(),
@@ -663,6 +684,7 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
         } catch (error) {
             ctx.log.debug(`Network idle failed due to ${error}`);
         }
+        
 
 
         if (ctx.config.allowedAssets && ctx.config.allowedAssets.length) {
@@ -852,6 +874,28 @@ export default async function processSnapshot(snapshot: Snapshot, ctx: Context):
         ctx.log.debug(`Processed options: ${JSON.stringify(processedOptions)}`);
     }
 
+    // Wait for pending requests to complete
+    const checkPending = async () => {
+        let startTime = Date.now();
+        ctx.log.debug(`${pendingRequests.size} Pending requests before wait for ${snapshot.name}: ${Array.from(pendingRequests)}`);
+        while (pendingRequests.size > 0) {
+          const elapsedTime = Date.now() - startTime;
+          if (elapsedTime >= MAX_WAIT_FOR_REQUEST_CALL) {
+            ctx.log.debug(`Timeout reached (${MAX_WAIT_FOR_REQUEST_CALL/1000}s). Stopping wait for pending requests.`);
+            ctx.log.debug(`${pendingRequests.size} Pending requests after wait for ${snapshot.name}: ${Array.from(pendingRequests)}`);
+            break;
+          }
+          await new Promise(resolve => setTimeout(resolve, 1000)); 
+        }
+        if(pendingRequests.size === 0) {
+            ctx.log.debug(`No pending requests for ${snapshot.name}.`);
+        }
+      };
+
+      if (ctx.build.checkPendingRequests) {
+        await checkPending();
+      }
+
 
     let hasBrowserErrors = false;
     for (let browser in discoveryErrors.browsers) {

src/lib/utils.ts

@@ -758,12 +758,14 @@ export async function listenToSmartUISSE(
     const abortController = new AbortController();
 
     try {
+        const cookieKey = baseURL === 'https://server-events.lambdatest.com' ? 'accessToken' : 'stageAccessToken';
+        
         const response = await fetch(url, {
             method: 'GET',
             headers: {
                 'Accept': 'text/event-stream',
                 'Cache-Control': 'no-cache',
-                'Cookie': `stageAccessToken=Basic ${accessToken}`
+                'Cookie': `${cookieKey}=Basic ${accessToken}`
             },
             signal: abortController.signal
         });

src/tasks/createBuild.ts

@@ -18,6 +18,7 @@ export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRen
                         url: resp.data.buildURL,
                         baseline: resp.data.baseline,
                         useKafkaFlow: resp.data.useKafkaFlow || false,
+                        checkPendingRequests: resp.data.checkPendingRequests || false,
                     }
                     process.env.SMARTUI_BUILD_ID = resp.data.buildId;
                     process.env.SMARTUI_BUILD_NAME = resp.data.buildName;

src/tasks/createBuildExec.ts

@@ -20,6 +20,7 @@ export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRen
                             url: resp.data.buildURL,
                             baseline: resp.data.baseline,
                             useKafkaFlow: resp.data.useKafkaFlow || false,
+                            checkPendingRequests: resp.data.checkPendingRequests || false,
                         }
                         process.env.SMARTUI_BUILD_ID = resp.data.buildId;
                         process.env.SMARTUI_BUILD_NAME = resp.data.buildName;

src/types.ts

@@ -201,6 +201,7 @@ export interface Build {
     useKafkaFlow: boolean;
     hasDiscoveryError: boolean;
     projectId?: string;
+    checkPendingRequests: boolean;
 }
 
 export interface WebConfig {