@@ -16,7 +16,7 @@ export default class Queue {
1616 private processing : boolean = false ;
1717 private processingSnapshot : string = '' ;
1818 private ctx : Context ;
19-
19+
2020 constructor ( ctx : Context ) {
2121 this . ctx = ctx ;
2222 }
@@ -28,7 +28,7 @@ export default class Queue {
2828 this . processNext ( ) ;
2929 }
3030 }
31-
31+
3232 private async processNext ( ) : Promise < void > {
3333 if ( ! this . isEmpty ( ) ) {
3434 const snapshot = this . snapshots . shift ( ) ;
@@ -37,10 +37,10 @@ export default class Queue {
3737 let { processedSnapshot, warnings } = await processSnapshot ( snapshot , this . ctx ) ;
3838 await this . ctx . client . uploadSnapshot ( this . ctx , processedSnapshot ) ;
3939 this . ctx . totalSnapshots ++ ;
40- this . processedSnapshots . push ( { name : snapshot . name , warnings} ) ;
40+ this . processedSnapshots . push ( { name : snapshot . name , warnings } ) ;
4141 } catch ( error : any ) {
4242 this . ctx . log . debug ( `snapshot failed; ${ error } ` ) ;
43- this . processedSnapshots . push ( { name : snapshot . name , error : error . message } ) ;
43+ this . processedSnapshots . push ( { name : snapshot . name , error : error . message } ) ;
4444 }
4545 // Close open browser contexts and pages
4646 if ( this . ctx . browser ) {
@@ -77,8 +77,8 @@ export default class Queue {
7777}
7878
7979async function processSnapshot ( snapshot : Snapshot , ctx : Context ) : Promise < Record < string , any > > {
80- updateLogContext ( { task : 'discovery' } ) ;
81- ctx . log . debug ( `Processing snapshot ${ snapshot . name } ` ) ;
80+ updateLogContext ( { task : 'discovery' } ) ;
81+ ctx . log . debug ( `Processing snapshot ${ snapshot . name } ${ snapshot . url } ` ) ;
8282
8383 let launchOptions : Record < string , any > = { headless : true }
8484 let contextOptions : Record < string , any > = {
@@ -92,35 +92,44 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
9292 }
9393 const context = await ctx . browser . newContext ( contextOptions ) ;
9494 ctx . log . debug ( `Browser context created with options ${ JSON . stringify ( contextOptions ) } ` ) ;
95-
9695 // Setting the cookies in playwright context
97- const domainName = new URL ( snapshot . url ) . hostname ;
98- ctx . log . debug ( 'Setting cookies in context for domain:' , domainName ) ;
99-
100-
101- const cookieArray = snapshot . dom . cookies . split ( '; ' ) . map ( cookie => {
102- if ( ! cookie ) return null ;
103- const [ name , value ] = cookie . split ( '=' ) ;
104- if ( ! name || ! value ) return null ;
105-
106- return {
107- name : name . trim ( ) ,
108- value : value . trim ( ) ,
109- domain : domainName ,
110- path : '/'
111- } ;
112- } ) . filter ( Boolean ) ;
113-
114-
115- if ( cookieArray && Array . isArray ( cookieArray ) && cookieArray . length > 0 ) {
116- await context . addCookies ( cookieArray ) ;
117- ctx . log . debug ( 'Cookies added' ) ;
118- } else {
119- ctx . log . debug ( 'No valid cookies to add' ) ;
96+ if ( snapshot . dom . cookies ) {
97+ const domainName = new URL ( snapshot . url ) . hostname ;
98+ ctx . log . debug ( `Setting cookies for domain: ${ domainName } ` ) ;
99+
100+ const cookieArray = snapshot . dom . cookies . split ( '; ' ) . map ( cookie => {
101+ if ( ! cookie ) return null ;
102+ const [ name , value ] = cookie . split ( '=' ) ;
103+ if ( ! name || ! value ) return null ;
104+
105+ return {
106+ name : name . trim ( ) ,
107+ value : value . trim ( ) ,
108+ domain : domainName ,
109+ path : '/'
110+ } ;
111+ } ) . filter ( Boolean ) ;
112+
113+ if ( cookieArray . length > 0 ) {
114+ await context . addCookies ( cookieArray ) ;
115+ } else {
116+ ctx . log . debug ( 'No valid cookies to add' ) ;
117+ }
120118 }
121-
122119 const page = await context . newPage ( ) ;
120+
121+ // populate cache with already captured resources
123122 let cache : Record < string , any > = { } ;
123+ if ( snapshot . dom . resources . length ) {
124+ for ( let resource of snapshot . dom . resources ) {
125+ // convert text/css content to base64
126+ let body = resource . mimetype == 'text/css' ? Buffer . from ( resource . content ) . toString ( 'base64' ) : resource . content ;
127+ cache [ resource . url ] = {
128+ body : body ,
129+ type : resource . mimetype
130+ }
131+ }
132+ }
124133
125134 // Use route to intercept network requests and discover resources
126135 await page . route ( '**/*' , async ( route , request ) => {
@@ -138,7 +147,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
138147 ctx . config . allowedHostnames . push ( new URL ( snapshot . url ) . hostname ) ;
139148 if ( ctx . config . enableJavaScript ) ALLOWED_RESOURCES . push ( 'script' ) ;
140149 if ( ctx . config . basicAuthorization ) {
141- ctx . log . debug ( `Adding basic authorization to the headers for root url` ) ;
150+ ctx . log . debug ( `Adding basic authorization to the headers for root url` ) ;
142151 let token = Buffer . from ( `${ ctx . config . basicAuthorization . username } :${ ctx . config . basicAuthorization . password } ` ) . toString ( 'base64' ) ;
143152 requestOptions . headers = {
144153 ...request . headers ( ) ,
@@ -151,9 +160,15 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
151160 if ( requestUrl === snapshot . url ) {
152161 response = {
153162 status : ( ) => 200 ,
154- headers : ( ) => ( { 'content-type' : 'text/html' } )
163+ headers : ( ) => ( { 'content-type' : 'text/html' } )
164+ }
165+ body = snapshot . dom . html ;
166+ } else if ( cache [ requestUrl ] ) {
167+ response = {
168+ status : ( ) => 200 ,
169+ headers : ( ) => ( { 'content-type' : cache [ requestUrl ] . mimetype } )
155170 }
156- body = snapshot . dom . html
171+ body = cache [ requestUrl ] . body ;
157172 } else {
158173 response = await page . request . fetch ( request , requestOptions ) ;
159174 body = await response . body ( ) ;
@@ -162,7 +177,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
162177 // handle response
163178 if ( ! body ) {
164179 ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping no response` ) ;
165- } else if ( ! body . length ) {
180+ } else if ( ! body . length ) {
166181 ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping empty response` ) ;
167182 } else if ( requestUrl === snapshot . url ) {
168183 ctx . log . debug ( `Handling request ${ requestUrl } \n - skipping root resource` ) ;
@@ -204,7 +219,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
204219 let ignoreOrSelectBoxes : string ;
205220 if ( options && Object . keys ( options ) . length ) {
206221 ctx . log . debug ( `Snapshot options: ${ JSON . stringify ( options ) } ` ) ;
207-
222+
208223 const isNotAllEmpty = ( obj : Record < string , Array < string > > ) : boolean => {
209224 for ( let key in obj ) if ( obj [ key ] ?. length ) return true ;
210225 return false ;
@@ -240,7 +255,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
240255 selectors . push ( ...value ) ;
241256 break ;
242257 }
243- }
258+ }
244259 }
245260 }
246261
@@ -261,14 +276,14 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
261276 // Update `previousDeviceType` to the current device type for comparison in the next iteration
262277 previousDeviceType = device ;
263278
264- await page . setViewportSize ( { width : viewport . width , height : viewport . height || MIN_VIEWPORT_HEIGHT } ) ;
265- ctx . log . debug ( `Page resized to ${ viewport . width } x${ viewport . height || MIN_VIEWPORT_HEIGHT } ` ) ;
266-
279+ await page . setViewportSize ( { width : viewport . width , height : viewport . height || MIN_VIEWPORT_HEIGHT } ) ;
280+ ctx . log . debug ( `Page resized to ${ viewport . width } x${ viewport . height || MIN_VIEWPORT_HEIGHT } ` ) ;
281+
267282 // navigate to snapshot url once
268283 if ( ! navigated ) {
269284 try {
270285 // domcontentloaded event is more reliable than load event
271- await page . goto ( snapshot . url , { waitUntil : "domcontentloaded" } ) ;
286+ await page . goto ( snapshot . url , { waitUntil : "domcontentloaded" } ) ;
272287 // adding extra timeout since domcontentloaded event is fired pretty quickly
273288 await new Promise ( r => setTimeout ( r , 1250 ) ) ;
274289 if ( ctx . config . waitForTimeout ) await page . waitForTimeout ( ctx . config . waitForTimeout ) ;
@@ -278,7 +293,7 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
278293 ctx . log . debug ( `Navigation to discovery page failed; ${ error } ` )
279294 throw new Error ( error . message )
280295 }
281-
296+
282297 }
283298 if ( ctx . config . cliEnableJavaScript && fullPage ) await page . evaluate ( scrollToBottomAndBackToTop , { frequency : 100 , timing : ctx . config . scrollTime } ) ;
284299
@@ -289,8 +304,6 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
289304 ctx . log . debug ( `Network idle failed due to ${ error } ` ) ;
290305 }
291306
292- await new Promise ( r => setTimeout ( r , 1000 ) ) ;
293-
294307 // snapshot options
295308 if ( processedOptions . element ) {
296309 let l = await page . locator ( processedOptions . element ) . all ( )
@@ -323,18 +336,6 @@ async function processSnapshot(snapshot: Snapshot, ctx: Context): Promise<Record
323336 }
324337 }
325338
326- // add dom resources to cache
327- if ( snapshot . dom . resources . length ) {
328- for ( let resource of snapshot . dom . resources ) {
329- // convert text/css content to base64
330- let body = resource . mimetype == 'text/css' ? Buffer . from ( resource . content ) . toString ( 'base64' ) : resource . content ;
331- cache [ resource . url ] = {
332- body : body ,
333- type : resource . mimetype
334- }
335- }
336- }
337-
338339 return {
339340 processedSnapshot : {
340341 name : snapshot . name ,
0 commit comments