@@ -26,10 +26,16 @@ import {
2626 discoverSnapshotResources ,
2727 createDiscoveryQueue
2828} from './discovery.js' ;
29+ import Monitoring from '@percy/monitoring' ;
2930import { WaitForJob } from './wait-for-job.js' ;
3031
3132const MAX_SUGGESTION_CALLS = 10 ;
3233
34+ // If no activity is done for 5 mins, we will stop monitoring
35+ // system metric eg: (cpu load && memory usage)
36+ const MONITOR_ACTIVITY_TIMEOUT = 300000 ;
37+ const MONITORING_INTERVAL_MS = 5000 ; // 5 sec
38+
3339// A Percy instance will create a new build when started, handle snapshot creation, asset discovery,
3440// and resource uploads, and will finalize the build when stopped. Snapshots are processed
3541// concurrently and the build is not finalized until all snapshots have been handled.
@@ -107,8 +113,15 @@ export class Percy {
107113 this . browser = new Browser ( this ) ;
108114
109115 this . #discovery = createDiscoveryQueue ( this ) ;
116+ this . discoveryMaxConcurrency = this . #discovery. concurrency ;
110117 this . #snapshots = createSnapshotsQueue ( this ) ;
111118
119+ this . monitoring = new Monitoring ( ) ;
120+ // used continue monitoring if there is activity going on
121+ // if there is none, stop it
122+ this . resetMonitoringId = null ;
123+ this . monitoringCheckLastExecutedAt = null ;
124+
112125 // generator methods are wrapped to autorun and return promises
113126 for ( let m of [ 'start' , 'stop' , 'flush' , 'idle' , 'snapshot' , 'upload' ] ) {
114127 // the original generator can be referenced with percy.yield.<method>
@@ -117,6 +130,29 @@ export class Percy {
117130 }
118131 }
119132
133+ systemMonitoringEnabled ( ) {
134+ return ( process . env . PERCY_DISABLE_SYSTEM_MONITORING !== 'true' ) ;
135+ }
136+
137+ async configureSystemMonitor ( ) {
138+ await this . monitoring . startMonitoring ( { interval : MONITORING_INTERVAL_MS } ) ;
139+ this . resetSystemMonitor ( ) ;
140+ }
141+
142+ // Debouncing logic to only stop Monitoring system
143+ // if there is no any activity for 5 mins
144+ // means, no job is pushed in queue from 5 mins
145+ resetSystemMonitor ( ) {
146+ if ( this . resetMonitoringId ) {
147+ clearTimeout ( this . resetMonitoringId ) ;
148+ this . resetMonitoringId = null ;
149+ }
150+
151+ this . resetMonitoringId = setTimeout ( ( ) => {
152+ this . monitoring . stopMonitoring ( ) ;
153+ } , MONITOR_ACTIVITY_TIMEOUT ) ;
154+ }
155+
120156 // Shortcut for controlling the global logger's log level.
121157 loglevel ( level ) {
122158 return logger . loglevel ( level ) ;
@@ -169,6 +205,13 @@ export class Percy {
169205 this . cliStartTime = new Date ( ) . toISOString ( ) ;
170206
171207 try {
208+ // started monitoring system metrics
209+
210+ if ( this . systemMonitoringEnabled ( ) ) {
211+ await this . configureSystemMonitor ( ) ;
212+ await this . monitoring . logSystemInfo ( ) ;
213+ }
214+
172215 if ( process . env . PERCY_CLIENT_ERROR_LOGS !== 'false' ) {
173216 this . log . warn ( 'Notice: Percy collects CI logs to improve service and enhance your experience. These logs help us debug issues and provide insights on your dashboards, making it easier to optimize the product experience. Logs are stored securely for 30 days. You can opt out anytime with export PERCY_CLIENT_ERROR_LOGS=false, but keeping this enabled helps us offer the best support and features.' ) ;
174217 }
@@ -298,13 +341,66 @@ export class Percy {
298341 this . log . error ( err ) ;
299342 throw err ;
300343 } finally {
344+ // stop monitoring system metric, if not already stopped
345+ this . monitoring . stopMonitoring ( ) ;
346+ clearTimeout ( this . resetMonitoringId ) ;
347+
301348 // This issue doesn't comes under regular error logs,
302349 // it's detected if we just and stop percy server
303350 await this . checkForNoSnapshotCommandError ( ) ;
304351 await this . sendBuildLogs ( ) ;
305352 }
306353 }
307354
355+ checkAndUpdateConcurrency ( ) {
356+ // early exit if monitoring is disabled
357+ if ( ! this . systemMonitoringEnabled ( ) ) return ;
358+
359+ // early exit if asset discovery concurrency change is disabled
360+ // NOTE: system monitoring will still be running as only concurrency
361+ // change is disabled
362+ if ( process . env . PERCY_DISABLE_CONCURRENCY_CHANGE === 'true' ) return ;
363+
364+ // start system monitoring if not already doing...
365+ // this doesn't handle cases where there is suggest cpu spikes
366+ // in less 1 sec range and if monitoring is not in running state
367+ if ( this . monitoringCheckLastExecutedAt && Date . now ( ) - this . monitoringCheckLastExecutedAt < MONITORING_INTERVAL_MS ) return ;
368+
369+ if ( ! this . monitoring . running ) this . configureSystemMonitor ( ) ;
370+ else this . resetSystemMonitor ( ) ;
371+
372+ // early return if last executed was less than 5 seconds
373+ // as we will get the same cpu/mem info under 5 sec interval
374+ const { cpuInfo, memoryUsageInfo } = this . monitoring . getMonitoringInfo ( ) ;
375+ this . log . debug ( `cpuInfo: ${ JSON . stringify ( cpuInfo ) } ) ;
376+ this . log . debug ( `memoryInfo: ${ JSON . stringify ( memoryUsageInfo ) } ) ;
377+
378+ if ( cpuInfo . currentUsagePercent >= 80 || memoryUsageInfo . currentUsagePercent >= 80 ) {
379+ let currentConcurrent = this . #discovery. concurrency ;
380+
381+ // concurrency must be betweeen [1, (default/user defined value)]
382+ let newConcurrency = Math . max ( 1 , parseInt ( currentConcurrent / 2 ) ) ;
383+ newConcurrency = Math . min ( this . discoveryMaxConcurrency , newConcurrency ) ;
384+
385+ this . log . debug ( `Downscaling discovery browser concurrency from ${ this . #discovery. concurrency } ${ newConcurrency } ) ;
386+ this . #discovery. set ( { concurrency : newConcurrency } ) ;
387+ } else if ( cpuInfo . currentUsagePercent <= 50 && memoryUsageInfo . currentUsagePercent <= 50 ) {
388+ let currentConcurrent = this . #discovery. concurrency ;
389+ let newConcurrency = currentConcurrent + 2 ;
390+
391+ // concurrency must be betweeen [1, (default/user-defined value)]
392+ newConcurrency = Math . min ( this . discoveryMaxConcurrency , newConcurrency ) ;
393+ newConcurrency = Math . max ( 1 , newConcurrency ) ;
394+
395+ this . log . debug ( `Upscaling discovery browser concurrency from ${ this . #discovery. concurrency } ${ newConcurrency } ) ;
396+ this . #discovery. set ( { concurrency : newConcurrency } ) ;
397+ }
398+
399+ // reset timeout to stop monitoring after no-activity of 5 mins
400+ this . resetSystemMonitor ( ) ;
401+ this . monitoringCheckLastExecutedAt = Date . now ( ) ;
402+ }
403+
308404 // Takes one or more snapshots of a page while discovering resources to upload with the resulting
309405 // snapshots. Once asset discovery has completed for the provided snapshots, the queued task will
310406 // resolve and an upload task will be queued separately.
@@ -351,7 +447,7 @@ export class Percy {
351447 yield * discoverSnapshotResources ( this . #discovery, {
352448 skipDiscovery : this . skipDiscovery ,
353449 dryRun : this . dryRun ,
354-
450+ checkAndUpdateConcurrency : this . checkAndUpdateConcurrency . bind ( this ) ,
355451 snapshots : yield * gatherSnapshots ( options , {
356452 meta : { build : this . build } ,
357453 config : this . config
0 commit comments