system store - reduce load time (part of performance effort 4.20)

Signed-off-by: Amit Prinz Setter <[email protected]>

Author: alphaprinz

config.js

@@ -235,8 +235,8 @@ config.ROOT_KEY_MOUNT = '/etc/noobaa-server/root_keys';
 
 config.DB_TYPE = /** @type {nb.DBType} */ (process.env.DB_TYPE || 'postgres');
 
-config.POSTGRES_DEFAULT_MAX_CLIENTS = 10;
-config.POSTGRES_MD_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 70 : 10;
+config.POSTGRES_DEFAULT_MAX_CLIENTS = 5;
+config.POSTGRES_MD_MAX_CLIENTS = (process.env.LOCAL_MD_SERVER === 'true') ? 70 : 5;
 
 ///////////////////
 // SYSTEM CONFIG //

src/server/bg_services/md_aggregator.js

@@ -68,7 +68,7 @@ async function run_md_aggregator(md_store, system_store, target_now, delay) {
         });
         if (changes) {
             const update = _.omit(changes, 'more_updates');
-            await system_store.make_changes({ update });
+            await system_store.make_changes({ update }, false);
             update_range = !changes.more_updates;
             if (update_range) {
                 await system_store.make_changes({
@@ -78,7 +78,7 @@ async function run_md_aggregator(md_store, system_store, target_now, delay) {
                             global_last_update: range.till_time,
                         }]
                     }
-                });
+                }, false);
             }
             await P.delay(delay);
         } else {
@@ -206,7 +206,7 @@ function find_next_range({
                     },
                 }))
             }
-        });
+        }, false);
     }
 
     // on normal operation the time_diff to close can be closed within a single MD_AGGREGATOR_INTERVAL

src/server/system_services/system_store.js

@@ -394,17 +394,21 @@ class SystemStore extends EventEmitter {
         if (this.data) {
             load_time = this.data.time;
         }
+        let res;
         const since_load = Date.now() - load_time;
         if (since_load < this.START_REFRESH_THRESHOLD) {
-            return this.data;
+            res = this.data;
         } else if (since_load < this.FORCE_REFRESH_THRESHOLD) {
             dbg.warn(`system_store.refresh: system_store.data.time > START_REFRESH_THRESHOLD, since_load = ${since_load}, START_REFRESH_THRESHOLD = ${this.START_REFRESH_THRESHOLD}`);
             this.load().catch(_.noop);
-            return this.data;
+            res = this.data;
         } else {
             dbg.warn(`system_store.refresh: system_store.data.time > FORCE_REFRESH_THRESHOLD, since_load = ${since_load}, FORCE_REFRESH_THRESHOLD = ${this.FORCE_REFRESH_THRESHOLD}`);
-            return this.load();
+            res = this.load();
         }
+        //call refresh periodically
+        P.delay_unblocking(this.START_REFRESH_THRESHOLD).then(this.refresh);
+        return res;
     }
 
     async load(since) {