Improved handling of DB connection acquisition timeouts

* syncVersions now atomic - if listing either S3 or COMS versions fails, throw an error and don't continue
* Increase DB acquire connection timeout to 1.5 mins (configurable via new envar: db.acquireConnectionTimeout)
* Separate out getBucket() errors instead of throwing everything as a 404 (including DB connection-related errors)

Author: norrisng-bc

app/README.md

@@ -48,15 +48,17 @@ The following variables enable and enforce the use of Basic Authentication for r
 
 The following variables configure the use of a backend database to support user-based access control, tagging and other advanced features
 
-| Config Var | Env Var       | Default   | Notes                       |
-| ---------- | ------------- | --------- | --------------------------- |
-| `database` | `DB_DATABASE` | coms      | COMS database name          |
-| `host`     | `DB_HOST`     | localhost | Database conection hostname |
-| `username` | `DB_USERNAME` | app       | Database account username   |
-| `password` | `DB_PASSWORD` |           | Database account password   |
-| `port`     | `DB_PORT`     | 5432      | Database connection port    |
-| `poolMin`  | `DB_POOL_MIN` | 2         | avalable connections        |
-| `poolMax`  | `DB_POOL_MAX` | 10        | available connections       |
+| Config Var                 | Env Var                         | Default   | Notes                                                               |
+|----------------------------|---------------------------------|-----------|---------------------------------------------------------------------|
+| `acquireConnectionTimeout` | `DB_ACQUIRE_CONNECTION_TIMEOUT` | 90000     | Timeout length on acquiring a database connection (in milliseconds) |
+| `database`                 | `DB_DATABASE`                   | coms      | COMS database name                                                  |
+| `host`                     | `DB_HOST`                       | localhost | Database conection hostname                                         |
+| `username`                 | `DB_USERNAME`                   | app       | Database account username                                           |
+| `password`                 | `DB_PASSWORD`                   |           | Database account password                                           |
+| `port`                     | `DB_PORT`                       | 5432      | Database connection port                                            |
+| `poolMin`                  | `DB_POOL_MIN`                   | 2         | avalable connections                                                |
+| `poolMax`                  | `DB_POOL_MAX`                   | 10        | available connections                                               |
+
 
 ### Keycloak Variables
 

app/config/custom-environment-variables.json

@@ -6,6 +6,7 @@
     "username": "BASICAUTH_USERNAME"
   },
   "db": {
+    "acquireConnectionTimeout": "DB_ACQUIRE_CONNECTION_TIMEOUT",
     "database": "DB_DATABASE",
     "host": "DB_HOST",
     "password": "DB_PASSWORD",

app/config/default.json

@@ -1,5 +1,6 @@
 {
   "db": {
+    "acquireConnectionTimeout": 90000,
     "database": "coms",
     "host": "localhost",
     "port": "5432",

app/knexfile.js

@@ -32,6 +32,7 @@ types.setTypeParser(1184, (value) => new Date(value).toISOString());
 const logWrapper = (level, msg) => log.log(level, msg);
 
 module.exports = {
+  acquireConnectionTimeout: config.get('db.acquireConnectionTimeout'),
   client: 'pg',
   connection: {
     host: config.get('db.host'),

app/src/components/utils.js

@@ -115,7 +115,15 @@ const utils = {
       return data;
     } catch (err) {
       log.error(err.message, { function: 'getBucket' });
-      throw new Problem(404, { detail: err.message });
+      if (err.name === 'NotFoundError') {
+        throw new Problem(404, { detail: `bucketId ${bucketId} not found` });
+      }
+      else if (err.name == 'KnexTimeoutError') {
+        throw new Problem(504, { detail: 'Database timeout' });
+      }
+      else {
+        throw new Problem(500, { detail: err.message });
+      }
     }
   },
 

app/src/services/objectQueue.js

@@ -3,7 +3,7 @@ const { NIL: SYSTEM_USER } = require('uuid');
 const { ObjectQueue } = require('../db/models');
 
 /**
- * Max number of parameters in a prepared statement (this is a Postrgres hard-coded limit).
+ * Max number of parameters in a prepared statement (this is a Postgres hard-coded limit).
  * https://www.postgresql.org/docs/current/limits.html ("query parameters")
 */
 const POSTGRES_QUERY_MAX_PARAM_LIMIT = 65535;

app/src/services/sync.js

@@ -238,10 +238,10 @@ const service = {
       const comsObject = typeof object === 'object' ? object : await objectService.read(object, trx);
 
       // Check for COMS and S3 Version statuses
-      const [comsVersions, s3VersionsRaw] = await Promise.allSettled([
+      const [comsVersions, s3VersionsRaw] = await Promise.all([
         versionService.list(comsObject.id, trx),
         storageService.listAllObjectVersions({ filePath: comsObject.path, bucketId: comsObject.bucketId })
-      ]).then(settled => settled.map(promise => promise.value));
+      ]);
 
       // Combine S3 DeleteMarkers and Versions into one array
       const s3Versions = s3VersionsRaw.DeleteMarkers