Fb/plugin fts config automation (#54)

* initialize mixes runtime config with config from file

* initialize mixes runtime config with config from file 2

* config dir should fall back to working dir if it is not explicit

* testing for mixing 1

* remove postinstall script

* slightly better makeExclusiveReturning implementation

* snapshots

* finish config precedence test

* fix eslint problem

* start of consolidateFromFile

* wip 1

* wip 2

* wip 3

* wip 4

* wip 5

* wip 6

* wip 7

* wip 8

* wip 9

* use auto configuration in example-cap-server

* better naming for example cap server code

* isolate fts features always enabled to dedicated user zork

* add an explanatory note to _discoverFtsAutoConfig

* saner usage of CONFIG_SOURCE enum

* better __config source setting

* fix for processConfig verror info

* simpler and safer total count

Author: rlindner81

cds-plugin.js

@@ -1,26 +1,36 @@
 // https://cap.cloud.sap/docs/node.js/cds-plugins
 "use strict";
 
+const { promisify } = require("util");
+const fs = require("fs");
+const pathlib = require("path");
+
 const cds = require("@sap/cds");
 const cdsPackage = require("@sap/cds/package.json");
 const toggles = require("./src/");
 const { closeMainClient, closeSubscriberClient } = require("./src/redisWrapper");
 
 const FEATURE_KEY_REGEX = /\/fts\/([^\s/]+)$/;
+const FTS_AUTO_CONFIG = {
+  type: "boolean",
+  fallbackValue: false,
+};
+
+const readDirAsync = promisify(fs.readdir);
 
 const doEnableHeaderFeatures = cds.env.profiles?.includes("development");
 const isBuild = cds.build?.register;
 
 const _overwriteUniqueName = (envFeatureToggles) => {
-  const uniqueName = envFeatureToggles.uniqueName;
+  const uniqueName = envFeatureToggles?.uniqueName;
   if (!uniqueName) {
     return;
   }
   toggles._reset({ uniqueName });
 };
 
 const _overwriteServiceAccessRoles = (envFeatureToggles) => {
-  if (!Array.isArray(envFeatureToggles.serviceAccessRoles)) {
+  if (!Array.isArray(envFeatureToggles?.serviceAccessRoles)) {
     return;
   }
   cds.on("loaded", (csn) => {
@@ -77,9 +87,26 @@ const _registerClientCloseOnShutdown = () => {
   });
 };
 
+const _discoverFtsAutoConfig = async () => {
+  const root = process.env.ROOT ?? process.cwd();
+  const ftsRoot = pathlib.join(root, "fts");
+  let result;
+  try {
+    result = (await readDirAsync(ftsRoot, { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory())
+      .reduce((acc, curr) => {
+        const key = `/fts/${curr.name}`; // NOTE: this has to match FEATURE_KEY_REGEX
+        acc[key] = Object.assign({}, FTS_AUTO_CONFIG);
+        return acc;
+      }, {});
+  } catch (err) {} // eslint-disable-line no-empty
+  return result;
+};
+
 const activate = async () => {
   const envFeatureToggles = cds.env.featureToggles;
-  if (!envFeatureToggles?.config && !envFeatureToggles?.configFile) {
+  const ftsAutoConfig = await _discoverFtsAutoConfig();
+  if (!envFeatureToggles?.config && !envFeatureToggles?.configFile && !ftsAutoConfig) {
     return;
   }
   _overwriteUniqueName(envFeatureToggles);
@@ -90,8 +117,9 @@ const activate = async () => {
     return;
   }
   await toggles.initializeFeatures({
-    config: envFeatureToggles.config,
-    configFile: envFeatureToggles.configFile,
+    config: envFeatureToggles?.config,
+    configFile: envFeatureToggles?.configFile,
+    configAuto: ftsAutoConfig,
   });
 
   _registerFeatureProvider();
@@ -102,8 +130,4 @@ const activate = async () => {
 const doExportActivateAsProperty =
   cdsPackage.version.localeCompare("7.3.0", undefined, { numeric: true, sensitivity: "base" }) < 0;
 
-module.exports = doExportActivateAsProperty
-  ? {
-      activate,
-    }
-  : activate();
+module.exports = doExportActivateAsProperty ? { activate } : activate();

example-cap-server/.cdsrc.json

@@ -17,8 +17,7 @@
         "bob": {
           "tenant": "people",
           "password": "bob",
-          "id": "[email protected]",
-          "features": ["*"]
+          "id": "[email protected]"
         },
         "clark": {
           "tenant": "pets",
@@ -29,6 +28,12 @@
           "tenant": "pets",
           "password": "danny",
           "id": "[email protected]"
+        },
+        "zork": {
+          "tenant": "people",
+          "password": "zork",
+          "id": "[email protected]",
+          "features": ["*"]
         }
       }
     }

example-cap-server/http/check-service.http

@@ -13,3 +13,7 @@ Authorization: Basic clark clark
 ### priority | danny
 GET {{base_url}}/rest/check/priority
 Authorization: Basic danny danny
+
+### priority | zork | features = [*]
+GET {{base_url}}/rest/check/priority
+Authorization: Basic zork zork

example-cap-server/srv/feature/toggles.yaml

@@ -1,10 +1,11 @@
 ---
-# info: this is a cds feature, identified by the /fts/<feature-name> pattern. it is active if the value is truthy.
-/fts/check-service-extension:
-  type: boolean
-  fallbackValue: false
-  validations:
-    - scopes: [user, tenant]
+# NOTE: uncomment in case you want to define custom validation for this fts toggle
+# info: this is a cds feature, identified by the /fts/<feature-name> pattern
+#/fts/check-service-extension:
+#  type: boolean
+#  fallbackValue: false
+#  validations:
+#    - scopes: [user, tenant]
 
 # info: check api priority; 0 means access is disabled
 /check/priority:

example-cap-server/srv/service/check-service.js

@@ -14,15 +14,17 @@ const HIGH_BOUNDARY = 100;
 
 const priorityHandler = async (context) => {
   const { "CheckService.priority": priority } = context.model.definitions;
-  const isToggled = Boolean(priority["@marked"]);
+  const isFtsToggled = Boolean(priority["@marked"]);
   const value = toggles.getFeatureValue(CHECK_API_PRIORITY, { user: context.user.id, tenant: context.tenant });
   const messages =
     value >= HIGH_BOUNDARY
       ? HIGH_VALUE_RESPONSES
       : value >= MEDIUM_BOUNDARY
         ? MEDIUM_VALUE_RESPONSES
         : LOW_VALUE_RESPONSES;
-  const message = [value, messages[Math.floor(Math.random() * messages.length)], `isToggled ${isToggled}`].join(" | ");
+  const message = [value, messages[Math.floor(Math.random() * messages.length)], `isFtsToggled ${isFtsToggled}`].join(
+    " | "
+  );
   return context.reply(message);
 };
 

package.json

@@ -9,7 +9,7 @@
     "index.cds"
   ],
   "scripts": {
-    "postinstall": "npx patch-package",
+    "patch": "npx patch-package",
     "test": "jest",
     "test:remove-inline-snapshots": "npx replace '\\.toMatchInlineSnapshot\\(\\s*`[\\s\\S]*?`\\s*\\);' '.toMatchInlineSnapshot();' test -r --include='*.test.js'",
     "lint": "npm run prettier && npm run eslint",
@@ -21,7 +21,7 @@
     "docs": "cd docs && bundle exec jekyll serve",
     "docs:install": "cd docs && npx shx rm -rf vendor Gemfile.lock && bundle install",
     "cloc": "npx cloc --vcs=git --read-lang-def=cloc.def src",
-    "upgrade-lock": "npx shx rm -rf package-lock.json node_modules && npm i --package-lock"
+    "upgrade-lock": "npx shx rm -rf package-lock.json node_modules && npm i --package-lock && npm run patch"
   },
   "engines": {
     "node": ">=18.0.0"

src/featureToggles.js

@@ -23,7 +23,7 @@ const { REDIS_INTEGRATION_MODE } = redis;
 const { Logger } = require("./logger");
 const cfEnv = require("./env");
 const { HandlerCollection } = require("./shared/handlerCollection");
-const { ENV, isObject, tryRequire } = require("./shared/static");
+const { ENV, isObject, tryRequire, tryPathReadable } = require("./shared/static");
 const { promiseAllDone } = require("./shared/promiseAllDone");
 const { LimitedLazyCache } = require("./shared/cache");
 const { Semaphore } = require("./shared/semaphore");
@@ -39,9 +39,16 @@ const SCOPE_KEY_INNER_SEPARATOR = "::";
 const SCOPE_KEY_OUTER_SEPARATOR = "##";
 const SCOPE_ROOT_KEY = "//";
 
+const CONFIG_SOURCE = Object.freeze({
+  RUNTIME: "RUNTIME",
+  FILE: "FILE",
+  AUTO: "AUTO",
+});
+
 const CONFIG_KEY = Object.freeze({
   TYPE: "TYPE",
   ACTIVE: "ACTIVE",
+  SOURCE: "SOURCE",
   APP_URL: "APP_URL",
   APP_URL_ACTIVE: "APP_URL_ACTIVE",
   VALIDATIONS: "VALIDATIONS",
@@ -52,6 +59,7 @@ const CONFIG_KEY = Object.freeze({
 const CONFIG_INFO_KEY = {
   [CONFIG_KEY.TYPE]: true,
   [CONFIG_KEY.ACTIVE]: true,
+  [CONFIG_KEY.SOURCE]: true,
   [CONFIG_KEY.APP_URL]: true,
   [CONFIG_KEY.APP_URL_ACTIVE]: true,
   [CONFIG_KEY.VALIDATIONS]: true,
@@ -103,7 +111,7 @@ class FeatureToggles {
 
   _processValidations(featureKey, validations, configFilepath) {
     const workingDir = process.cwd();
-    const configDir = configFilepath ? path.dirname(configFilepath) : __dirname;
+    const configDir = configFilepath ? path.dirname(configFilepath) : workingDir;
 
     const validationsScopesMap = {};
     const validationsRegex = [];
@@ -177,17 +185,41 @@ class FeatureToggles {
     }
   }
 
-  /**
-   * Populate this.__config.
-   */
-  _processConfig(config, configFilepath) {
+  _processConfigSource(source, configFromSource, configFilepath) {
+    let count = 0;
+    if (!isObject(configFromSource)) {
+      return count;
+    }
+
     const { uris: cfAppUris } = cfEnv.cfApp;
-    const configEntries = Object.entries(config);
-    for (const [featureKey, { type, active, appUrl, fallbackValue, validations }] of configEntries) {
+    const entries = Object.entries(configFromSource);
+    for (const [featureKey, value] of entries) {
+      if (this.__config[featureKey]) {
+        continue;
+      }
+      count++;
+
+      if (!isObject(value)) {
+        throw new VError(
+          {
+            name: VERROR_CLUSTER_NAME,
+            info: {
+              featureKey,
+              source,
+            },
+          },
+          "feature configuration is not an object"
+        );
+      }
+
+      const { type, active, appUrl, fallbackValue, validations } = value;
+
       this.__featureKeys.push(featureKey);
       this.__fallbackValues[featureKey] = fallbackValue;
       this.__config[featureKey] = {};
 
+      this.__config[featureKey][CONFIG_KEY.SOURCE] = source;
+
       if (type) {
         this.__config[featureKey][CONFIG_KEY.TYPE] = type;
       }
@@ -210,8 +242,23 @@ class FeatureToggles {
       }
     }
 
+    return count;
+  }
+
+  /**
+   * Populate this.__config.
+   */
+  _processConfig([configRuntime, configFromFile, configAuto], configFilepath) {
+    const configRuntimeCount = this._processConfigSource(CONFIG_SOURCE.RUNTIME, configRuntime, configFilepath);
+    const configFromFileCount = this._processConfigSource(CONFIG_SOURCE.FILE, configFromFile, configFilepath);
+    const configAutoCount = this._processConfigSource(CONFIG_SOURCE.AUTO, configAuto, configFilepath);
+
     this.__isConfigProcessed = true;
-    return configEntries.length;
+    return {
+      [CONFIG_SOURCE.RUNTIME]: configRuntimeCount,
+      [CONFIG_SOURCE.FILE]: configFromFileCount,
+      [CONFIG_SOURCE.AUTO]: configAutoCount,
+    };
   }
 
   _ensureInitialized() {
@@ -641,43 +688,45 @@ class FeatureToggles {
     );
   }
 
-  /**
-   * Initialize needs to run and finish before other APIs are called. It processes the configuration, sets up
-   * related internal state, and starts communication with redis.
-   */
-  async _initializeFeatures({ config: configInput, configFile: configFilepath = DEFAULT_CONFIG_FILEPATH } = {}) {
+  async _initializeFeatures({ config: configRuntime, configFile: configFilepath, configAuto } = {}) {
     if (this.__isInitialized) {
       return;
     }
 
-    let config;
+    let configFromFile;
     try {
-      config = configInput ? configInput : await FeatureToggles.readConfigFromFile(configFilepath);
+      if (!configFilepath && (await tryPathReadable(DEFAULT_CONFIG_FILEPATH))) {
+        configFilepath = DEFAULT_CONFIG_FILEPATH;
+      }
+      if (configFilepath) {
+        configFromFile = await FeatureToggles.readConfigFromFile(configFilepath);
+      }
     } catch (err) {
       throw new VError(
         {
           name: VERROR_CLUSTER_NAME,
           cause: err,
           info: {
             configFilepath,
-            ...(configInput && { configBaseInput: JSON.stringify(configInput) }),
-            ...(config && { configBase: JSON.stringify(config) }),
           },
         },
-        "initialization aborted, could not resolve configuration"
+        "initialization aborted, could not read config file"
       );
     }
 
-    let toggleCount;
+    let toggleCounts;
     try {
-      toggleCount = this._processConfig(config, configFilepath);
+      toggleCounts = this._processConfig([configRuntime, configFromFile, configAuto], configFilepath);
     } catch (err) {
       throw new VError(
         {
           name: VERROR_CLUSTER_NAME,
           cause: err,
           info: {
-            ...(config && { config: JSON.stringify(config) }),
+            ...(configAuto && { configAuto: JSON.stringify(configAuto) }),
+            ...(configFromFile && { configFromFile: JSON.stringify(configFromFile) }),
+            ...(configRuntime && { configRuntime: JSON.stringify(configRuntime) }),
+            configFilepath,
           },
         },
         "initialization aborted, could not process configuration"
@@ -751,16 +800,25 @@ class FeatureToggles {
       }
     }
 
+    const totalCount =
+      toggleCounts[CONFIG_SOURCE.RUNTIME] + toggleCounts[CONFIG_SOURCE.FILE] + toggleCounts[CONFIG_SOURCE.AUTO];
     logger.info(
-      "finished initialization with %i feature toggle%s with %s",
-      toggleCount,
-      toggleCount === 1 ? "" : "s",
+      "finished initialization with %i feature toggle%s (%i runtime, %i file, %i auto) with %s",
+      totalCount,
+      totalCount === 1 ? "" : "s",
+      toggleCounts[CONFIG_SOURCE.RUNTIME],
+      toggleCounts[CONFIG_SOURCE.FILE],
+      toggleCounts[CONFIG_SOURCE.AUTO],
       redisIntegrationMode
     );
     this.__isInitialized = true;
     return this;
   }
 
+  /**
+   * Initialize needs to run and finish before other APIs are called. It processes the configuration, sets up
+   * related internal state, and starts communication with redis.
+   */
   async initializeFeatures(options) {
     return await Semaphore.makeExclusiveReturning(this._initializeFeatures.bind(this))(options);
   }

src/shared/semaphore.js

@@ -78,12 +78,11 @@ class Semaphore {
     let isRunning;
     let runningPromise;
     return async (...args) => {
-      if (isRunning) {
-        return runningPromise;
-      }
-      isRunning = true;
       try {
-        runningPromise = cb(...args);
+        if (!isRunning) {
+          isRunning = true;
+          runningPromise = cb(...args);
+        }
         return await runningPromise;
       } finally {
         isRunning = false;