Add sandbox queue management when using buildSandboxPerVisit

kratiahuja · kratiahuja · commit c3e4f941e90d · 2020-03-24T13:08:22.000-07:00
In order to increase the efficiency of the system when using `buildSandboxPerVisit` flag, we currently lazily create the sandbox for the next request. This is a single sandbox that is leveraged. In order to help with performance, we will be maintaining a queue of these sandboxes that can be leveraged by the requests. When the sandbox is empty, it will eagerly create the sandbox instead of caching the pending request and letting it run when the sandbox is available. This is primarily so that we don't end up starving the system when number of requests is far greater than queue size and the system cannot recover from storing the sandbox for next requests. Finishing the request (even if in degraded fashion) brings in better resilience.

In addition, when requests cannot use the sandbox from the queue, we now emit analytics information about the request in the result object. This helps applications using this flag to track if their QPS is greater than Queue size and bump the queue size.

The queue size is provided when the app is created and pre-emptively the queue is filled up so initial requests don't starve.

Updated README as well with this new public API.
diff --git a/README.md b/README.md
@@ -36,6 +36,10 @@ let app = new FastBoot({
       // additional global properties to define within the sandbox
     });
   },
+
+  // optional number to be provided when using `buildSandboxPerVisit` which defines the queue size for sandboxes.
+  // This number should represent your QPS of your service
+  maxSandboxQueueSize: <Number> // defaults to 1 if not provided
 });
 
 app.visit('/photos', options)
@@ -68,7 +72,7 @@ configuration:
 - `shouldRender`: boolean to indicate whether the app should do rendering or not. If set to false, it puts the app in routing-only. Defaults to true.
 - `disableShoebox`: boolean to indicate whether we should send the API data in the shoebox. If set to false, it will not send the API data used for rendering the app on server side in the index.html. Defaults to false.
 - `destroyAppInstanceInMs`: whether to destroy the instance in the given number of ms. This is a failure mechanism to not wedge the Node process
-- `buildSandboxPerVisit`: whether to create a new sandbox context per-visit (slows down each visit, but guarantees no prototype leakages can occur), or reuse the existing sandbox (faster per-request, but each request shares the same set of prototypes). Defaults to false.
+- `buildSandboxPerVisit`: whether to create a new sandbox context per-visit (slows down each visit, but guarantees no prototype leakages can occur), or reuse the existing sandbox (faster per-request, but each request shares the same set of prototypes). Defaults to false. When using this flag, also set `maxSandboxQueue` to represent the QPS of your application so that sandboxes can be queued for next requests. When not provided, it defaults to storing only one sandbox
 
 ### Build Your App
 
@@ -107,6 +111,43 @@ rendering your Ember.js application using the [FastBoot App Server](https://gith
 Run `fastboot` with the `DEBUG` environment variable set to `fastboot:*`
 for detailed logging.
 
+### Result
+
+The result from `fastboot` is a `Result` object that has the following API:
+
+```
+
+type DOMContents = () => {
+  /**
+    The `<head>` contents generated by the visit.
+  */
+  head: string;
+
+  /**
+    The `<body>` contents generated by the visit.
+  */
+  body: string;
+}
+
+interface FastBootVisitResult {
+  /**
+    The serialized DOM contents after completing the `visit` request.
+
+    Note: this combines the `domContents.head` and `domContents.body`.
+  */
+  html(): string;
+
+  domContents(): DOMContents
+
+  analytics: {
+    /**
+     * Boolean to know if the request used a prebuilt sandbox
+     */
+    usedPrebuiltSandbox: <Boolean>
+  }
+}
+```
+
 ### The Shoebox
 
 You can pass application state from the FastBoot rendered application to
diff --git a/src/ember-app.js b/src/ember-app.js
@@ -14,6 +14,7 @@ const FastBootInfo = require('./fastboot-info');
 const Result = require('./result');
 const FastBootSchemaVersions = require('./fastboot-schema-versions');
 const getPackageName = require('./utils/get-package-name');
+const Queue = require('./utils/queue');
 
 /**
  * @private
@@ -29,6 +30,7 @@ class EmberApp {
    * @param {Object} options
    * @param {string} options.distPath - path to the built Ember application
    * @param {Function} [options.buildSandboxGlobals] - the function used to build the final set of global properties accesible within the sandbox
+   * @param {Number} [options.maxSandboxQueueSize] - maximum sandbox queue size when using buildSandboxPerRequest flag.
    */
   constructor(options) {
     this.buildSandboxGlobals = options.buildSandboxGlobals || defaultBuildSandboxGlobals;
@@ -64,9 +66,30 @@ class EmberApp {
     );
     this.scripts = buildScripts(filePaths);
 
+    // default to 1 if maxSandboxQueueSize is not defined so the sandbox is pre-warmed when process comes up
+    const maxSandboxQueueSize = options.maxSandboxQueueSize || 1;
     // Ensure that the dist files can be evaluated and the `Ember.Application`
     // instance created.
-    this.buildApp();
+    this.buildSandboxQueue(maxSandboxQueueSize);
+  }
+
+  /**
+   * @private
+   *
+   * Function to build queue of sandboxes which is later leveraged if application is using `buildSandboxPerRequest`
+   * flag. This is an optimization to help with performance.
+   *
+   * @param {Number} maxSandboxQueueSize - maximum size of queue (this is should be a derivative of your QPS)
+   */
+  buildSandboxQueue(maxSandboxQueueSize) {
+    this._sandboxApplicationInstanceQueue = new Queue(
+      () => this.buildNewApplicationInstance(),
+      maxSandboxQueueSize
+    );
+
+    for (let i = 0; i < maxSandboxQueueSize; i++) {
+      this._sandboxApplicationInstanceQueue.enqueue();
+    }
   }
 
   /**
@@ -237,27 +260,35 @@ class EmberApp {
     return app;
   }
 
+  /**
+   * @private
+   *
+   * @param {Promise<instance>} appInstance - the instance that is pre-warmed or built on demand
+   * @param {Boolean} isAppInstancePreBuilt - boolean representing how the instance was built
+   *
+   * @returns {Object}
+   */
+  getAppInstanceInfo(appInstance, isAppInstancePreBuilt = true) {
+    return { app: appInstance, isSandboxPreBuilt: isAppInstancePreBuilt };
+  }
+
   /**
    * @private
    *
    * Get the new sandbox off if it is being created, otherwise create a new one on demand.
    * The later is needed when the current request hasn't finished or wasn't build with sandbox
    * per request turned on and a new request comes in.
    *
+   * @param {Boolean} buildSandboxPerVisit if true, a new sandbox will
+   *                                       **always** be created, otherwise one
+   *                                       is created for the first request
+   *                                       only
    */
-  async _getNewApplicationInstance() {
-    let app;
-
-    if (this._pendingNewApplicationInstance) {
-      let pendingAppInstancePromise = this._pendingNewApplicationInstance;
-      this._pendingNewApplicationInstance = undefined;
-      app = await pendingAppInstancePromise;
-    } else {
-      // if there is no current pending application instance, create a new one on-demand.
-      app = await this.buildApp();
-    }
+  async getNewApplicationInstance() {
+    const queueObject = this._sandboxApplicationInstanceQueue.dequeue();
+    const app = await queueObject.item;
 
-    return app;
+    return this.getAppInstanceInfo(app, queueObject.isItemPreBuilt);
   }
 
   /**
@@ -285,13 +316,19 @@ class EmberApp {
   async _visit(path, fastbootInfo, bootOptions, result, buildSandboxPerVisit) {
     let shouldBuildApp = buildSandboxPerVisit || this._applicationInstance === undefined;
 
-    let app = shouldBuildApp ? await this._getNewApplicationInstance() : this._applicationInstance;
+    let { app, isSandboxPreBuilt } = shouldBuildApp
+      ? await this.getNewApplicationInstance()
+      : this.getAppInstanceInfo(this._applicationInstance);
 
     if (buildSandboxPerVisit) {
       // entangle the specific application instance to the result, so it can be
       // destroyed when result._destroy() is called (after the visit is
       // completed)
       result.applicationInstance = app;
+
+      // we add analytics information about the current request to know
+      // whether it used sandbox from the pre-built queue or built on demand.
+      result.analytics.usedPrebuiltSandbox = isSandboxPreBuilt;
     } else {
       // save the created application instance so that we can clean it up when
       // this instance of `src/ember-app.js` is destroyed (e.g. reload)
@@ -387,7 +424,7 @@ class EmberApp {
       if (buildSandboxPerVisit) {
         // if sandbox was built for this visit, then build a new sandbox for the next incoming request
         // which is invoked using buildSandboxPerVisit
-        this._pendingNewApplicationInstance = this.buildNewApplicationInstance();
+        this._sandboxApplicationInstanceQueue.enqueue();
       }
     }
 
diff --git a/src/index.js b/src/index.js
@@ -41,9 +41,10 @@ class FastBoot {
    * @param {string} options.distPath the path to the built Ember application
    * @param {Boolean} [options.resilient=false] if true, errors during rendering won't reject the `visit()` promise but instead resolve to a {@link Result}
    * @param {Function} [options.buildSandboxGlobals] a function used to build the final set of global properties setup within the sandbox
+   * @param {Number} [options.maxSandboxQueueSize] - maximum sandbox queue size when using buildSandboxPerRequest flag.
    */
   constructor(options = {}) {
-    let { distPath, buildSandboxGlobals } = options;
+    let { distPath, buildSandboxGlobals, maxSandboxQueueSize } = options;
 
     this.resilient = 'resilient' in options ? Boolean(options.resilient) : false;
 
@@ -58,8 +59,9 @@ class FastBoot {
     }
 
     this.buildSandboxGlobals = buildSandboxGlobals;
+    this.maxSandboxQueueSize = maxSandboxQueueSize;
 
-    this._buildEmberApp(this.distPath, this.buildSandboxGlobals);
+    this._buildEmberApp(this.distPath, this.buildSandboxGlobals, maxSandboxQueueSize);
   }
 
   /**
@@ -106,7 +108,11 @@ class FastBoot {
     this._buildEmberApp(distPath);
   }
 
-  _buildEmberApp(distPath = this.distPath, buildSandboxGlobals = this.buildSandboxGlobals) {
+  _buildEmberApp(
+    distPath = this.distPath,
+    buildSandboxGlobals = this.buildSandboxGlobals,
+    maxSandboxQueueSize = this.maxSandboxQueueSize
+  ) {
     if (!distPath) {
       throw new Error(
         'You must instantiate FastBoot with a distPath ' +
@@ -124,6 +130,7 @@ class FastBoot {
     this._app = new EmberApp({
       distPath,
       buildSandboxGlobals,
+      maxSandboxQueueSize,
     });
   }
 }
diff --git a/src/result.js b/src/result.js
@@ -20,6 +20,7 @@ class Result {
     this._fastbootInfo = fastbootInfo;
     this.applicationInstance = undefined;
     this.applicationInstanceInstance = undefined;
+    this.analytics = {};
   }
 
   /**
diff --git a/src/utils/queue.js b/src/utils/queue.js
@@ -0,0 +1,59 @@
+'use strict';
+
+const debug = require('debug')('fastboot:ember-app');
+
+/**
+ * Utility Queue class to store queue of sandboxes that can be leveraged when using `buildSandboxPerVisit`.
+ *
+ * @public
+ */
+class Queue {
+  constructor(builderFn, maxSize = 1) {
+    this.items = [];
+    this.maxSize = maxSize;
+    this.builderFn = builderFn;
+  }
+
+  _buildItem() {
+    return this.builderFn();
+  }
+
+  _addToQueue() {
+    this.items.push(this._buildItem());
+  }
+
+  enqueue() {
+    // when the queue is not full, we add the item into the queue, otherwise ignore adding
+    // to the queue.
+    if (!this.isFull()) {
+      this._addToQueue();
+    } else {
+      debug('Ignoring adding appInstance to queue as Queue is already full!');
+    }
+  }
+
+  dequeue() {
+    if (this.isEmpty()) {
+      // build on demand if the queue does not have a pre-warmed version to avoid starving
+      // the system
+      return { item: this._buildItem(), isItemPreBuilt: false };
+    } else {
+      // return a pre-warmed version
+      return { item: this.items.shift(), isItemPreBuilt: true };
+    }
+  }
+
+  isEmpty() {
+    return this.size() === 0;
+  }
+
+  size() {
+    return this.items.length;
+  }
+
+  isFull() {
+    return this.size() === this.maxSize;
+  }
+}
+
+module.exports = Queue;
diff --git a/test/fastboot-test.js b/test/fastboot-test.js
diff --git a/test/queue-test.js b/test/queue-test.js

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ class Result {`
`20`	`20`	`this._fastbootInfo = fastbootInfo;`
`21`	`21`	`this.applicationInstance = undefined;`
`22`	`22`	`this.applicationInstanceInstance = undefined;`
	`23`	`+ this.analytics = {};`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`/**`