Worker Support (#22)

* proof of concept use paralleljs for inline workers

* use Paralleljs and reintroduce workers to SiteMap

* automatic babel polyfill support in worker libs

* document worker deatils in README; SiteMap feature tweak

* workerize parsing of time series data

* comments and commented code cleanup

Author: Frencil

README.md

@@ -101,21 +101,57 @@ If you have added or modified third-party dependencies then it is important to v
 
 Run `rm -rf node_modules && npm install` and re-run the app to validate a fresh install. This mimics how other apps importing `portal-core-components` will see your changes.
 
-### Worker Caveats
+### Using Workers in Components
 
-This library does support workers in its current build process using `worker-loader`. To create a worker
-name any worker file `*.worker.js`.
+This library supports parallel processing using web workers for components by using [Parallel.js](https://parallel.js.org/).
 
-Note, however, that a bug in `react-app-rewired` can mean lint errors in workers may silently break
-production builds but not development builds. See [here](https://github.com/timarney/react-app-rewired/issues/362) for details.
+To see how workers are currently in use in this library, see `src/lib_components/workers`. Example:
 
-If you have added or modified a worker file and are seeing empty production builds then manually look for
-and fix any lint errors using this command (from the root directory):
+```
+import Parallel from 'paralleljs';
+
+export default function myWorker(argument) {
+  const worker = new Parallel(argument);
+  return worker.spawn((inData) => {
+    /* do processing to generate outData */
+    return outData;
+  });
+}
+```
+
+A worker like this could then be imported elsewhere and used with promise-stype syntax. Example:
 
 ```
-npx eslint ./PATH_TO_YOUR_WORKER_FILE -c ./node_modules/eslint-config-react-app/index.js
+import myWorker from 'path/to/workers/myWorker.js';
+
+myWorker.then((result) => {
+  /* do stuff with result */
+});
 ```
 
+Critical rules for this worker pattern:
+
+* **Only put worker files in `src/lib_components/workers`.**
+* **Only define one worker function per worker file.**
+* **A worker file should only import Parallel and nothing else.**  
+  Any logic inside of `worker.spawn` will have no access to external definitions, even if defined in the worker file.
+  
+And most important: **Always test the lib export!**
+
+How a worker runs when developing core-components locally and how it runs when pulled in through the
+lib export in another app are *very* different. The former uses run-time webpack and the latter uses
+babel only, along with whatever bundling toolchain is in use by the other app.
+
+If you have developed a worker but find it is not working when pulled in as a lib export, run a clean
+lib build and then inspect the transpiled worker file in `lib/workers`. Look for any babel polyfill
+definitions that appear outside of `worker.spawn` but are used inside. This is the most common reason
+a worker fails to perform when pulled in through core components as an app dependency.
+
+The lib build for core components includes a step to migrate any babel polyfills (other than the one
+for `import` used to import Parallel.js) directly into the worker logic. This migration is not perfect.
+If it missed something that it should have caught please update `lif-fix-worker-babel.js` to suit.
+
+
 ## Modifying Existing Components
 
 Have nodejs.

config-overrides.js

@@ -16,19 +16,7 @@ const svgLoadRule = {
   ]
 };
 
-const workerLoadRule = {
-  test: /\.worker\.js$/,
-  use: {
-    loader: 'worker-loader',
-    options: {
-      filename: '[name].[contenthash].worker.js',
-      chunkFilename: '[id].[contenthash].worker.js',
-    },
-  },
-}
-
 module.exports = override(
   useEslintRc(path.resolve(__dirname, '.eslintrc')),
   addWebpackModuleRule(svgLoadRule),
-  addWebpackModuleRule(workerLoadRule),
 );

lib-fix-worker-babel.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+
+/**
+   Fix Worker Babel
+
+   The lib export runs everything through babel, even the workers. But the transpiled internal logic
+   of a given worker will fail if it uses polyfills injected by babel at the top of the worker file
+   (outside of worker.spawn(...)).
+
+   This script runs as part of the post-babel cleanup of building the lib export. It edits any
+   workers that have babel polyfills by moving the polyfill definition lines directly into the
+   worker's internal body. This allows us to build workers with ES* syntax and trust that they
+   will work as expected when transpiled into the lib export.
+*/
+
+console.log('Adjusting babel polyfills for workers...');
+
+const workers = fs.readdirSync(path.resolve(__dirname, 'lib/workers/'));
+workers.forEach((worker) => {
+  // Only look at .js files
+  if (!/\.js$/.test(worker)) { return; }
+  const uri = path.resolve(__dirname, 'lib/workers/', worker);
+  const inSource = fs.readFileSync(uri, 'utf8');
+  let workerEntered = false;
+  const preLines = [];
+  const polyfillLines = ['', '    // Babel polyfills for worker logic'];
+  const postLines = [''];
+  // Go through the file one line at a time
+  inSource.split('\n').forEach((line) => {
+    // Store babel polyfills OTHER THAN _interopRequireDefault
+    if (/^function _[a-zA-Z0-9]+\(.*\) \{.*\}$/.test(line) && !line.includes('interopRequireDefault')) {
+      polyfillLines.push(`    ${line}`);
+      return;
+    }
+    // Store lines before entering the worker and note when we do enter it
+    if (!workerEntered) {
+      preLines.push(line);
+      if (/^\s*return [_a-zA-Z0-9]+\.spawn\(function \(.*\) \{$/.test(line)) {
+        workerEntered = true;
+      }
+      return;
+    }
+    // Store lines after entering the worker
+    postLines.push(line);
+  });
+  // Final sanity checks
+  if (!workerEntered) {
+    console.log(`* ${worker} - SKIPPED - no worker entrypoint found (not good! go fix it!)`);
+    return;
+  }
+  if (polyfillLines.length === 2) {
+    console.log(`* ${worker} - SKIPPED - no polyfills in need of moving`);
+    return;    
+  }
+  // Write the updated file back out to lib
+  const outSource = [...preLines, ...polyfillLines, ...postLines].join('\n').replace(/[\n]{2,}/g, '\n\n');
+  fs.writeFileSync(uri, outSource);
+  console.log(`* ${worker} - UPDATED - ${polyfillLines.length - 2} polyfill(s) moved`);
+});

lib/components/SiteMap/FetchLocationUtils.d.ts

@@ -1,3 +1,3 @@
-export function parseLocationHierarchy(inHierarchy: any, parent?: any): {};
 export function fetchDomainHierarchy(domain: any): Promise<any>;
-export function fetchLocations(locations: any): Promise<any>;
+export function fetchSingleLocationREST(location: any): Promise<any>;
+export function fetchManyLocationsGraphQL(locations: any): Promise<any>;

lib/components/SiteMap/FetchLocationUtils.js

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.fetchLocations = exports.fetchDomainHierarchy = exports.parseLocationHierarchy = void 0;
+exports.fetchManyLocationsGraphQL = exports.fetchSingleLocationREST = exports.fetchDomainHierarchy = void 0;
 
 var _rxjs = require("rxjs");
 
@@ -13,43 +13,11 @@ var _NeonApi = _interopRequireDefault(require("../NeonApi/NeonApi"));
 
 var _NeonGraphQL = _interopRequireDefault(require("../NeonGraphQL/NeonGraphQL"));
 
-var _SiteMapWorkerSafeUtils = require("./SiteMapWorkerSafeUtils");
+var _parseDomainHierarchy = _interopRequireDefault(require("../../workers/parseDomainHierarchy"));
 
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-function _extends() { _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; }; return _extends.apply(this, arguments); }
-
-/**
- Recursive function to parse a deeply nest hierarchy object into a flat key/value object
- where keys are location names and values are objects containing only those location attributes
- the hierarchy affords us (type, description, and parent)
-*/
-var parseLocationHierarchy = function parseLocationHierarchy(inHierarchy) {
-  var parent = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : null;
-  var outHierarchy = {};
-  var name = inHierarchy.locationParentHierarchy ? null : inHierarchy.locationName;
-  var description = inHierarchy.locationDescription || null;
-  var type = inHierarchy.locationType || null;
-
-  if (description.includes('Not Used')) {
-    return outHierarchy;
-  }
-
-  if (name !== null) {
-    outHierarchy[name] = {
-      type: type,
-      description: description,
-      parent: parent
-    };
-  }
-
-  inHierarchy.locationChildHierarchy.forEach(function (subLocation) {
-    outHierarchy = _extends(_extends({}, outHierarchy), parseLocationHierarchy(subLocation, name));
-  });
-  return outHierarchy;
-};
+var _parseLocationsArray = _interopRequireDefault(require("../../workers/parseLocationsArray"));
 
-exports.parseLocationHierarchy = parseLocationHierarchy;
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 var domainIsValid = function domainIsValid(domainString) {
   if (typeof domainString !== 'string') {
@@ -74,22 +42,13 @@ var domainIsValid = function domainIsValid(domainString) {
 var fetchDomainHierarchy = function fetchDomainHierarchy(domain) {
   if (!domainIsValid) {
     return Promise.reject(new Error('Domain is not valid'));
-  } // Execute Locations REST API hierarchy fetch and pipe results to processing function
+  } // Execute Locations REST API hierarchy fetch
 
 
   return new Promise(function (resolve, reject) {
     _NeonApi.default.getSiteLocationHierarchyObservable(domain).pipe((0, _operators.map)(function (response) {
       if (response && response.data) {
-        var data = {};
-        response.data.locationChildHierarchy.forEach(function (child) {
-          // At the top level we only care about sites and don't want the HQ test site
-          if (child.locationType !== 'SITE' || child.locationName === 'HQTW') {
-            return;
-          }
-
-          data[child.locationName] = parseLocationHierarchy(child);
-        });
-        resolve(data);
+        resolve(response.data);
         return (0, _rxjs.of)(true);
       }
 
@@ -99,12 +58,42 @@ var fetchDomainHierarchy = function fetchDomainHierarchy(domain) {
       reject(new Error(error.message));
       return (0, _rxjs.of)(false);
     })).subscribe();
+  }).then(function (data) {
+    return (0, _parseDomainHierarchy.default)(data);
   });
 };
 
 exports.fetchDomainHierarchy = fetchDomainHierarchy;
 
-var fetchLocations = function fetchLocations(locations) {
+var fetchSingleLocationREST = function fetchSingleLocationREST(location) {
+  if (typeof location !== 'string' || !location.length) {
+    return Promise.reject(new Error('Location is not valid; must be non-empty string'));
+  } // Execute REST query and pipe results to processing function
+
+
+  return new Promise(function (resolve, reject) {
+    _NeonApi.default.getLocationObservable(location).pipe((0, _operators.map)(function (response) {
+      if (response && response.data) {
+        resolve([response.data]);
+        return (0, _rxjs.of)(true);
+      }
+
+      reject(new Error('Malformed response'));
+      return (0, _rxjs.of)(false);
+    }), (0, _operators.catchError)(function (error) {
+      reject(new Error(error.message));
+      return (0, _rxjs.of)(false);
+    })).subscribe();
+  }).then(function (data) {
+    return (0, _parseLocationsArray.default)(data);
+  }).then(function (locationMap) {
+    return (locationMap || {})[location];
+  });
+};
+
+exports.fetchSingleLocationREST = fetchSingleLocationREST;
+
+var fetchManyLocationsGraphQL = function fetchManyLocationsGraphQL(locations) {
   // Extract locations list and validate
   if (!Array.isArray(locations) || !locations.length || !locations.every(function (loc) {
     return typeof loc === 'string';
@@ -121,24 +110,16 @@ var fetchLocations = function fetchLocations(locations) {
         return (0, _rxjs.of)(false);
       }
 
-      var data = {};
-      result.response.data.locations.forEach(function (rawLocationData) {
-        var locationName = rawLocationData.locationName;
-
-        if (!locationName) {
-          return;
-        }
-
-        data[locationName] = (0, _SiteMapWorkerSafeUtils.parseLocationData)(rawLocationData);
-      });
-      resolve(data);
+      resolve(result.response.data.locations);
       return (0, _rxjs.of)(true);
     }), // Error
     (0, _operators.catchError)(function (error) {
       reject(new Error(error.message));
       return (0, _rxjs.of)(false);
     })).subscribe();
+  }).then(function (data) {
+    return (0, _parseLocationsArray.default)(data);
   });
 };
 
-exports.fetchLocations = fetchLocations;
+exports.fetchManyLocationsGraphQL = fetchManyLocationsGraphQL;