Merge pull request #800 from orchestracities/expression-lib-alternative

Provide an alternative expression plugin

Author: fgalan

CHANGES_NEXT_RELEASE

@@ -3,3 +3,4 @@ Add NGSIv2 metadata support to device provisioned attributes
 Fix: Error message when sending measures with unknown/undefined attribute
 Add Null check within executeWithSecurity() to avoid crash (#829)
 Add NGSIv2 metadata support to attributeAlias plugin.
+Add support for JEXL as expression language (#801)

README.md

@@ -100,7 +100,7 @@ decommission devices.
 ## Testing
 
 
-Contribitions to development can be found [here](doc/development.md) - additional contributions are welcome.
+Contributions to development can be found [here](doc/development.md) - additional contributions are welcome.
 
 ### Agent Console
 

doc/expressionLanguage.md

@@ -16,8 +16,8 @@
 
 The IoTAgent Library provides an expression language for measurement transformation, that can be used to adapt the
 information coming from the South Bound APIs to the information reported to the Context Broker. Expressions in this
-language can be configured for provisioned attributes as explained in the Device Provisioning API section in the main
-README.md.
+language can be configured for provisioned attributes as explained in the Device Provisioning API section in the
+main README.md.
 
 ## Measurement transformation
 
@@ -301,3 +301,94 @@ two possible types of expressions: Integer (arithmetic operations) or Strings.
 -   update (of type "Boolean"): true -> ${@update * 20} -> ${ 1 \* 20 } -> $ { 20 } -> $ { "20"} -> False
 -   update (of type "Boolean"): false -> ${@update * 20} -> ${ 0 \* 20 } -> $ { 0 } -> $ { "0"} -> False
 -   update (of type "Boolean"): true -> ${trim(@updated)} -> ${trim("true")} -> $ { "true" } -> $ { "true"} -> True
+
+## JEXL Based Transformations
+
+As an alternative, the IoTAgent Library supports as well [JEXL](https://github.com/TomFrost/jexl).
+To use JEXL, you will need to either configure it as default
+language using the `defaultExpressionLanguage` field to `jexl`
+(see [configuration documentation](installationguide.md)) or configuring
+the usage of JEXL as expression language for a given device:
+
+```
+{
+   "devices":[
+      {
+         "device_id":"45",
+         "protocol":"GENERIC_PROTO",
+         "entity_name":"WasteContainer:WC45",
+         "entity_type":"WasteContainer",
+         "expressionLanguage": "jexl",
+         "attributes":[
+            {
+               "name":"location",
+               "type":"geo:point",
+               "expression": "..."
+            },
+            {
+               "name":"fillingLevel",
+               "type":"Number",
+               "expression": "..."
+            }
+         ]
+      }
+   ]
+}
+```
+
+In the following we provide examples of using JEXL to apply transformations.
+
+### Quick comparison to default language
+
+* JEXL supports the following types: Boolean, String, Number, Object, Array.
+* JEXL allows to navigate and [filter](https://github.com/TomFrost/jexl#collections) objects and arrays.
+* JEXL supports if..then...else... via [ternary operator](https://github.com/TomFrost/jexl#ternary-operator).
+* JEXL additionally supports the following operations:
+  Divide and floor `//`, Modulus `%`, Logical AND `&&` and 
+  Logical OR `||`. Negation operator is `!`
+* JEXL supports [comparisons](https://github.com/TomFrost/jexl#comparisons).
+
+
+For more details, check JEXL language details
+[here](https://github.com/TomFrost/jexl#all-the-details).
+
+### Examples of expressions
+
+The following table shows expressions and their expected outcomes taking into
+account the following measures at southbound interface:
+ 
+* `value` with value 6 (number)
+* `name` with value `"DevId629"` (string)
+* `object` with value `{name: "John", surname: "Doe"}` (JSON object)
+* `array` with value `[1, 3]` (JSON Array)
+
+
+| Expression                  | Expected outcome        |
+|:--------------------------- |:----------------------- |
+| `5 * value`                 | `30`                    |
+| `(6 + value) * 3`           | `36`                    |
+| `value / 12 + 1`            | `1.5`                   |
+| `(5 + 2) * (value + 7)`     | `91`                    |
+| `value * 5.2`               | `31.2`                  |
+| `"Pruebas " + "De Strings"` | `"Pruebas De Strings"`  |
+| `name + "value is " +value` | `"DevId629 value is 6"` |
+
+Support for `trim`, `length`, `substr` and `indexOf` transformations was added.
+
+| Expression                  | Expected outcome        |
+|:--------------------------- |:----------------------- |
+| <code>" a "&vert;trim</code>  | `a`                     |
+| <code>name&vert;length</code> | `8`                     |
+| <code>name&vert;indexOf("e")</code>| `1`                     |
+| <code>name&vert;substring(0,name&vert;indexOf("e")+1)</code>| `"De"`       |
+
+The following are some expressions not supported by the legacy expression
+language:
+
+| Expression                          | Expected outcome        |
+|:----------------------------------- |:----------------------- |
+| `value == 6? true : false`          | `true`                  |
+| <code>value == 6 && name&vert;indexOf("e")>0</code> | `true`                  |
+| `array[1]+1`                        | `3`                     |
+| `object.name`                       | `"John"`                |
+| `{type:"Point",coordinates: [value,value]}`| `{type:"Point",coordinates: [6,6]}` |

doc/installationguide.md

@@ -232,6 +232,10 @@ used for the same purpose. For instance:
     the IoTAgent runs in a single thread. For more details about multi-core functionality, please refer to the
     [Cluster](https://nodejs.org/api/cluster.html) module in Node.js and
     [this section](howto.md#iot-agent-in-multi-thread-mode) of the library documentation.
+-   **defaultExpressionLanguage**: the default expression language used to
+    compute expressions, possible values are: `legacy` or `jexl`. When not set or 
+    wrongly set, `legacy` is used as default value.
+
 
 ### Configuration using environment variables
 
@@ -289,3 +293,4 @@ overrides.
 | IOTA_POLLING_DAEMON_FREQ  | `pollingDaemonFrequency`        |
 | IOTA_AUTOCAST             | `autocast`                      |
 | IOTA_MULTI_CORE           | `multiCore`                     |
+| IOTA_DEFAULT_EXPRESSION_LANGUAGE | defaultExpressionLanguage    |

lib/commonConfig.js

@@ -91,7 +91,8 @@ function processEnvironmentVariables() {
              'IOTA_APPEND_MODE',
              'IOTA_POLLING_EXPIRATION',
              'IOTA_POLLING_DAEMON_FREQ',
-             'IOTA_MULTI_CORE'
+             'IOTA_MULTI_CORE',
+             'IOTA_DEFAULT_EXPRESSION_LANGUAGE'
         ],
         iotamVariables = [
             'IOTA_IOTAM_URL',
@@ -355,6 +356,11 @@ function processEnvironmentVariables() {
     } else {
         config.multiCore = config.multiCore === true;
     }
+
+    if (process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE) {
+        config.defaultExpressionLanguage = process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE;
+    }
+
 }
 
 function setConfig(newConfig) {

lib/fiware-iotagent-lib.js

@@ -84,6 +84,16 @@ function doActivate(newConfig, callback) {
         commandRegistry,
         securityService;
 
+    logger.format = logger.formatters.pipe;
+
+    config.setConfig(newConfig); //moving up here because otherwise env variable are not considered by the code below
+
+    if (!config.getConfig().dieOnUnexpectedError) {
+        process.on('uncaughtException', globalErrorHandler);
+    }
+
+    newConfig = config.getConfig();
+
     if (newConfig.contextBroker) {
         if (! newConfig.contextBroker.url && newConfig.contextBroker.host && newConfig.contextBroker.port) {
             newConfig.contextBroker.url = 'http://' + newConfig.contextBroker.host + ':' + newConfig.contextBroker.port;
@@ -113,7 +123,16 @@ function doActivate(newConfig, callback) {
         }
     }
 
-    config.setConfig(newConfig);
+    if (newConfig.defaultExpressionLanguage &&
+        (newConfig.defaultExpressionLanguage === 'legacy' ||
+          newConfig.defaultExpressionLanguage ==='jexl')){
+      logger.info(context, 'Using ' + newConfig.defaultExpressionLanguage + ' as default expression language');
+    } else {
+      logger.info(context, 'Default expression language not set, or invalid, using legacy configuration');
+      newConfig.defaultExpressionLanguage = 'legacy';
+    }
+
+    config.setConfig(newConfig); //after chaging some configuration, we re apply the configuration
 
     logger.info(context, 'Activating IOT Agent NGSI Library.');
 
@@ -156,10 +175,6 @@ function doActivate(newConfig, callback) {
         ], callback);
     };
 
-    if (!config.getConfig().dieOnUnexpectedError) {
-        process.on('uncaughtException', globalErrorHandler);
-    }
-
     config.setSecurityService(securityService);
     config.setRegistry(registry);
     config.setGroupRegistry(groupRegistry);

lib/model/Device.js

@@ -48,7 +48,8 @@ var Device = new Schema({
     internalId: String,
     creationDate: { type: Date, default: Date.now },
     internalAttributes: Object,
-    autoprovision: Boolean
+    autoprovision: Boolean,
+    expressionLanguage: String
 });
 
 function load(db) {

lib/model/Group.js

@@ -41,7 +41,8 @@ var Group = new Schema({
     lazy: Array,
     attributes: Array,
     internalAttributes: Array,
-    autoprovision: Boolean
+    autoprovision: Boolean,
+    expressionLanguage: String
 });
 
 function load(db) {

lib/plugins/expressionPlugin.js

@@ -22,12 +22,14 @@
  * please contact with::daniel.moranjimenez@telefonica.com
  *
  * Modified by: Daniel Calvo - ATOS Research & Innovation
+ * Modified by: Federico M. Facca - Martel Innovate
  */
 
 'use strict';
 
 var _ = require('underscore'),
-    parser = require('./expressionParser'),
+    legacyParser = require('./expressionParser'),
+    jexlParser = require('./jexlParser'),
     config = require('../commonConfig'),
      /*jshint unused:false*/
     logger = require('logops'),
@@ -62,7 +64,27 @@ function mergeAttributes(attrList1, attrList2) {
 
 
 function update(entity, typeInformation, callback) {
+
+    function checkJexl(typeInformation){
+        if (config.getConfig().defaultExpressionLanguage === 'jexl' &&
+            typeInformation.expressionLanguage &&
+            typeInformation.expressionLanguage !== 'legacy') {
+          return true;
+        } else if (config.getConfig().defaultExpressionLanguage === 'jexl' &&
+            !typeInformation.expressionLanguage) {
+          return true;
+        } else if (config.getConfig().defaultExpressionLanguage === 'legacy' &&
+            typeInformation.expressionLanguage && typeInformation.expressionLanguage === 'jexl') {
+          return true;
+        }
+        return false;
+    }
+
     function processEntityUpdateNgsi1(entity) {
+        var parser = legacyParser;
+        if (checkJexl(typeInformation)){
+          parser = jexlParser;
+        }
         var expressionAttributes = [],
             ctx = parser.extractContext(entity.attributes);
 
@@ -76,6 +98,10 @@ function update(entity, typeInformation, callback) {
     }
 
     function processEntityUpdateNgsi2(attributes) {
+        var parser = legacyParser;
+        if (checkJexl(typeInformation)){
+          parser = jexlParser;
+        }
         var expressionAttributes = [],
             ctx = parser.extractContext(attributes);