Merge branch 'master' into feature/842_ngsi_ld

Author: jason-fox

CHANGES_NEXT_RELEASE

@@ -6,10 +6,13 @@ Add basic NGSI-LD support as experimental feature (#842)
     - Multi-measures
     - Lazy Attributes
     - Commands
+Add: enable use group commands in device and register it in iotagent-manager
+Add: extends commands definition to add mqtt options (qos, retain)
 Add: include findTypeSilently for groups to log some false errors as debug instead of alarm 
 Add: include `description` field in group schema
 Add: include from in log context (#918)
 Fix: Update internal attributes in Group update (#917)
 Fix: Static attributes from service not applied if device has static attributes (#757)
 Move Docker secret support inside the Node Application - remove Entrypoint (#885)
-Fix: IOTA_EXPLICIT_ATTRS env var was not working
+Fix: IOTA_EXPLICIT_ATTRS env var was not working
+Add lax validation mode using IOTA_RELAX_TEMPLATE_VALIDATION (#920)

doc/installationguide.md

@@ -227,7 +227,7 @@ used for the same purpose. For instance:
 -   **singleConfigurationMode**: enables the Single Configuration mode for backwards compatibility (see description in
     the Overview). Default to false.
 -   **timestamp**: if this flag is activated:
-    -   For NGSIv1/NGSIv2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from device information. 
+    -   For NGSIv1/NGSIv2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from device information.
         This flag is overwritten by `timestamp` flag in group or device
     -   With NGSI-LD, the standard `observedAt` property-of-a-property is created instead.
 -   **defaultResource**: default string to use as resource for the registration of new Configurations (if no resource is
@@ -255,6 +255,11 @@ used for the same purpose. For instance:
     Note that for backwards compatibility with NGSI v2, the `fiware-servicepath` header is already used as alternative if the `NGSILD-Path` header is not supplied. Note that NGSILD-Path has not yet been included in the NGSI-LD standard (it has been proposed for the next update of the standard, but the final decision has yet been confirmed), take into account it could change
 -   **explicitAttrs**: if this flag is activated, only provisioned attributes will be processed to Context Broker.
     This flag is overwritten by `explicitAttrs` flag in group or device provision.
+-   **relaxTemplateValidation**: if this flag is activated, `objectId` attributes for incoming devices are not validated,
+    and may exceptionally include characters (such as semi-colons) which are
+    [forbidden](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) according to the NGSI
+    specification.  When provisioning devices, it is necessary that the developer provides valid  `objectId`-`name` mappings
+    whenever relaxed mode is used, to prevent the consumption of forbidden characters.
 
 ### Configuration using environment variables
 
@@ -264,7 +269,7 @@ with container-based technologies, like Docker, Heroku, etc...
 The following table shows the accepted environment variables, as well as the configuration parameter the variable
 overrides.
 
-| Environment variable      | Configuration attribute         |
+| Environment variable             | Configuration attribute         |
 | :------------------------------- | :------------------------------ |
 | IOTA_CB_URL                      | `contextBroker.url`             |
 | IOTA_CB_HOST                     | `contextBroker.host`            |
@@ -317,6 +322,8 @@ overrides.
 | IOTA_FALLBACK_PATH               | `fallbackPath`                  |
 | IOTA_DEFAULT_EXPRESSION_LANGUAGE | `defaultExpressionLanguage`     |
 | IOTA_EXPLICIT_ATTRS              | `explicitAttrs`                 |
+| IOTA_RELAX_TEMPLATE_VALIDATION   | `relaxTemplateValidation`       |
 
 Note:
 -   If you need to pass more than one JSON-LD context, you can define the IOTA_JSON_LD_CONTEXT environment variable as a comma separated list of contexts (e.g. `'http://context1.json-ld,http://context2.json-ld'`)
+

lib/commonConfig.js

@@ -164,6 +164,7 @@ function processEnvironmentVariables() {
             'IOTA_POLLING_DAEMON_FREQ',
             'IOTA_MULTI_CORE',
             'IOTA_DEFAULT_EXPRESSION_LANGUAGE',
+            'IOTA_RELAX_TEMPLATE_VALIDATION',
             'IOTA_JSON_LD_CONTEXT',
             'IOTA_FALLBACK_TENANT',
             'IOTA_FALLBACK_PATH'
@@ -474,6 +475,11 @@ function processEnvironmentVariables() {
         config.defaultExpressionLanguage =
             process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE;
     }
+    if (process.env.IOTA_RELAX_TEMPLATE_VALIDATION) {
+        config.relaxTemplateValidation = process.env.IOTA_RELAX_TEMPLATE_VALIDATION === 'true';
+    } else {
+        config.relaxTemplateValidation = config.relaxTemplateValidation === true;
+    }
 }
 
 function setConfig(newConfig) {

lib/services/common/iotManagerService.js

@@ -53,6 +53,7 @@ function register(callback) {
             service_path: service.subservice,
             attributes: service.attributes,
             static_attributes: service.staticAttributes,
+            commands: service.commands,
             description: service.description,
             timezone: service.timezone,
             timestamp: service.timestamp,

lib/services/devices/deviceService.js

@@ -122,10 +122,9 @@ function mergeArrays(original, newArray) {
  * @param {Object} configuration        Configuration data.
  */
 function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuration, callback) {
-    logger.debug(context, 'deviceData after merge with conf: %j', deviceData);
+    logger.debug(context, 'deviceData before merge with conf: %j', deviceData);
     for (let i = 0; i < fields.length; i++) {
         const confField = fields[i] === 'active' ? 'attributes' : fields[i];
-
         if (deviceData && deviceData[fields[i]] && ['active', 'lazy', 'commands'].indexOf(fields[i]) >= 0) {
             deviceData[fields[i]] = deviceData[fields[i]].map(setDefaultAttributeIds);
         } else if (deviceData && deviceData[fields[i]] && ['internalAttributes'].indexOf(fields[i]) >= 0) {
@@ -159,7 +158,7 @@ function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuratio
     if (configuration && configuration.cbHost) {
         deviceData.cbHost = configuration.cbHost;
     }
-    logger.debug(context, 'deviceData before merge with conf: %j', deviceData);
+    logger.debug(context, 'deviceData after merge with conf: %j', deviceData);
     callback(null, deviceData);
 }
 
@@ -187,7 +186,13 @@ function findConfigurationGroup(deviceObj, callback) {
     } else {
         config
             .getGroupRegistry()
-            .findTypeSilently(deviceObj.service, deviceObj.subservice, deviceObj.type, deviceObj.apikey, handlerGroupFind);
+            .findTypeSilently(
+                deviceObj.service, 
+                deviceObj.subservice,
+                deviceObj.type, 
+                deviceObj.apikey,
+                handlerGroupFind
+            );
     }
 }
 
@@ -258,35 +263,33 @@ function registerDevice(deviceObj, callback) {
 
         logger.debug(context, 'Registering device into NGSI Service:\n%s', JSON.stringify(deviceData, null, 4));
 
-        async.waterfall(
-            [
-                apply(registrationUtils.sendRegistrations, false, deviceData),
-                apply(registrationUtils.processContextRegistration, deviceData),
-                apply(createInitialEntity, deviceData)
-            ],
-            function(error, results) {
-                if (error) {
-                    callback(error);
-                } else {
-                    deviceObj.registrationId = results.registrationId;
-                    deviceObj.name = deviceData.name;
-                    deviceObj.service = deviceData.service;
-                    deviceObj.subservice = deviceData.subservice;
-                    deviceObj.type = deviceData.type;
-                    deviceObj.staticAttributes = deviceData.staticAttributes;
-                    if ('timestamp' in deviceData && deviceData.timestamp !== undefined) {
-                        deviceObj.timestamp = deviceData.timestamp;
-                    }
-                    if ('autoprovision' in deviceData && deviceData.autoprovision !== undefined) {
-                        deviceObj.autoprovision = deviceData.autoprovision;
-                    }
-                    if ('explicitAttrs' in deviceData && deviceData.explicitAttrs !== undefined) {
-                        deviceObj.explicitAttrs = deviceData.explicitAttrs;
-                    }
-                    config.getRegistry().store(deviceObj, callback);
+        async.waterfall([
+            apply(registrationUtils.sendRegistrations, false, deviceData),
+            apply(registrationUtils.processContextRegistration, deviceData),
+            apply(createInitialEntity, deviceData)
+        ], function(error, results) {
+            if (error) {
+                callback(error);
+            } else {
+                deviceObj.registrationId = results.registrationId;
+                deviceObj.name = deviceData.name;
+                deviceObj.service = deviceData.service;
+                deviceObj.subservice = deviceData.subservice;
+                deviceObj.type = deviceData.type;
+                deviceObj.staticAttributes = deviceData.staticAttributes;
+                deviceObj.commands = deviceData.commands;
+                if ('timestamp' in deviceData && deviceData.timestamp !== undefined) {
+                    deviceObj.timestamp = deviceData.timestamp;
                 }
+                if ('autoprovision' in deviceData && deviceData.autoprovision !== undefined) {
+                    deviceObj.autoprovision = deviceData.autoprovision;
+                }
+                if ('explicitAttrs' in deviceData && deviceData.explicitAttrs !== undefined) {
+                    deviceObj.explicitAttrs = deviceData.explicitAttrs;
+                }
+                config.getRegistry().store(deviceObj, callback);
             }
-        );
+        });
     }
 
     async.waterfall(

lib/services/northBound/deviceProvisioningServer.js

@@ -36,8 +36,8 @@ var async = require('async'),
     apply = async.apply,
     provisioningHandler,
     removeDeviceHandler,
-    updateDeviceTemplate = require('../../templates/updateDevice.json'),
-    createDeviceTemplate = require('../../templates/createDevice.json'),
+    updateDeviceTemplate,
+    createDeviceTemplate,
     mandatoryHeaders = [
         'fiware-service',
         'fiware-servicepath'
@@ -65,6 +65,20 @@ var async = require('async'),
         explicitAttrs: 'explicitAttrs'
     };
 
+/**
+ * Load the templates for validating provisioning requests. The introduction of "Lax Mode" enables the addition of 
+ * characters which valid in the OPU-UA IoT Agent (e.g. semi-colons) but are usually forbidden by other IoT Agents
+ */
+function setConfiguration(config){
+    if (config.relaxTemplateValidation) {
+        updateDeviceTemplate = require('../../templates/updateDeviceLax.json');
+        createDeviceTemplate = require('../../templates/createDeviceLax.json');
+    } else {
+        updateDeviceTemplate = require('../../templates/updateDevice.json');
+        createDeviceTemplate = require('../../templates/createDevice.json');
+    }
+}
+
 /**
  * Express middleware to handle incoming device provisioning requests. Every request is validated and handled to the
  * NGSI Service for the registration.
@@ -163,7 +177,8 @@ function attributeToProvisioningAPIFormat(attribute) {
         expression: attribute.expression,
         reverse: attribute.reverse,
         entity_name: attribute.entity_name,
-        entity_type: attribute.entity_type
+        entity_type: attribute.entity_type,
+        mqtt: attribute.mqtt
     };
 }
 
@@ -249,7 +264,7 @@ function handleRemoveDevice(req, res, next) {
                 } else {
                     callback(new errors.DeviceNotFound(deviceId));
                 }
-            });        
+            });
     }
 
     function applyRemoveDeviceHandler(device, callback) {
@@ -365,6 +380,7 @@ function clear(callback) {
     callback();
 }
 
+exports.setConfiguration = setConfiguration;
 exports.loadContextRoutes = intoTrans(context, loadContextRoutes);
 exports.setProvisioningHandler = intoTrans(context, setProvisioningHandler);
 exports.setRemoveDeviceHandler = intoTrans(context, setRemoveDeviceHandler);

lib/services/northBound/northboundServer.js

@@ -42,6 +42,8 @@ function start(config, callback) {
     let baseRoot = '/';
     let iotaInformation;
 
+    deviceProvisioning.setConfiguration(config);
+
     northboundServer = {
         server: null,
         app: express(),

lib/templates/createDevice.json

@@ -168,6 +168,22 @@
                   "description": "Value of the command (currently ignored)",
                   "type": "string",
                   "pattern": "^([^<>();'=\"]+)*$"
+                },
+                "mqtt": {
+                  "description": "Mqtt properties",
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                      "retain": {
+                          "description": "MQTT Retain message",
+                          "type": "boolean"
+                      },
+                      "qos": {
+                          "description": "MQTT QoS level",
+                          "type": "string",
+                          "pattern": "^([0-2])$"
+                      }
+                  }
                 }
               }
             }