Merge pull request #929 from jason-fox/feature/842_ngsi_ld

fgalan · web-flow · commit f7f5a52ee5b2 · 2020-11-04T09:40:35.000+01:00
Fix merge conflicts again again again again.
diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
@@ -6,6 +6,13 @@ Add basic NGSI-LD support as experimental feature (#842)
     - Multi-measures
     - Lazy Attributes
     - Commands
+Add: enable use group commands in device and register it in iotagent-manager
+Add: extends commands definition to add mqtt options (qos, retain)
+Add: include findTypeSilently for groups to log some false errors as debug instead of alarm 
+Add: include `description` field in group schema
+Add: include from in log context (#918)
+Fix: Update internal attributes in Group update (#917)
 Fix: Static attributes from service not applied if device has static attributes (#757)
 Move Docker secret support inside the Node Application - remove Entrypoint (#885)
-Fix: IOTA_EXPLICIT_ATTRS env var was not working
+Fix: IOTA_EXPLICIT_ATTRS env var was not working
+Add lax validation mode using IOTA_RELAX_TEMPLATE_VALIDATION (#920)
diff --git a/doc/installationguide.md b/doc/installationguide.md
@@ -227,7 +227,7 @@ used for the same purpose. For instance:
 -   **singleConfigurationMode**: enables the Single Configuration mode for backwards compatibility (see description in
     the Overview). Default to false.
 -   **timestamp**: if this flag is activated:
-    -   For NGSIv1/NGSIv2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from device information. 
+    -   For NGSIv1/NGSIv2, the IoT Agent will add a `TimeInstant` metadata attribute to all the attributes updated from device information.
         This flag is overwritten by `timestamp` flag in group or device
     -   With NGSI-LD, the standard `observedAt` property-of-a-property is created instead.
 -   **defaultResource**: default string to use as resource for the registration of new Configurations (if no resource is
@@ -255,6 +255,11 @@ used for the same purpose. For instance:
     Note that for backwards compatibility with NGSI v2, the `fiware-servicepath` header is already used as alternative if the `NGSILD-Path` header is not supplied. Note that NGSILD-Path has not yet been included in the NGSI-LD standard (it has been proposed for the next update of the standard, but the final decision has yet been confirmed), take into account it could change
 -   **explicitAttrs**: if this flag is activated, only provisioned attributes will be processed to Context Broker.
     This flag is overwritten by `explicitAttrs` flag in group or device provision.
+-   **relaxTemplateValidation**: if this flag is activated, `objectId` attributes for incoming devices are not validated,
+    and may exceptionally include characters (such as semi-colons) which are
+    [forbidden](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) according to the NGSI
+    specification.  When provisioning devices, it is necessary that the developer provides valid  `objectId`-`name` mappings
+    whenever relaxed mode is used, to prevent the consumption of forbidden characters.
 
 ### Configuration using environment variables
 
@@ -264,7 +269,7 @@ with container-based technologies, like Docker, Heroku, etc...
 The following table shows the accepted environment variables, as well as the configuration parameter the variable
 overrides.
 
-| Environment variable      | Configuration attribute         |
+| Environment variable             | Configuration attribute         |
 | :------------------------------- | :------------------------------ |
 | IOTA_CB_URL                      | `contextBroker.url`             |
 | IOTA_CB_HOST                     | `contextBroker.host`            |
@@ -317,6 +322,8 @@ overrides.
 | IOTA_FALLBACK_PATH               | `fallbackPath`                  |
 | IOTA_DEFAULT_EXPRESSION_LANGUAGE | `defaultExpressionLanguage`     |
 | IOTA_EXPLICIT_ATTRS              | `explicitAttrs`                 |
+| IOTA_RELAX_TEMPLATE_VALIDATION   | `relaxTemplateValidation`       |
 
 Note:
 -   If you need to pass more than one JSON-LD context, you can define the IOTA_JSON_LD_CONTEXT environment variable as a comma separated list of contexts (e.g. `'http://context1.json-ld,http://context2.json-ld'`)
+
diff --git a/lib/commonConfig.js b/lib/commonConfig.js
@@ -164,6 +164,7 @@ function processEnvironmentVariables() {
             'IOTA_POLLING_DAEMON_FREQ',
             'IOTA_MULTI_CORE',
             'IOTA_DEFAULT_EXPRESSION_LANGUAGE',
+            'IOTA_RELAX_TEMPLATE_VALIDATION',
             'IOTA_JSON_LD_CONTEXT',
             'IOTA_FALLBACK_TENANT',
             'IOTA_FALLBACK_PATH'
@@ -474,6 +475,11 @@ function processEnvironmentVariables() {
         config.defaultExpressionLanguage =
             process.env.IOTA_DEFAULT_EXPRESSION_LANGUAGE;
     }
+    if (process.env.IOTA_RELAX_TEMPLATE_VALIDATION) {
+        config.relaxTemplateValidation = process.env.IOTA_RELAX_TEMPLATE_VALIDATION === 'true';
+    } else {
+        config.relaxTemplateValidation = config.relaxTemplateValidation === true;
+    }
 }
 
 function setConfig(newConfig) {
diff --git a/lib/constants.js b/lib/constants.js
@@ -62,6 +62,9 @@ module.exports = {
     NGSI_LD_PATH_HEADER: 'NGSILD-Path',
     //FIXME: check Keystone support this in lowercase, then change
     AUTH_HEADER: 'X-Auth-Token',
+    X_FORWARDED_FOR_HEADER: 'x-forwarded-for',
+    FORWARDED_HEADER: 'forwarded',
+    X_REAL_IP_HEADER: 'x-real-ip',
 
     COMMAND_RESULT_SUFIX: '_info',
     COMMAND_STATUS_SUFIX: '_status',
diff --git a/lib/fiware-iotagent-lib.js b/lib/fiware-iotagent-lib.js
@@ -269,6 +269,7 @@ function startServer(newConfig, iotAgent, callback) {
             corr: domainObj.corr,
             trans: domainObj.trans,
             op: domainObj.op,
+            from: domainObj.from,
             srv: domainObj.service,
             subsrv: domainObj.subservice,
             msg: domainObj.msg,
diff --git a/lib/model/Group.js b/lib/model/Group.js
@@ -32,6 +32,7 @@ var Group = new Schema({
     type: String,
     service: String,
     subservice: String,
+    description: String,
     trust: String,
     cbHost: String,
     timezone: String,
diff --git a/lib/services/common/domain.js b/lib/services/common/domain.js
@@ -47,6 +47,7 @@ function cleanDomain(domainToClean) {
     domainToClean.removeAllListeners('error');
     delete domainToClean.trans;
     delete domainToClean.corr;
+    delete domainToClean.from;
     delete domainToClean.op;
     delete domainToClean.path;
     domainToClean.exit();
@@ -80,6 +81,17 @@ function requestDomain(req, res, next) {
         reqDomain.subservice = req.headers[constants.SUBSERVICE_HEADER];
     }
 
+    // x-forwarded-for/forwarded overwrites x-real-ip
+    if (req.headers[constants.X_REAL_IP_HEADER]) {
+        reqDomain.from = req.headers[constants.X_REAL_IP_HEADER];
+    }
+    if (req.headers[constants.X_FORWARDED_FOR_HEADER]) {
+        reqDomain.from = req.headers[constants.X_FORWARDED_FOR_HEADER];
+    }
+    if (req.headers[constants.FORWARDED_HEADER]) {
+        reqDomain.from = req.headers[constants.FORWARDED_HEADER];
+    }
+
     function requestHandler() {
         var corr = req.get(CORRELATOR_HEADER);
 
diff --git a/lib/services/common/iotManagerService.js b/lib/services/common/iotManagerService.js
@@ -53,6 +53,8 @@ function register(callback) {
             service_path: service.subservice,
             attributes: service.attributes,
             static_attributes: service.staticAttributes,
+            commands: service.commands,
+            description: service.description,
             timezone: service.timezone,
             timestamp: service.timestamp,
             autoprovision: service.autoprovision,
diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js
@@ -122,10 +122,9 @@ function mergeArrays(original, newArray) {
  * @param {Object} configuration        Configuration data.
  */
 function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuration, callback) {
-    logger.debug(context, 'deviceData after merge with conf: %j', deviceData);
+    logger.debug(context, 'deviceData before merge with conf: %j', deviceData);
     for (let i = 0; i < fields.length; i++) {
         const confField = fields[i] === 'active' ? 'attributes' : fields[i];
-
         if (deviceData && deviceData[fields[i]] && ['active', 'lazy', 'commands'].indexOf(fields[i]) >= 0) {
             deviceData[fields[i]] = deviceData[fields[i]].map(setDefaultAttributeIds);
         } else if (deviceData && deviceData[fields[i]] && ['internalAttributes'].indexOf(fields[i]) >= 0) {
@@ -159,7 +158,7 @@ function mergeDeviceWithConfiguration(fields, defaults, deviceData, configuratio
     if (configuration && configuration.cbHost) {
         deviceData.cbHost = configuration.cbHost;
     }
-    logger.debug(context, 'deviceData before merge with conf: %j', deviceData);
+    logger.debug(context, 'deviceData after merge with conf: %j', deviceData);
     callback(null, deviceData);
 }
 
@@ -187,7 +186,13 @@ function findConfigurationGroup(deviceObj, callback) {
     } else {
         config
             .getGroupRegistry()
-            .findType(deviceObj.service, deviceObj.subservice, deviceObj.type, deviceObj.apikey, handlerGroupFind);
+            .findTypeSilently(
+                deviceObj.service, 
+                deviceObj.subservice,
+                deviceObj.type, 
+                deviceObj.apikey,
+                handlerGroupFind
+            );
     }
 }
 
@@ -258,35 +263,33 @@ function registerDevice(deviceObj, callback) {
 
         logger.debug(context, 'Registering device into NGSI Service:\n%s', JSON.stringify(deviceData, null, 4));
 
-        async.waterfall(
-            [
-                apply(registrationUtils.sendRegistrations, false, deviceData),
-                apply(registrationUtils.processContextRegistration, deviceData),
-                apply(createInitialEntity, deviceData)
-            ],
-            function(error, results) {
-                if (error) {
-                    callback(error);
-                } else {
-                    deviceObj.registrationId = results.registrationId;
-                    deviceObj.name = deviceData.name;
-                    deviceObj.service = deviceData.service;
-                    deviceObj.subservice = deviceData.subservice;
-                    deviceObj.type = deviceData.type;
-                    deviceObj.staticAttributes = deviceData.staticAttributes;
-                    if ('timestamp' in deviceData && deviceData.timestamp !== undefined) {
-                        deviceObj.timestamp = deviceData.timestamp;
-                    }
-                    if ('autoprovision' in deviceData && deviceData.autoprovision !== undefined) {
-                        deviceObj.autoprovision = deviceData.autoprovision;
-                    }
-                    if ('explicitAttrs' in deviceData && deviceData.explicitAttrs !== undefined) {
-                        deviceObj.explicitAttrs = deviceData.explicitAttrs;
-                    }
-                    config.getRegistry().store(deviceObj, callback);
+        async.waterfall([
+            apply(registrationUtils.sendRegistrations, false, deviceData),
+            apply(registrationUtils.processContextRegistration, deviceData),
+            apply(createInitialEntity, deviceData)
+        ], function(error, results) {
+            if (error) {
+                callback(error);
+            } else {
+                deviceObj.registrationId = results.registrationId;
+                deviceObj.name = deviceData.name;
+                deviceObj.service = deviceData.service;
+                deviceObj.subservice = deviceData.subservice;
+                deviceObj.type = deviceData.type;
+                deviceObj.staticAttributes = deviceData.staticAttributes;
+                deviceObj.commands = deviceData.commands;
+                if ('timestamp' in deviceData && deviceData.timestamp !== undefined) {
+                    deviceObj.timestamp = deviceData.timestamp;
                 }
+                if ('autoprovision' in deviceData && deviceData.autoprovision !== undefined) {
+                    deviceObj.autoprovision = deviceData.autoprovision;
+                }
+                if ('explicitAttrs' in deviceData && deviceData.explicitAttrs !== undefined) {
+                    deviceObj.explicitAttrs = deviceData.explicitAttrs;
+                }
+                config.getRegistry().store(deviceObj, callback);
             }
-        );
+        });
     }
 
     async.waterfall(
diff --git a/lib/services/groups/groupRegistryMemory.js b/lib/services/groups/groupRegistryMemory.js
@@ -277,6 +277,7 @@ exports.init = intoTrans(context, init);
 exports.find = intoTrans(context, find);
 exports.findBy = intoTrans(context, findBy);
 exports.findType = intoTrans(context, findBy(['service', 'subservice', 'type']));
+exports.findTypeSilently = intoTrans(context, findBy(['service', 'subservice', 'type']));
 exports.get = intoTrans(context, getSingleGroup);
 exports.getSilently = intoTrans(context, getSingleGroup);
 exports.getType = intoTrans(context, getSingleGroupType);
diff --git a/lib/services/groups/groupRegistryMongoDB.js b/lib/services/groups/groupRegistryMongoDB.js
@@ -64,6 +64,7 @@ function createGroup(group, callback) {
             'type',
             'service',
             'subservice',
+            'description',
             'trust',
             'cbHost',
             'timezone',
@@ -253,6 +254,7 @@ function update(id, body, callback) {
                 'type',
                 'service',
                 'subservice',
+                'description',
                 'trust',
                 'cbHost',
                 'timezone',
@@ -261,6 +263,7 @@ function update(id, body, callback) {
                 'lazy',
                 'attributes',
                 'staticAttributes',
+                'internalAttributes',
                 'explicitAttrs',
                 'expressionLanguage'
             ];
@@ -313,6 +316,7 @@ exports.init = alarmsInt(constants.MONGO_ALARM, intoTrans(context, init));
 exports.find = alarmsInt(constants.MONGO_ALARM, intoTrans(context, find));
 exports.findType = alarmsInt(constants.MONGO_ALARM,
                              intoTrans(context, findBy(['service', 'subservice', 'type', 'apikey'])));
+exports.findTypeSilently = intoTrans(context, findBy(['service', 'subservice', 'type', 'apikey']));
 exports.findBy = alarmsInt(constants.MONGO_ALARM, intoTrans(context, findBy));
 exports.get = alarmsInt(constants.MONGO_ALARM, intoTrans(context, findBy(['resource', 'apikey'])));
 exports.getSilently = intoTrans(context, findBy(['resource', 'apikey']));
diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js
@@ -36,8 +36,8 @@ var async = require('async'),
     apply = async.apply,
     provisioningHandler,
     removeDeviceHandler,
-    updateDeviceTemplate = require('../../templates/updateDevice.json'),
-    createDeviceTemplate = require('../../templates/createDevice.json'),
+    updateDeviceTemplate,
+    createDeviceTemplate,
     mandatoryHeaders = [
         'fiware-service',
         'fiware-servicepath'
@@ -65,6 +65,20 @@ var async = require('async'),
         explicitAttrs: 'explicitAttrs'
     };
 
+/**
+ * Load the templates for validating provisioning requests. The introduction of "Lax Mode" enables the addition of 
+ * characters which valid in the OPU-UA IoT Agent (e.g. semi-colons) but are usually forbidden by other IoT Agents
+ */
+function setConfiguration(config){
+    if (config.relaxTemplateValidation) {
+        updateDeviceTemplate = require('../../templates/updateDeviceLax.json');
+        createDeviceTemplate = require('../../templates/createDeviceLax.json');
+    } else {
+        updateDeviceTemplate = require('../../templates/updateDevice.json');
+        createDeviceTemplate = require('../../templates/createDevice.json');
+    }
+}
+
 /**
  * Express middleware to handle incoming device provisioning requests. Every request is validated and handled to the
  * NGSI Service for the registration.
@@ -163,7 +177,8 @@ function attributeToProvisioningAPIFormat(attribute) {
         expression: attribute.expression,
         reverse: attribute.reverse,
         entity_name: attribute.entity_name,
-        entity_type: attribute.entity_type
+        entity_type: attribute.entity_type,
+        mqtt: attribute.mqtt
     };
 }
 
@@ -249,7 +264,7 @@ function handleRemoveDevice(req, res, next) {
                 } else {
                     callback(new errors.DeviceNotFound(deviceId));
                 }
-            });        
+            });
     }
 
     function applyRemoveDeviceHandler(device, callback) {
@@ -365,6 +380,7 @@ function clear(callback) {
     callback();
 }
 
+exports.setConfiguration = setConfiguration;
 exports.loadContextRoutes = intoTrans(context, loadContextRoutes);
 exports.setProvisioningHandler = intoTrans(context, setProvisioningHandler);
 exports.setRemoveDeviceHandler = intoTrans(context, setRemoveDeviceHandler);
diff --git a/lib/services/northBound/northboundServer.js b/lib/services/northBound/northboundServer.js
@@ -42,6 +42,8 @@ function start(config, callback) {
     let baseRoot = '/';
     let iotaInformation;
 
+    deviceProvisioning.setConfiguration(config);
+
     northboundServer = {
         server: null,
         app: express(),
diff --git a/lib/templates/createDevice.json b/lib/templates/createDevice.json
@@ -168,6 +168,22 @@
                   "description": "Value of the command (currently ignored)",
                   "type": "string",
                   "pattern": "^([^<>();'=\"]+)*$"
+                },
+                "mqtt": {
+                  "description": "Mqtt properties",
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                      "retain": {
+                          "description": "MQTT Retain message",
+                          "type": "boolean"
+                      },
+                      "qos": {
+                          "description": "MQTT QoS level",
+                          "type": "string",
+                          "pattern": "^([0-2])$"
+                      }
+                  }
                 }
               }
             }
diff --git a/lib/templates/createDeviceLax.json b/lib/templates/createDeviceLax.json
diff --git a/lib/templates/updateDevice.json b/lib/templates/updateDevice.json
diff --git a/lib/templates/updateDeviceLax.json b/lib/templates/updateDeviceLax.json
diff --git a/test/unit/examples/iotamRequests/registrationWithGroups.json b/test/unit/examples/iotamRequests/registrationWithGroups.json
diff --git a/test/unit/examples/iotamRequests/registrationWithGroupsWithoutCB.json b/test/unit/examples/iotamRequests/registrationWithGroupsWithoutCB.json
diff --git a/test/unit/examples/iotamRequests/registrationWithStaticGroups.json b/test/unit/examples/iotamRequests/registrationWithStaticGroups.json
