Merge pull request #89 from boschrexroth/bugfix/rawdata

fix: support for raw data

Author: krauskopf

README.md

@@ -121,8 +121,8 @@ Any use of the source code and related documents of this repository in applicati
                      - fix: add more tests to handle node address with invalid symbols
 * 2024-12-06: 1.9.7  - fix: update dependencies in package-lock                     
                      - fix: further improve resilience on invalid data from server
+* 2025-01-27: 1.9.8  - fix: no support for raw binary data (types/datalayer/raw). E.g. for some Ethercat nodes, returned now as a buffer for further processing.
 ```
-  
 ## About
 
 Copyright © 2020-2024 Bosch Rexroth AG. All rights reserved.

lib/CtrlxDatalayerSubscription.js

@@ -176,7 +176,7 @@ class CtrlxDatalayerSubscription extends EventEmitter {
     // All subscription settings are transmitted as payload
     const settings = {
       'properties': properties,
-			'nodes': this._nodes,
+      'nodes': this._nodes,
     };
 
     let options = {
@@ -189,7 +189,7 @@ class CtrlxDatalayerSubscription extends EventEmitter {
         'Accept': 'application/json',
         'Authorization': this._authorization,
         'Connection': 'keep-alive',
-        },
+      },
       rejectUnauthorized: false   // accept self-signed certificates
     };
 
@@ -268,9 +268,9 @@ class CtrlxDatalayerSubscription extends EventEmitter {
    *
    * @memberof CtrlxCore
    */
-    get isEndByServer() {
-      return this._isEndByServer;
-    }
+  get isEndByServer() {
+    return this._isEndByServer;
+  }
 
   /**
    * Opens an event stream and starts the subscription.
@@ -320,8 +320,8 @@ class CtrlxDatalayerSubscription extends EventEmitter {
           }
         },
         agent: new https.Agent({ keepAlive: false })  // create a dedicated agent to have dedicated connection instance. Also disable the agent-keep-alive explicitly.
-                                                      // This is necessary because since node.js 19 the default behaviour was changed.
-                                                      // https://nodejs.org/en/blog/announcements/v19-release-announce#https11-keepalive-by-default
+        // This is necessary because since node.js 19 the default behaviour was changed.
+        // https://nodejs.org/en/blog/announcements/v19-release-announce#https11-keepalive-by-default
       };
 
       if (this._keepaliveIntervalMs) {
@@ -432,14 +432,13 @@ class CtrlxDatalayerSubscription extends EventEmitter {
             let payload = CtrlxDatalayer._parseData(e.data);
             if (!this.emit('update', payload, e.lastEventId)) {
               // Listener seems not yet to be attached. Retry on next tick.
-              setTimeout(()=>this.emit('update', payload, e.lastEventId), 0);
+              setTimeout(() => this.emit('update', payload, e.lastEventId), 0);
             }
-          } catch (err) {
+          } catch(err) {
             if (this.listeners('error').length > 0) {
               this.emit('error', new Error(`Error parsing update event: ${err.message}`));
             }
           }
-
         });
 
         this._es.addEventListener('keepalive', (e) => {
@@ -451,9 +450,9 @@ class CtrlxDatalayerSubscription extends EventEmitter {
             let payload = CtrlxDatalayer._parseData(e.data);
             if (!this.emit('keepalive', payload, e.lastEventId)) {
               // Listener seems not yet to be attached. Retry on next tick.
-              setTimeout(()=>this.emit('keepalive', payload, e.lastEventId), 0);
+              setTimeout(() => this.emit('keepalive', payload, e.lastEventId), 0);
             }
-          } catch (err) {
+          } catch(err) {
             if (this.listeners('error').length > 0) {
               this.emit('error', new Error(`Error parsing keepalive event: ${err.message}`));
             }

lib/CtrlxDatalayerV2.js

@@ -60,18 +60,20 @@ class CtrlxDatalayer {
    * Convert a Data Layer response body from string to a javascript object.
    *
    * This is basicly an extension to JSON.parse(), that supports also 64bit data types, which
-   * will be converted to BigInt. Which is not supported by the standard JSON.parse().
-   * With standard JSON.parse() a 64bit integer value might get rounded, because javascripts Number
+   * will be converted to BigInt, which is not supported by the standard JSON.parse().
+   * With standard JSON.parse() a 64bit integer value might get rounded, because JavaScripts Number
    * is based on a double precision floating point number.
-   * E.g. values greater than Number.MAX_SAFE_INTEGER.
+   * e.g. values greater than Number.MAX_SAFE_INTEGER.
    *
    * @static
-   * @param {string} data - The content data as returend by a Data Layer request.
+   * @param {string} data - The content data as returned by a Data Layer request.
    * @returns {object} - The parsed javascript object.
    * @memberof CtrlxDatalayer
    * @throws {SyntaxError} On invalid JSON objects.
    */
   static _parseData(data) {
+
+    // We expect JSON.parse to fail here for invalid JSON
     let payload = JSON.parse(data);
 
     if (payload.type === "int64" || payload.type === "uint64") {
@@ -81,7 +83,6 @@ class CtrlxDatalayer {
       let strBigArray = data.match(new RegExp(/(?:"value":)(?:.*\[)(.*?)(?:\])/))[1];
       payload.value = Array.from(strBigArray.split(","), (s) => BigInt(s))
     }
-
     return payload;
   }
 
@@ -301,27 +302,36 @@ class CtrlxDatalayer {
     }
 
     const req = https.request(options, (res) => {
-      let data = "";
 
-      res.setEncoding('utf8');
+      // We have to read binary to support types/datalayer/raw
+      let chunks = [];
       res.on('data', function(chunk) {
-        data += chunk;
+        chunks.push(Buffer.from(chunk));
       });
 
       res.on('end', function() {
+        const buffer = Buffer.concat(chunks);
 
         // We expect 200 on success
         if (res.statusCode !== 200) {
-          callback(CtrlxProblemError.fromHttpResponse(res, data));
+          callback(CtrlxProblemError.fromHttpResponse(res, buffer.toString('utf8')));
           return;
         }
 
-        // Try to parse the data
+        // If we don't receive JSON, we return the 'raw' binary buffer as-it-is.
         let payload;
-        try {
-          payload = CtrlxDatalayer._parseData(data);
-        } catch (err) {
-          callback(err, null);
+        if (res.headers['content-type'].includes('application/json')){
+          try {
+            payload = CtrlxDatalayer._parseData(buffer.toString('utf8'));
+          } catch (err) {
+            callback(err, null);
+            return;
+          }
+        } else {
+          payload = {
+            type: 'raw',
+            value: new Uint8Array(buffer),
+          }
         }
 
         // No error, return payload data.
@@ -438,4 +448,3 @@ class CtrlxDatalayer {
 }
 
 module.exports = CtrlxDatalayer;
-

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "node-red-contrib-ctrlx-automation",
-  "version": "1.9.7",
+  "version": "1.9.8",
   "description": "Node-RED nodes for ctrlX AUTOMATION",
   "repository": {
     "type": "git",
@@ -62,4 +62,4 @@
     "test_with_coverage": "nyc mocha --timeout 60000",
     "benchmark": "node ./test/helper/benchmark"
   }
-}
+}

package.json

@@ -312,6 +312,21 @@ describe('CtrlxCoreDataLayerNodes', function() {
 
     });
 
+    it('should return a raw buffer', function(done) {
+
+      let ctrlx = new CtrlxCore(getHostname(), getUsername(), getPassword());
+
+      ctrlx.logIn()
+        .then(() => { return ctrlx.datalayerRead('encoding/raw/buffer'); })
+        .then((data) => {
+          assert.deepEqual(data.value, new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05]));
+          done();
+        })
+        .catch((err) => done(err))
+        .finally(() => ctrlx.logOut());
+
+    });
+
   });
 
 

test/CtrlxCore.nodes.test.js

@@ -110,6 +110,14 @@ describe('CtrlxCore', function() {
       done();
     });
 
+    it('should throw exception on invalid JSON', function(done) {
+
+      const invalidJSON = 'test/invalid/json';
+      expect(() => CtrlxDatalayer._parseData(invalidJSON)).to.throw(SyntaxError);
+
+      done();
+    });
+
     it('should parse BigInt', function(done) {
 
       expect(CtrlxDatalayer._parseData(`{"type": "int64", "value": 9223372036854775807}`).value).to.equal(BigInt(9223372036854775807n))

test/CtrlxCore.test.js

@@ -272,7 +272,8 @@ describe('ctrlx-datalayer-subscribe', function () {
       let flow = [
         { "id": "h1", "type": "helper" },
         { "id": "n1", "type": "ctrlx-datalayer-subscribe", "subscription": "s1", "path": "test/options", "name": "subscribe", "wires": [["h1"]] },
-        { "id": "s1", "type": "ctrlx-config-subscription", "device": "c1",
+        {
+          "id": "s1", "type": "ctrlx-config-subscription", "device": "c1",
           "name": "sub1",
           "publishIntervalMs": "100",
           "publishIntervalUnits": "milliseconds",
@@ -468,11 +469,11 @@ describe('ctrlx-datalayer-subscribe', function () {
       let path = 'with/strange/symbols/abc=1;nichts-ist.wahr:("alles[ist]erlaubt")42/x.y.z';
 
       let flow = [
-        { "id": "f1", "type": "tab", "label": "Test flow"},
-        { "id": "h1", "z":"f1", "type": "helper" },
-        { "id": "n1", "z":"f1", "type": "ctrlx-datalayer-subscribe", "subscription": "s1", "path": path, "name": "subscribe", "wires": [["h1"]] },
-        { "id": "s1", "z":"f1", "type": "ctrlx-config-subscription", "device": "c1", "name": "sub1", "publishIntervalMs": "1000" },
-        { "id": "c1", "z":"f1", "type": "ctrlx-config", "name": "ctrlx", "hostname": getHostname(), "debug": true },
+        { "id": "f1", "type": "tab", "label": "Test flow" },
+        { "id": "h1", "z": "f1", "type": "helper" },
+        { "id": "n1", "z": "f1", "type": "ctrlx-datalayer-subscribe", "subscription": "s1", "path": path, "name": "subscribe", "wires": [["h1"]] },
+        { "id": "s1", "z": "f1", "type": "ctrlx-config-subscription", "device": "c1", "name": "sub1", "publishIntervalMs": "1000" },
+        { "id": "c1", "z": "f1", "type": "ctrlx-config", "name": "ctrlx", "hostname": getHostname(), "debug": true },
       ];
       let credentials = {
         c1: {
@@ -517,11 +518,11 @@ describe('ctrlx-datalayer-subscribe', function () {
     it('should handle invalid send json messages', function (done) {
 
       let flow = [
-        { "id": "f1", "type": "tab", "label": "Test flow"},
-        { "id": "h1", "z":"f1", "type": "helper" },
-        { "id": "n1", "z":"f1", "type": "ctrlx-datalayer-subscribe", "subscription": "s1", "path": "test/invalid/json", "name": "subscribe", "wires": [["h1"]] },
-        { "id": "s1", "z":"f1", "type": "ctrlx-config-subscription", "device": "c1", "name": "sub1", "publishIntervalMs": "1000" },
-        { "id": "c1", "z":"f1", "type": "ctrlx-config", "name": "ctrlx", "hostname": getHostname(), "debug": true },
+        { "id": "f1", "type": "tab", "label": "Test flow" },
+        { "id": "h1", "z": "f1", "type": "helper" },
+        { "id": "n1", "z": "f1", "type": "ctrlx-datalayer-subscribe", "subscription": "s1", "path": "test/invalid/json", "name": "subscribe", "wires": [["h1"]] },
+        { "id": "s1", "z": "f1", "type": "ctrlx-config-subscription", "device": "c1", "name": "sub1", "publishIntervalMs": "1000" },
+        { "id": "c1", "z": "f1", "type": "ctrlx-config", "name": "ctrlx", "hostname": getHostname(), "debug": true },
       ];
       let credentials = {
         c1: {

test/ctrlx-datalayer-subscribe.test.js

@@ -121,6 +121,7 @@ class CtrlxMockupV2 {
         type: 'bool'
       });
     });
+
     this.app.get('/automation/api/v2/nodes/framework/metrics/system/cpu-utilisation-percent', authenticateJWT, (req, res) => {
       switch (req.query.type) {
         case undefined:
@@ -166,6 +167,7 @@ class CtrlxMockupV2 {
           break;
       }
     });
+
     this.app.get('/automation/api/v2/nodes/framework/metrics/system', authenticateJWT, (req, res) => {
       if (req.query.type === 'browse') {
         res.statusCode = 200;
@@ -193,6 +195,7 @@ class CtrlxMockupV2 {
         type: 'int16'
       });
     });
+
     this.app.get('/automation/api/v2/nodes/plc/app/Application/sym/PLC_PRG/i', authenticateJWT, (req, res) => {
       res.statusCode = 200;
       res.json({
@@ -210,10 +213,13 @@ class CtrlxMockupV2 {
       }
       this.var_i64 = req.body.value;
       res.statusCode = 200;
+      res.setHeader('content-type', 'application/json');
       res.send(`{"type": "int64", "value":${this.var_i64.toString()}}`);
     });
+
     this.app.get('/automation/api/v2/nodes/plc/app/Application/sym/PLC_PRG/i64', authenticateJWT, (req, res) => {
       res.statusCode = 200;
+      res.setHeader('content-type', 'application/json');
       res.send(`{"type": "int64", "value":${this.var_i64.toString()}}`);
     });
 
@@ -231,6 +237,7 @@ class CtrlxMockupV2 {
         type: 'string'
       });
     });
+
     this.app.get('/automation/api/v2/nodes/plc/app/Application/sym/PLC_PRG/str', authenticateJWT, (req, res) => {
       res.statusCode = 200;
       res.json({
@@ -282,6 +289,12 @@ class CtrlxMockupV2 {
       });
     });
 
+    this.app.get('/automation/api/v2/nodes/encoding/raw/buffer', authenticateJWT, (req, res) => {
+      res.statusCode = 200;
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(Buffer.from([0x01, 0x02, 0x03, 0x04, 0x05]));
+    });
+
 
     //
     // Builtin Data Mockups - Create/Delete