Skip to content

Commit dbad6e5

Browse files
committed
Moved schema resolution out of the transform node
Now same logic than for the validator node, i.e. expecting the resolution node in front
1 parent dbfb38e commit dbad6e5

8 files changed

+232
-261
lines changed

README.md

Lines changed: 168 additions & 174 deletions
Large diffs are not rendered by default.

json-multi-schema-resolver.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ module.exports = RED => {
2121
const jsonCache = require('./json-cache.js')(node);
2222

2323
/**
24-
* Find the URL to the JSON Schema to use for the given payload.
24+
* Find the URL to the JSON Schema or JSONata expression to use for the given payload.
2525
*/
2626
async function resolveAsync(payload) {
2727
const mappings = await jsonCache.loadAsync(mappingsUrl);

json-multi-schema-transformer.html

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
color: '#00B4FF',
55
defaults: {
66
name: { value: '', },
7-
transformsUrl: { value: '', },
87
},
98
inputs: 1,
109
outputs: 1,
@@ -20,18 +19,12 @@
2019
<label for="node-input-name"><i class="icon-tag"></i> Name</label>
2120
<input type="text" id="node-input-name" placeholder="Name" />
2221
</div>
23-
24-
<div class="form-row">
25-
<label for="node-input-transformsUrl"><i class="icon-tag"></i> Transforms URL</label>
26-
<input type="text" id="node-input-transformsUrl" placeholder="transformsUrl" />
27-
</div>
2822
</script>
2923

3024
<script type="text/x-red" data-help-name="json-multi-schema-transformer">
3125
<p>Node to transform a JSON observation to another format using a configurable set of JSONata rules.</p>
3226
<ul>
33-
<li>The node will transform a given JSON payload received in <code>msg.payload</code> using some JSONata rules.</li>
34-
<li>To find the proper rules, the node uses the configuration file which URL is given as a <code>transformsUrl</code> property.</li>
27+
<li>The node will transform a given JSON payload received in <code>msg.payload</code> using some JSONata rules, which URL is given in <code>msg.schemaUrl</code></li>
3528
<li>The URL of the corresponding JSONata file is returned in <code>msg.transformUrl</code>.</li>
3629
<li>The transformed payload is returned in <code>msg.payload</code>.</li>
3730
<li>Errors are returned on <code>msg.error</code></li>

json-multi-schema-transformer.js

Lines changed: 42 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
/* jshint esversion:8, node:true, strict:true */
22
/**
33
* Node-RED node transforming a JSON observation from whichever format to another format using a specified JSONata URL.
4-
* Schemas are automatically downloaded and cached the first time they are needed.
4+
* Schemas are automatically downloaded and cached on disk the first time they are needed.
55
* JSONata expressions are cached in memory.
66
*/
77

8+
//JSONata: A declarative open-source query and transformation language for JSON data.
89
const jsonata = require('jsonata');
910
const util = require('util');
1011

@@ -14,105 +15,80 @@ module.exports = RED => {
1415
function JsonMultiSchemaTransformerNode(config) {
1516
RED.nodes.createNode(this, config);
1617
const node = this;
17-
const transformsUrl = config.transformsUrl;
1818

1919
let lastStatusError = true;
2020
node.status({ fill:'grey', shape:'ring', text:'Uninitialized', });
2121

2222
const jsonCache = require('./json-cache.js')(node);
2323

24-
/**
25-
* Find the URL to the JSONata expression to use for the given payload.
26-
*/
27-
async function resolveAsync(payload) {
28-
const transforms = await jsonCache.loadAsync(transformsUrl);
29-
let transformUrl = '';
30-
for (const mapping of transforms) {
31-
if (mapping.query && mapping.cases) {
32-
const expression = jsonata(mapping.query);
33-
let match = expression.evaluate(payload);
34-
if (match) {
35-
if (match === true) {
36-
//Special case for boolean
37-
match = "true";
38-
}
39-
const result = mapping.cases[match];
40-
if (result) {
41-
transformUrl = result;
42-
break;
43-
}
44-
}
45-
}
46-
}
47-
return transformUrl;
48-
}
49-
5024
//Cache of JSONata expressions
5125
const jsonatas = {};
5226

5327
/**
5428
* Transform the given payload with the JSONata expression given in URL.
5529
*/
5630
async function transformAsync(payload, transformUrl) {
57-
if (transformUrl) {
58-
let jsonataExpression;
59-
let jsonataCache = jsonatas[transformUrl];
60-
if (jsonataCache) {
61-
if (jsonataCache.expression === null) {
62-
//Wait for another task to be done building the same JSONata, so that we can use its cache
63-
await new Promise((resolve, reject) => jsonataCache.mutexQueue.push(resolve));
64-
}
65-
jsonataExpression = jsonataCache.expression;
66-
} else {
67-
//Build JSONata expression for the given transformation URL
68-
jsonataCache = { expression: null, mutexQueue: [] };
69-
jsonatas[transformUrl] = jsonataCache;
70-
const transform = await jsonCache.loadAsync(transformUrl, false);
71-
node.debug('Build JSONata expression for: ' + transformUrl);
72-
jsonataExpression = jsonata(transform);
73-
jsonataCache.expression = jsonataExpression;
31+
if (!transformUrl) {
32+
return 'Error: Invalid JSONata URL';
33+
}
7434

75-
//Resume tasks waiting for the same JSONata expression
76-
let next;
77-
while ((next = jsonataCache.mutexQueue.shift()) != undefined) {
78-
next(); //Resolve promise
79-
}
35+
let jsonataExpression;
36+
let jsonataCache = jsonatas[transformUrl];
37+
if (jsonataCache) {
38+
if (jsonataCache.expression === null) {
39+
//Wait for another task to be done building the same JSONata, so that we can use its cache
40+
await new Promise((resolve, reject) => jsonataCache.mutexQueue.push(resolve));
8041
}
42+
jsonataExpression = jsonataCache.expression;
43+
} else {
44+
//Build JSONata expression for the given transformation URL
45+
jsonataCache = { expression: null, mutexQueue: [] };
46+
jsonatas[transformUrl] = jsonataCache;
47+
const transform = await jsonCache.loadAsync(transformUrl, false);
48+
node.debug('Build JSONata expression for: ' + transformUrl);
49+
jsonataExpression = jsonata(transform);
50+
jsonataCache.expression = jsonataExpression;
8151

82-
if (jsonataExpression) {
83-
//Perform transformation
84-
return jsonataExpression.evaluate(payload);
52+
//Resume tasks waiting for the same JSONata expression
53+
let next;
54+
while ((next = jsonataCache.mutexQueue.shift()) != undefined) {
55+
next(); //Resolve promise
8556
}
8657
}
58+
59+
if (jsonataExpression) {
60+
//Perform transformation
61+
return jsonataExpression.evaluate(payload);
62+
}
63+
8764
return false;
8865
}
8966

9067
node.on('input', async msg => {
91-
delete msg.transformUrl;
9268
msg.error = msg.error ? msg.error + ' ; ' : '';
93-
try {
94-
const transformUrl = await resolveAsync(msg.payload);
95-
if (transformUrl != '') {
96-
msg.transformUrl = transformUrl;
97-
const result = await transformAsync(msg.payload, msg.transformUrl);
69+
if (msg.schemaUrl == '') {
70+
msg.error += 'Unknown schema!';
71+
} else {
72+
msg.transformUrl = msg.schemaUrl;
73+
try {
74+
const result = await transformAsync(msg.payload, msg.schemaUrl);
9875
if (result) {
9976
msg.payload = result;
10077
msg.error = msg.error != '';
10178
} else {
102-
lastStatusError = true;
103-
node.status({ fill:'red', shape:'ring', text:'Error', });
104-
msg.error += util.format('Failed tranforming using "%s"', transformsUrl);
79+
msg.error += util.format('Failed tranforming using "%s"', msg.schemaUrl);
10580
}
10681
if (lastStatusError) {
10782
node.status({ fill:'green', shape:'dot', text:'OK', });
10883
lastStatusError = false;
10984
}
85+
} catch (ex) {
86+
lastStatusError = true;
87+
node.status({ fill:'red', shape:'ring', text:'Error', });
88+
msg.error += util.format('Error tranforming using "%s": %s', msg.schemaUrl, ex);
11089
}
111-
} catch (ex) {
112-
lastStatusError = true;
113-
node.status({ fill:'red', shape:'ring', text:'Error', });
114-
msg.error += util.format('Error tranforming using "%s": %s : %s', transformsUrl, ex, JSON.stringify(msg.payload));
11590
}
91+
delete msg.schemaUrl;
11692
node.send(msg);
11793
});
11894
}

json-multi-schema-validator.html

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
<script type="text/x-red" data-help-name="json-multi-schema-validator">
2525
<ul>
2626
<li>The node will validate the JSON data given in <code>msg.payload</code> against the JSON Schema URL given in <code>msg.schemaUrl</code></li>
27+
<li>The URL of the corresponding JSON Schema file is returned in <code>msg.validUrl</code>.</li>
28+
<li>The content of <code>msg.payload</code> is unchanged.</li>
2729
<li>Errors are returned on <code>msg.error</code></li>
2830
</ul>
2931
</script>

json-multi-schema-validator.js

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
/* jshint esversion:8, node:true, strict:true */
22
/**
3-
* Node-RED node that can validate a JSON payload against a specified JSON Schema URL.
4-
* JSON Schemas are automatically downloaded and cached the first time they are needed.
3+
* Node-RED node that validates a JSON payload against a specified JSON Schema URL.
4+
* JSON Schemas are automatically downloaded and cached on disk the first time they are needed.
5+
* JSON Schema validators are cached in memory.
56
*/
67

8+
//Ajv: Another JSON Schema Validator
9+
const Ajv = require('ajv');
710
const util = require('util');
811

912
module.exports = RED => {
@@ -18,8 +21,6 @@ module.exports = RED => {
1821

1922
const jsonCache = require('./json-cache.js')(node);
2023

21-
//Ajv: Another JSON Schema Validator
22-
const Ajv = require('ajv');
2324
const ajv = Ajv({
2425
allErrors: true, //TODO: Make a parameter
2526
loadSchema: jsonCache.loadAsync,
@@ -29,9 +30,12 @@ module.exports = RED => {
2930
//Cache of validators for different schemas
3031
const validators = {};
3132

32-
async function validateAsync(schemaUrl, payload) {
33+
/**
34+
* Validate the given payload with the JSON Schema given in URL.
35+
*/
36+
async function validateAsync(payload, schemaUrl) {
3337
if (!schemaUrl) {
34-
return 'Error: Invalid schema URL';
38+
return 'Error: Invalid JSON schema URL';
3539
}
3640

3741
let validatorCache = validators[schemaUrl];
@@ -62,7 +66,7 @@ module.exports = RED => {
6266
const validator = await ajv.compileAsync({"$ref":schemaUrl});
6367
if (validator) {
6468
validatorCache.validator = validator;
65-
task = validateAsync(schemaUrl, payload);
69+
task = validateAsync(payload, schemaUrl);
6670
} else {
6771
validatorCache.validator = false;
6872
node.error('Unknown error compiling schema: ' + schemaUrl);
@@ -89,8 +93,9 @@ module.exports = RED => {
8993
if (msg.schemaUrl == '') {
9094
msg.error += 'Unknown schema!';
9195
} else {
96+
msg.validUrl = msg.schemaUrl;
9297
try {
93-
const result = await validateAsync(msg.schemaUrl, msg.payload);
98+
const result = await validateAsync(msg.payload, msg.schemaUrl);
9499
if (result === true) {
95100
msg.error = msg.error != '';
96101
} else {
@@ -103,9 +108,10 @@ module.exports = RED => {
103108
} catch (ex) {
104109
lastStatusError = true;
105110
node.status({ fill:'red', shape:'ring', text:'Error', });
106-
msg.error += util.format('Failed validatation against "%s": %s', msg.schemaUrl, ex);
111+
msg.error += util.format('Error validatating using "%s": %s', msg.schemaUrl, ex);
107112
}
108113
}
114+
delete msg.schemaUrl;
109115
node.send(msg);
110116
});
111117
}

package-lock.json

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "node-red-contrib-json-multi-schema",
3-
"version": "0.4.1",
3+
"version": "0.5.0",
44
"description": "Generic JSON data pipeline tools, with dynamic transformation (using JSONata rules), resolving JSON Schema (using JSONata rules), and then validation (using JSON Schema). For Node-RED and for command-line.",
55
"main": "index.js",
66
"readmeFilename": "readme.md",
@@ -47,6 +47,6 @@
4747
"start": "node ./index.js",
4848
"lint": "node ./node_modules/.bin/jshint *.js",
4949
"pretest": "npm run-script lint",
50-
"test": "printf '{\"payload\":{\"id\":\"TA120-T246177\",\"type\":\"Cesva-TA120\",\"NoiseLevelObserved\":{\"id\":\"TA120-T246177-NoiseLevelObserved-2018-09-17T07:01:09.000000Z\",\"sonometerClass\":\"1\",\"location\":{\"coordinates\":[24.985891,60.274286],\"type\":\"Point\"},\"measurand\":[\"LAeq | 48.6 | A-weighted, equivalent, sound level\"],\"dateObserved\":\"2018-09-17T07:01:09.000000Z\",\"LAeq\":48.6,\"type\":\"NoiseLevelObserved\"}}} \\n {\"payload\":{\"id\":\"TA120-T246183\",\"type\":\"Cesva-TA120\",\"NoiseLevelObserved\":{\"id\":\"TA120-T246183-NoiseLevelObserved-2018-09-17T07:01:15.000000Z\",\"sonometerClass\":\"1\",\"location\":{\"coordinates\":[24.9030921,60.161804],\"type\":\"Point\"},\"measurand\":[\"LAeq | 37.6 | A-weighted, equivalent, sound level\"],\"dateObserved\":\"2018-09-17T07:01:15.000000Z\",\"LAeq\":37.6,\"type\":\"NoiseLevelObserved\"}}}' | node ./index.js json-multi-schema-transformer --transformsUrl='\"https://raw.githubusercontent.com/alexandrainst/node-red-contrib-json-multi-schema/master/examples/smart-data-transforms.json\"' | node ./index.js json-multi-schema-resolver --mappingsUrl='\"https://raw.githubusercontent.com/alexandrainst/node-red-contrib-json-multi-schema/master/examples/smart-data-models.json\"' | node ./index.js json-multi-schema-validator"
50+
"test": "rm /tmp/*.tmp.js ; printf '{\"payload\":{\"id\":\"TA120-T246177\",\"type\":\"Cesva-TA120\",\"NoiseLevelObserved\":{\"id\":\"TA120-T246177-NoiseLevelObserved-2018-09-17T07:01:09.000000Z\",\"sonometerClass\":\"1\",\"location\":{\"coordinates\":[24.985891,60.274286],\"type\":\"Point\"},\"measurand\":[\"LAeq | 48.6 | A-weighted, equivalent, sound level\"],\"dateObserved\":\"2018-09-17T07:01:09.000000Z\",\"LAeq\":48.6,\"type\":\"NoiseLevelObserved\"}}} \\n {\"payload\":{\"id\":\"TA120-T246183\",\"type\":\"Cesva-TA120\",\"NoiseLevelObserved\":{\"id\":\"TA120-T246183-NoiseLevelObserved-2018-09-17T07:01:15.000000Z\",\"sonometerClass\":\"1\",\"location\":{\"coordinates\":[24.9030921,60.161804],\"type\":\"Point\"},\"measurand\":[\"LAeq | 37.6 | A-weighted, equivalent, sound level\"],\"dateObserved\":\"2018-09-17T07:01:15.000000Z\",\"LAeq\":37.6,\"type\":\"NoiseLevelObserved\"}}}' | node ./index.js json-multi-schema-resolver --mappingsUrl='\"https://raw.githubusercontent.com/alexandrainst/node-red-contrib-json-multi-schema/master/examples/smart-data-transforms.json\"' | node ./index.js json-multi-schema-transformer | node ./index.js json-multi-schema-resolver --mappingsUrl='\"https://raw.githubusercontent.com/alexandrainst/node-red-contrib-json-multi-schema/master/examples/smart-data-models.json\"' | node ./index.js json-multi-schema-validator"
5151
}
5252
}

0 commit comments

Comments
 (0)