diff --git a/ditto/mappers/lib/comparator.js b/ditto/mappers/lib/comparator.js new file mode 100644 index 0000000..55d0f4a --- /dev/null +++ b/ditto/mappers/lib/comparator.js @@ -0,0 +1,28 @@ +module.exports = function comperator(operation, firstValue, secondValue) { + + // Make sure we always apply strict comparison + if (operation === '==' || operation === '!=') { + operation = operation.replace('=', '=='); + } + + switch(operation) { + case '===': + return firstValue === secondValue; + case '==': + return firstValue === secondValue; + case '!==': + return firstValue !== secondValue; + case '!=': + return firstValue !== secondValue; + case '>': + return firstValue > secondValue; + case '<': + return firstValue < secondValue; + case '<=': + return firstValue <= secondValue; + case '>=': + return firstValue >= secondValue; + } + return false; + +} \ No newline at end of file diff --git a/ditto/mappers/lib/extractor.js b/ditto/mappers/lib/extractor.js new file mode 100644 index 0000000..9e9d5ed --- /dev/null +++ b/ditto/mappers/lib/extractor.js @@ -0,0 +1,38 @@ + +const _ = require('lodash'); +const comparator = require('./comparator'); + +module.exports = class Extractor { + + constructor(plugins, transformer){ + this.plugins = plugins; + this.transformer = transformer; + } + + extract (document, path, output) { + + const extractor = this; + + if (_.isNil(path)) return; + + if (path === '!') return document; + + if (_.startsWith(path, '>>')) { + return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); + } else if (_.startsWith(path, '!')) { + return _.get(output, path.replace('!', '')); + } else if (/\|\|/.test(path) && !path.includes('??') ) { + let pathWithDefault = path.split(/\|\|/); + return extractor.extract(document, pathWithDefault[0], output) || extractor.extract(document, `${pathWithDefault[1]}`, output); + } else if (path.includes('??') ){ + const parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], + path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); + const _comparator = extractor.transformer.transform(document, parameters.comparator, output); + const _condition = extractor.transformer.transform(document, parameters.condition, output); + + return comparator(parameters.comparison, _comparator, _condition) ? extractor.transformer.transform(document, parameters.targetValue, output) : null; + + } else return _.get(document, path); + } + +} \ No newline at end of file diff --git a/ditto/mappers/lib/hoover.js b/ditto/mappers/lib/hoover.js new file mode 100644 index 0000000..ba30ea7 --- /dev/null +++ b/ditto/mappers/lib/hoover.js @@ -0,0 +1,32 @@ +/** + * Remove all specified keys from an object, no matter how deep they are. + * The removal is done in place, so run it on a copy if you don't want to modify the original object. + * This function has no limit so circular objects will probably crash the browser + * + * @param obj The object from where you want to remove the keys + * @param keys An array of property names (strings) to remove + */ +module.exports = function hoover(obj, keys) { + var index; + for (var prop in obj) { + // check that this is objects own property not from prototype prop inherited + if (obj.hasOwnProperty(prop)) { + switch(typeof(obj[prop])) { + case 'string': + index = keys.indexOf(prop); + if(index > -1) { + delete obj[prop]; + } + break; + case 'object': + index = keys.indexOf(prop); + if (index > -1) { + delete obj[prop]; + } else { + hoover(obj[prop], keys); + } + break; + } + } + } +} \ No newline at end of file diff --git a/ditto/mappers/lib/transformer.js b/ditto/mappers/lib/transformer.js new file mode 100644 index 0000000..828962f --- /dev/null +++ b/ditto/mappers/lib/transformer.js @@ -0,0 +1,38 @@ + +const _ = require('lodash'); + +const Extractor = require('./extractor'); + +module.exports = class Transformer { + + constructor(plugins){ + this.plugins = plugins; + this.extractor = new Extractor(plugins, this); + } + + transform (document, path, output, $key) { + + const transformer = this; + + if (path.includes('??')) { + return transformer.extractor.extract(document, path, output); + } else if (_.startsWith(path, '$')) { + return eval(path); + } else if (_.startsWith(path, '@!')) { + return eval(path.replace('@!', '')); + } else if (_.startsWith(path, '@')) { + + const parameters = _.zipObject(['name', 'arguments'], path.split(/\((.*)\)/).filter(Boolean)); + const functionCall = parameters.name.replace('@', ''); + const paramteresValues = _.map(parameters.arguments.split('|'), param => { + return transformer.transform(document, param.replace(',', '|'), output, $key) + }).filter(Boolean); + + if (paramteresValues.length && transformer.plugins[functionCall]) { + return transformer.plugins[functionCall](...paramteresValues); + } + + } + return transformer.extractor.extract(document, path, output); + } +} \ No newline at end of file diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 340fb1d..d6bdf74 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -1,5 +1,7 @@ const _ = require('lodash'); +const Transformer = require('./lib/transformer'); + /** * @function map * @@ -9,249 +11,64 @@ const _ = require('lodash'); */ async function map(document, mappings, plugins) { - /** - * @function processMappings - * - * @description Process the mappings file and map it to the actual values - * Check if the key is static or dynamic assignment. Static assignment assigns directly the - * the hardcoded value into the result object after removing the extra >> signs - * Otherwise, check if the path of the object exists. If so, check if the values have to be - * added in an array (push) or just simple object assignment - * If a prerequisite is defined then we will only assign the value if the prerequisite has passed - * - * @param {Object} document the object document we want to map - * @param {Object} result the object representing the result file - * @param {Object} mappings the object presenting the mappings between target document and mapped one - */ - function processMappings(mappings, document, result, parentKey) { - - _.each(mappings, function(path, key) { - - key = parentKey ? `${parentKey}.${key}` : key; - - // The path can be multiple definitions for retreiving the same piece of information from multiple places - // here we check for that and construct the appropriate document and mappings objects to deal with it - if (_.isArray(path)) { - _.each(path, function(subPath){ - var subMapping = {}; - subMapping[key] = subPath; - return processMappings(subMapping, document, result); - }); + const transformer = new Transformer(plugins); + function processMappings(document, mappings, output, options = {isIterable: false}) { + + if (!mappings) return document; + + else if (options.isIterable) { + return _.map(document, (_document, key) => { + return processMappings(_document, mappings, output, {isIterable: false, key}); + }); + } else if (typeof mappings === 'string') { + return transformer.transform(document, mappings, output, _.get(options, 'key')); + } else if (mappings.hasOwnProperty('output')) { + let _output, _innerDocument = '!', options = {}; + if (mappings.hasOwnProperty('innerDocument')) { + options['isIterable'] = true; + _innerDocument = mappings.innerDocument; } - - // Check if the path presents a flat object with a value that can be accessible directly with an _.get - if (!_.isPlainObject(path)) { - let value = applyTransformation(key, path, document.$key, document.$value); - - // here we check if the parent key of the value is not defined and only define it at least one value is there - // this resolves the issue of having an empty object defined e.g. location: {} - // the check if (!_.isUndefined(value)) will make sure we have null values to be picked up - // by our postMapper rather than having if (!_.isUndefined(value) && !!value) :) - if (_.isString(value)) { - if (!!value.length) _.set(result, key, value); - } else if (!_.isUndefined(value) && !_.isNull(value)) _.set(result, key, value); - - } else { - - // Check if the object is a nested object of objects or array of objects or not - if (!path.output) { - // Instantiate the empty object in the desired key and pass it to the recursive function - return processMappings(path, document, result, key); - - } else { - - // Reaching here we now know for sure that we will be creating a set of objects (array or object of objects) - // Assign the result object with its correct type defined in the path output - if (!_.has(result, key)) _.set(result, key, path.output) - - _.each(applyTransformation('', path.innerDocument), function($value, $key) { - - // first we need to check if we will be pushing objects or just plain items to an array - // This can be done by checking if we define a mappings object or not - if (path.mappings) { - - var innerResult = {}; - var processingDocument = path.innerDocument === '!' ? document : _.merge(_.cloneDeep($value), {$value: $value, $key: $key}); - - processMappings(path.mappings, processingDocument, innerResult); - - if (_.isArray(path.required) && - _.find(path.required, requiredPath => _.isNil(_.get(innerResult, requiredPath)))){ - innerResult = null; - } - - parseMappings(result, innerResult, path, key, $value, $key); - - } else { - // reaching here means that we are pushing only to a flat array and not an object - if (_.startsWith(path.value, '@')) { - _.updateWith(result, key, function(theArray){ return applyTransformation(key, path.value) }, []); - return false; - } else return _.updateWith(result, key, function(theArray){ theArray.push($value[path.value]); return theArray }, []); - } - - // here we are breaking out of the each if we have defined the innerDocument as the parent document - if (path.innerDocument === '!') return false; - }); - - function parseMappings(result, innerResult, path, key, $value, $key) { - // based on the type of the result [] or {} we will either push or assign with key - if (!!innerResult) { - if (_.isArray(_.get(result,key))) { - if (!!path.prerequisite) { - if (!!eval(path.prerequisite)) _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); - } else _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); - } else { - let fullPath = `${key}.${applyTransformation(key, path['key'], $key, $value)}`; - if (!!path.prerequisite) { - if (!!eval(path.prerequisite)) _.set(result, fullPath, innerResult); - } else _.set(result, fullPath, innerResult); - } - } - - // After assigning the innerResult we need to make sure that there are no requirements on it (e.g., unique array) - if (!!path.requirements) { - _.each(path.requirements, function(requirement){ - _.set(result, key, applyTransformation(key, requirement, $key, $value)); - }); - } - return; - } - } + _output = processMappings(transformer.transform(document, _innerDocument, output), mappings.mappings, output, options); + if (mappings.hasOwnProperty('required')) { + _output = _.last(_.map(mappings.required, _required => { return _.filter(_.flatten([_output]), _required) })); } - }); - - /** @function applyTransformation - * @description Apply a tranformation function on a path - * - * @param {String} path the path to pass for the _.get to retrieve the value - * @param {String} key the key of the result object that will contain the new mapped value - */ - function applyTransformation(key, path, $key, $value) { - - if (path.includes('??')) { - return getValue(path, $value); - } else if (_.startsWith(path, '$')) { - return eval(path); - } else if (_.startsWith(path, '@!')) { - return eval(path.replace('@!', '')); - } else if (_.startsWith(path, '@')) { - - /** - * The parts in the string function are split into: - * before the @ is the first paramteres passed to the function - * the function name is anything after the @ and the () if exists - * the paramteres are anything inside the () separated by a | - */ - let paramteresArray, paramteresValues = []; - - // Regular expression to extract any text between () - let functionParameteres = path.match(/.+?\((.*)\)/); - let functionCall = path.split('(')[0].replace('@', ''); - - // Now we want to split the paramteres by a | in case we pass more than one param - // We also need to check if we are assigning a default value for that function denoted by a * - if (!!functionParameteres) { - - // We need to check if the function parameters have inlined functions that we need to execute - paramteresArray = _.compact(functionParameteres[1].split('|')); + if (_.isPlainObject(mappings.output)) { + const __output = _.flattenDeep([_output]); + _output = _.zipObject(_.map(__output, $ => { return $.$$key}), __output); + } else if (mappings.hasOwnProperty('$push')) { + _output = _.compact(_output.map($ => {return $.$value})); + } + return _output; - if (_.last(paramteresArray).includes('*') && !! applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value)) { - return applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value) - } else { - // we compact the array here to remove any undefined objects that are not caught by the _.get in the map function - paramteresValues = _.union(paramteresValues, _.map(paramteresArray, function(param){ return _.startsWith(param, '$') ? eval(param) : applyTransformation(key, param.replace(',', '|'), $key, $value) })); + } else { + const output = {}; + const reducer = (input, fn) => { + return _.reduce(input,(accumulator, currentValue) => fn(accumulator,currentValue)) + } + _.each(mappings, (mapping, path) => { + if (mappings.hasOwnProperty(path)) { + if (mapping.hasOwnProperty('key')) { + mapping.mappings['$$key'] = mapping.key; + } + let _output = processMappings(document, mapping, output, options); + if (Array.isArray(mapping)) { + _output = Array.isArray(mapping[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); + } + if (mapping.hasOwnProperty('requirements')) { + _.each(mapping.requirements, requirement => { + _output = transformer.transform(_output, requirement); + }); } - } - - // Extract the function call and the first parameter - // Now the paramteres array contains the PATH for each element we want to pass to the @function - // We need now to actually get the actual values of these paths - // If the getValues fails that means that we are passing a hardocded value, so pass it as is - // Only execute the function if the parameters array is not empty - if (!!_.compact(paramteresValues).length && plugins[functionCall]) { - return plugins[functionCall].apply(null, paramteresValues); + if (!_.isNil(_output)) output[path] = _output; } + }); - } else return getValue(path, $value); - } - - /** - * @function getValue - * @description This function will get the value using the _.get by inspecting the scope of the get - * The existence of the ! will define a local scope in the result object rather than the document - * - * The Flat structure can contain the following cases: - * - starts with !: This will denote that the contact on which the _.get will be on a previously extracted - * value in the result file - * - starts with >>: This means a hardcoded value e.g., >>test -> test - * - contains @: This means that a function will be applied on the value before the @ sign - * The functions first parameter will be the value before the @ and any additional parameters will be defined - * between the parenthesis e.g., name@strip(','') -- will call --> strip(name, ',') - * - contains %: This will denote a casting function to the value using eval - * e.g., >>%true -> will be a true as a boolean and not as a string - * - contains ?? means that a condition has to be applied before assigning the value. The condition to apply - * is the one defined after the -> - * - * @param {String} path the path to pass for the _.get to retrieve the value - * @return {Object} result the object representing the result file - */ - function getValue(path, subDocument) { - - if (!path) return; - - if (path === '!') return document; - - if (_.startsWith(path, '>>')) { - return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); - } else if (_.startsWith(path, '!')) { - return _.get(result, path.replace('!', '')); - } else if (/\|\|/.test(path) && !path.includes('??') ) { - let pathWithDefault = path.split(/\|\|/); - return getValue(pathWithDefault[0], subDocument) || getValue(`${pathWithDefault[1]}`); - } else if (path.includes('??') ){ - // First we need to get the value the condition is checking against .. and get the main value only if it is truthy - let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], - path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); - - // Run a comparison between the values, and if fails skip the current data - const firstValue = applyTransformation('', parameters.comparator, '', JSON.stringify(subDocument)); - const secondValue = applyTransformation('', parameters.condition, '', JSON.stringify(subDocument)) - let isValidValue = operation(parameters.comparison, firstValue, secondValue); - - return isValidValue ? applyTransformation(null, parameters.targetValue, null, subDocument) : null; - } else { - // Here we check if the subDocument is a string (which indicates we need to get the value from a sub-path) - // We check for it to be a string because otherwise it can be the index of an array in the _.map() - return _.isPlainObject(subDocument) ? _.get(subDocument, path) : _.get(document, path); - } - - /** - * Runs a comparison ( === , ==, !==, != ) against the given values - * @param {string} op - * @param {*} value1 - * @param {*} value2 - */ - function operation(op, value1, value2){ - switch(op){ - case '===': - return value1 === value2; - case '==': - return value1 == value2; - case '!==': - return value1 !== value2; - case '!=': - return value1 != value2; - } - return false; - } + return output; } } - var output = {} - processMappings(mappings, document, output); - return output; + return processMappings(document, mappings); } module.exports = map; \ No newline at end of file diff --git a/ditto/mappers/postmap.js b/ditto/mappers/postmap.js index 218493a..eaf913e 100644 --- a/ditto/mappers/postmap.js +++ b/ditto/mappers/postmap.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const hoover = require('./lib/hoover'); /** * @function postMap @@ -11,7 +12,10 @@ async function postMap(result) { function isEmptyObject(value, key) { return _.isPlainObject(value) && _.isEmpty(value) ? true : false; } - return _(result).omitBy(_.isUndefined).omitBy(_.isNull).omitBy(isEmptyObject).value(); + + // Make sure we remove Ditto built in keys + hoover(result, ['$$key']); + return _(result).omitBy(_.isNil).omitBy(isEmptyObject).value(); } module.exports = postMap; \ No newline at end of file diff --git a/test/ditto.js b/test/ditto.js index 3ba9cb9..df486b5 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -15,11 +15,9 @@ describe('ditto Interface', function(){ let dummyMappings = require('./mappings/test'); let dummyPlugin = { - transformTwitterHandle: function transformTwitterHandle(target){ - _.each(target, function(link){ - link.value = "@ahmadaassaf" - }); - return target; + transformTwitterHandle: (target) => { + const twitterHandle = target.match(/^https?:\/\/(www\.)?twitter\.com\/(#!\/)?([^\/]+)(\/\w+)*$/); + return !!twitterHandle ? `@${twitterHandle[3]}` : null; } }; @@ -51,7 +49,11 @@ describe('ditto Interface', function(){ }); it('should be able to assign a new value based on an already mapped one', function(){ - assert.strictEqual(this.result.displayName, "Ahmad Assaf"); + assert.strictEqual(this.result.displayName, "Ahmad Ahmad AbdelMuti Assaf"); + }); + + it('should be able to allow for duplicate values without flattening/compacting them', function(){ + assert.strictEqual(this.result.fullName, "Ahmad Ahmad AbdelMuti Assaf"); }); it('should be able to apply a condition on an output path', function(){ @@ -200,7 +202,7 @@ describe('ditto Interface', function(){ "prerequisite": "!!innerResult.value", "required": ["value"], "mappings": { - "value": "value??type#==#>>social" + "value": "value??type#===#>>social" } } }).unify(badObj).then((result) => { diff --git a/test/mappings/services/facebook.js b/test/mappings/services/facebook.js index f30f132..0ccc50e 100644 --- a/test/mappings/services/facebook.js +++ b/test/mappings/services/facebook.js @@ -19,7 +19,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile.profileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile.profileUrl)", @@ -32,8 +31,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -50,15 +52,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile.id)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile.id)", @@ -68,8 +72,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { @@ -88,8 +95,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -108,8 +118,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/facebook_raw.js b/test/mappings/services/facebook_raw.js index d5272cc..0ba0437 100644 --- a/test/mappings/services/facebook_raw.js +++ b/test/mappings/services/facebook_raw.js @@ -22,7 +22,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.link)", @@ -35,15 +34,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -53,8 +54,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { @@ -74,8 +78,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -94,14 +101,16 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output" : {}, - "innerDocument": "!", "key" : "@generateId(data.picture.data.url)", "required" : ["value"], "mappings" : { @@ -111,8 +120,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/github.js b/test/mappings/services/github.js index 6e1ab1d..868f70b 100644 --- a/test/mappings/services/github.js +++ b/test/mappings/services/github.js @@ -19,7 +19,6 @@ module.exports = { "values" : [{ "output": {}, "key": "@generateIdForLinks(data.profile.profileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile.profileUrl)", @@ -34,7 +33,6 @@ module.exports = { },{ "output": {}, "key": "@generateId(data.profile._json.blog)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.blog)", @@ -44,15 +42,17 @@ module.exports = { }], "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { "values" : { "output": {}, "key": "@generateId(data.profile._json.company)", - "innerDocument": "!", "required": ["organisationName"], "mappings" : { "id" : "@generateId(data.profile._json.company)", @@ -61,15 +61,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.email)", @@ -79,8 +81,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { diff --git a/test/mappings/services/github_raw.js b/test/mappings/services/github_raw.js index 10dbcb2..fcda236 100644 --- a/test/mappings/services/github_raw.js +++ b/test/mappings/services/github_raw.js @@ -19,7 +19,6 @@ module.exports = { "values" : [{ "output": {}, "key": "@generateIdForLinks(data.html_url)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.html_url)", @@ -33,7 +32,6 @@ module.exports = { } },{ "output": {}, - "innerDocument": "!", "key": "@generateId(data.blog)", "required": ["value"], "mappings" : { @@ -44,15 +42,17 @@ module.exports = { }], "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { "values" : { "output": {}, "key": "@generateId(data.company)", - "innerDocument": "!", "required": ["organisationName"], "mappings" : { "id" : "@generateId(data.company)", @@ -61,15 +61,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -79,15 +81,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.avatar_url)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.avatar_url)", @@ -96,8 +100,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { diff --git a/test/mappings/services/google.js b/test/mappings/services/google.js index 59023b8..22da5e2 100644 --- a/test/mappings/services/google.js +++ b/test/mappings/services/google.js @@ -21,7 +21,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile._json.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile._json.link)", @@ -34,15 +33,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.picture)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.picture)", @@ -52,8 +53,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -70,8 +74,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/google_raw.js b/test/mappings/services/google_raw.js index 5b248a1..11913cd 100644 --- a/test/mappings/services/google_raw.js +++ b/test/mappings/services/google_raw.js @@ -26,7 +26,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.link)", @@ -39,15 +38,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.picture)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.picture)", @@ -57,15 +58,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -75,8 +78,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/linkedin.js b/test/mappings/services/linkedin.js index f601da9..7f09095 100644 --- a/test/mappings/services/linkedin.js +++ b/test/mappings/services/linkedin.js @@ -20,7 +20,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile._json.publicProfileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile._json.publicProfileUrl)", @@ -35,8 +34,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -53,8 +55,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { @@ -79,8 +84,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -102,8 +110,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "languages" : { @@ -119,15 +130,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!languages.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.pictureUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.pictureUrl)", @@ -136,8 +149,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/linkedin_raw.js b/test/mappings/services/linkedin_raw.js index b37474a..13ebf1c 100644 --- a/test/mappings/services/linkedin_raw.js +++ b/test/mappings/services/linkedin_raw.js @@ -20,7 +20,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.publicProfileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.publicProfileUrl)", @@ -35,15 +34,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.emailAddress)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.emailAddress)", @@ -53,8 +54,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { @@ -79,8 +83,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -102,8 +109,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "languages" : { @@ -119,25 +129,31 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!languages.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, - "key": "@generateId($value)", + "key": "@generateId(!)", "innerDocument": "data.pictureUrls.values", "required": ["value"], "mappings" : { - "id" : "@generateId($value)", - "value": "$value" + "id" : "@generateId(!)", + "value": "!" } }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/test.js b/test/mappings/test.js index 601de22..9441af9 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -1,56 +1,90 @@ -'use strict'; +const _ = require('lodash'); module.exports = { - "name" : "firstName", - "default_name" : "nonExistingProperty||>>this_should_be_the_firstName", - "nickname" : "nickname||>>nickname_not_found", - "isNickNameFound" : "nickname||>>%false", - "isDynamicDefault" : "nickname||firstName", - "fullName" : "@concatName(firstName|lastName)", - "fullNameDefault" : "@concatName(firstName|*!fullName)", - "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", - "completeName" : "@concatName(firstName|!fullName)", - "displayName" : "!fullName", + "name": "firstName", + "default_name": "nonExistingProperty||>>this_should_be_the_firstName", + "nickname": "nickname||>>nickname_not_found", + "isNickNameFound": "nickname||>>%false", + "isDynamicDefault": "nickname||firstName", + "fullName": "@concatName(firstName|middleName||>>AbdelMuti|lastName)", + "fullNameDefaultHardcoded": "@concatName(nonExistingProperty)||>>default", + "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", + "fullNameDefault": "!fullName_withNotFoundMiddle", + "completeName": "@concatName(firstName|!fullName)", + "displayName": "!fullName", "email": { "value": "email" }, "links": "links", + "social_links_objectified": [{ + "output": {}, + "innerDocument": "links", + "required": ["value"], + "key": "!", + "mappings": { + "value": "!", + "type": ">>test", + "order": "$key", + "social": ">>%true" + } + }, { + "output": {}, + "innerDocument": "social", + "required": ["value"], + "key": "value", + "mappings": { + "value": "value", + "service": "service", + "type": ">>social" + } + }], "social_links": [{ - "output" : [], - "innerDocument": "!links", + "output": [], + "innerDocument": "links", "required": ["value"], - "mappings" : { - "value": "$value", + "mappings": { + "value": "!", "type": ">>test", "order": "$key", "social": ">>%true" } - },{ - "output" : [], + }, { + "output": [], "innerDocument": "social", "required": ["value"], - "mappings" : { - "value" : "value", + "mappings": { + "value": "value", "service": "service", "type": ">>social" } }], + "messaging": { + "output": [], + "innerDocument": "linksv2.values", + "required": ["value"], + "mappings": { + "service": "@getLinkService(value|service)", + "type": "@getLinkType(value|@getLinkService(value,service))", + "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#===#>>messaging" + } + }, "website_addresses_keyless": { "output": [], "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", + "required": ["value"], "mappings": { - "value": "value??type#==#>>website", + "value": "value??type#===#>>website", "type": ">>other", } }, "website_addresses": { "output": {}, "innerDocument": "linksv2.values", - "key": "id", + "required": ["value"], "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", + "key": "id", "mappings": { - "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", + "value": "value??keys[0]#===#>>f5e32a6faaa7ead6ba201e8fa25733ee", "type": ">>other", "keys": "keys" } @@ -58,58 +92,50 @@ module.exports = { "social_media_addresses": { "output": [], "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", - "requirements": ["@uniqueArray(!social_media_addresses|>>value)", "@transformTwitterHandle(!social_media_addresses)"], + "required": ["value"], + "requirements": ["@uniqueArray(!|>>value)"], "mappings": { - "value": "value??type#==#>>social" + "value": "@transformTwitterHandle(value)??service#===#>>twitter" } }, - "messaging": { - "output": [], - "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", + "social_links_objects": { + "output": {}, + "innerDocument": "links", + "key": "@generateId(!)", "mappings": { - "service": "@getLinkService(value|service)", - "type" : "@getLinkType(value|@getLinkService(value,service))", - "value" : "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" + "value": "!" } }, - "social_links_objects": { - "output" : {}, - "innerDocument": "!links", - "key": "@generateId($value)", - "mappings" : { - "value": "$value" + "experience": { + "output": [], + "innerDocument": "work", + "mappings": { + "name": "companyName", + "role": "title", + "startDate": "@parseDate(startDate)", + "current": "current" } }, "experience_primary": { "values": { - "output" : {}, - "innerDocument": "!", - "key" : "@generateId(title|company)", - "mappings" : { - "id" : "@generateId(title|company)", - "role" : "title", + "output": {}, + "key": "@generateId(title|company)", + "mappings": { + "id": "@generateId(title|company)", + "role": "title", "organisationName": "company" } } }, - "experience": { - "output": [], - "innerDocument": "work", - "mappings": { - "name" : "companyName", - "role" : "title", - "startDate": "@parseDate(startDate)", - "current" : "current" - } - }, "primaryExperience": "!experience[0]", "primaryRole": "!experience[0].role", "experiences": { "output": [], "innerDocument": "work", - "value": "companyName" + "$push": true, + "mappings": { + "$value": "companyName" + } }, "experience_object": { "values": { @@ -117,7 +143,7 @@ module.exports = { "innerDocument": "work", "key": "@generateId(companyName|title)", "mappings": { - "id": "@generateId(companyName|title)", + "id": "@generateId(companyName|title)", "name": "companyName", "role": "title", "startDate": "startDate", @@ -136,13 +162,19 @@ module.exports = { }, "education_object": { "output": {}, - "key": "@generateId($key|degree)", "innerDocument": "json.education", + "key": "@generateId($key|degree)", "mappings": { "degree": "degree", "location": "location", "universityName": "universityName" } }, - "primaryPhoto": "@createURL(>>http://photo.com/|!fullName)" + // "volunteer": { + // "output": [], + // "innerDocument": "volunteer", + // "value": "@concatName(organisation|>> at |title)" + // }, + "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)", + "createdAt": "@!new Date('2019').toISOString()" } diff --git a/test/results/test.js b/test/results/test.js index 07a6409..83f36aa 100644 --- a/test/results/test.js +++ b/test/results/test.js @@ -6,11 +6,12 @@ module.exports = { "nickname": "nickname_not_found", "isNickNameFound": false, "isDynamicDefault": "Ahmad", - "fullName": "Ahmad Assaf", + "fullName": "Ahmad Ahmad AbdelMuti Assaf", "fullNameDefault": "Ahmad Assaf", "fullNameDefaultHardcoded": "default", - "completeName": "Ahmad Ahmad Assaf", - "displayName": "Ahmad Assaf", + "fullName_withNotFoundMiddle" : "Ahmad Assaf", + "completeName": "Ahmad Ahmad Ahmad AbdelMuti Assaf", + "displayName": "Ahmad Ahmad AbdelMuti Assaf", "email": { "value": "ahmad.a.assaf@gmail.com" }, @@ -48,6 +49,36 @@ module.exports = { "service": "twitter", "type": "social" }], + "social_links_objectified": { + "http://a.com": { + "value": "http://a.com", + "type": "test", + "order": 0, + "social": true + }, + "http://b.com": { + "value": "http://b.com", + "type": "test", + "order": 1, + "social": true + }, + "http://c.com": { + "value": "http://c.com", + "type": "test", + "order": 2, + "social": true + }, + "http://github.com/ahmadassaf": { + "value": "http://github.com/ahmadassaf", + "service": "github", + "type": "social" + }, + "http://twitter.com/ahmadaassaf": { + "value": "http://twitter.com/ahmadaassaf", + "service": "twitter", + "type": "social" + } + }, "website_addresses_keyless": [{ "value": "https://gravatar.com/ahmadassaf", "type": "other" @@ -146,5 +177,6 @@ module.exports = { "universityName": "University of St.Andrews" } }, - "primaryPhoto": "http://photo.com/ahmadassaf" + "primaryPhoto": "http://photo.com/ahmadassaf", + "createdAt": "2019-01-01T00:00:00.000Z" } diff --git a/test/samples/test.js b/test/samples/test.js index d6462fd..86f62fe 100644 --- a/test/samples/test.js +++ b/test/samples/test.js @@ -3,6 +3,7 @@ module.exports = { "firstName" : "Ahmad", "lastName": "Assaf", + "middleName": "Ahmad", "email" : "ahmad.a.assaf@gmail.com", "title": "Data Scientist", "company": "Beamery", @@ -85,6 +86,13 @@ module.exports = { "current": false } ], + "volunteer": [ + { + "organisation": "PandaBear", + "title": "Volunteer", + "years": 5 + } + ], "json": { "education" : { "telecomParisTech" : {