WIP attribute type system

Author: alaricsp

lib/importer/attribute-types.js

@@ -0,0 +1,110 @@
+// "input values" are what's found in the input data, and will generally be
+// simple string/boolean/number/date/etc things. "attribute values" are the
+// actual attributes in our output data model and can be some arbitrary type.
+
+// Am attribute type is a pair of functions.
+
+// One validates an input value to see if it can be made into a valid attribute
+// value. It should return a list of warnings and a list of errors.
+
+// One maps an input value to a value of the attribute type.
+
+// ABS TODOs: Make validation and translation happen together - drop
+// AttributeType and just implement it as a function from input value ->
+// (outputValue, [warning], [error]).
+
+// This will enable optionalType to handle errors from the base type by turning
+// them into warnings *and* returning undefined as the translater value.
+
+// Then add a requiredType that wraps any type and makes it "required", treating
+// "" or undefined inputs as errors.
+
+// Base types without being wrapped by requiredType or optionalType may then
+// handled undefined or "" as they see fit.
+
+class AttributeType {
+  constructor(validator, translator) {
+    this.v = validator;
+    this.t = translator;
+  }
+
+  get validator() { return this.v; }
+  get translator() { return this.t; }
+
+  validate(inputValue) { return this.v(inputValue); }
+  translate(inputValue) { return this.t(inputValue); }
+}
+
+exports.AttributeType = AttributeType;
+
+// Create an optional version of an existing type, that allows empty strings or
+// undefined values and maps then to "undefined", and converts any validation errors to warnings
+exports.optionalType = (baseType) => {
+  return new AttributeType(
+    (inputValue) => {
+      if(inputValue !== undefined && inputValue != "") {
+        const [warnings, errors] = baseType.validate(inputValue);
+        // Convert any errors into warnings, as this is an optional field
+        return [warnings.concat(errors), []];
+      }
+      else {
+        return [[],[]];
+      }
+    },
+    (inputValue) => {
+      if(inputValue !== undefined && inputValue != "") return baseType.translate(inputValue);
+      else return undefined;
+    }
+  );
+}
+
+// Create a required version of an existing type, that 
+exports.optionalType = (baseType) => {
+  return new AttributeType(
+    (inputValue) => {
+      if(inputValue !== undefined && inputValue != "") {
+        const [warnings, errors] = baseType.validate(inputValue);
+        // Convert any errors into warnings, as this is an optional field
+        return [warnings.concat(errors), []];
+      }
+      else {
+        return [[],[]];
+      }
+    },
+    (inputValue) => {
+      if(inputValue !== undefined && inputValue != "") return baseType.translate(inputValue);
+      else return undefined;
+    }
+  );
+}
+
+// The default string type. Does not allow empty strings!
+exports.basicStringType = new AttributeType(
+  (inputValue) => {
+    if(inputValue === undefined || inputValue == "") {
+      return [[],["A value must be provided"]];
+    }
+    // Anything else can be converted into a string
+    return [[],[]];
+  },
+  (inputValue) => {
+    return String(inputValue);
+  }
+);
+
+exports.basicNumberType = new AttributeType(
+  (inputValue) => {
+    if(inputValue === undefined || inputValue == "") {
+      return [[],["A value must be provided"]];
+    }
+
+    if(parseFloat(inputValue) === NaN) {
+      return [[],[inputValue + " is not a valid number"]];
+    }
+
+    return [[],[]];
+  },
+  (inputValue) => {
+    return parseFloat(inputValue);
+  }
+);

lib/importer/attribute-types.test.js

@@ -0,0 +1,54 @@
+const attributeTypes = require('./attribute-types');
+
+function testType(t, invalidInputs, fixtures) {
+  for([input,warnings,errors] of invalidInputs) {
+    expect(t.validate(input)).toMatchObject([warnings, errors]);
+  }
+
+  for ([input, output] of fixtures) {
+    expect(t.validate(input)).toMatchObject([[],[]]);
+    expect(t.translate(input)).toEqual(output);
+  }
+};
+
+test('basic string', () => {
+  const invalidInputs = [
+    [undefined, [], ["A value must be provided"]],
+    ["", [], ["A value must be provided"]]
+  ];
+
+  const fixtures = [
+    ["foo", "foo"],
+    [123, "123"],
+  ];
+
+  testType(attributeTypes.basicStringType, invalidInputs, fixtures);
+});
+
+test('optional basic string', () => {
+  const invalidInputs = [];
+
+  const fixtures = [
+    ["foo", "foo"],
+    [123, "123"],
+    ["", undefined],
+    [undefined, undefined]
+  ];
+
+  testType(attributeTypes.optionalType(attributeTypes.basicStringType), invalidInputs, fixtures);
+});
+
+test('basic number', () => {
+  const invalidInputs = [
+    [undefined, [], ["A value must be provided"]],
+    ["", [], ["A value must be provided"]],
+    ["foo", [], ["foo is not a valid number"]]
+  ];
+
+  const fixtures = [
+    [123, 123],
+    ["123", "123"],
+  ];
+
+  testType(attributeTypes.basicNumberType, invalidInputs, fixtures);
+});

lib/importer/backend.js

@@ -1,6 +1,7 @@
 const xlsx = require("xlsx");
 const crypto = require("crypto");
 const assert = require('node:assert').strict;
+const attributeTypes = require('./attribute-types');
 
 // An implementation of the interface described in https://struct.register-dynamics.co.uk/trac/wiki/DataImporter/API
 
@@ -454,6 +455,13 @@ function validateMapping(range, mapping) {
     assert(attrSource >= 0);
     assert(attrSource < columnsInRange);
   }
+
+  // Output types are optional, we fall back to basicStringType if not specified
+  if(mapping.attributeTypes) {
+    for(const [attribute, attrType] of Object.entries(mapping.attributeTypes)) {
+      assert(attrType instanceof AttributeType);
+    }
+  }
 }
 
 function mapCellValue(cell) {
@@ -477,6 +485,7 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
   const data = sheet["!data"];
   const merges = sheet["!merges"];
   const attrMap = Object.entries(mapping.attributeMappings);
+  const attrTypes = mapping.attributeTypes || {};
 
   for(let rowIdx=range.start.row; rowIdx <= range.end.row; rowIdx++) {
     let row = getMergedRow(data, merges, rowIdx);
@@ -487,6 +496,7 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
     if (row) {
       attrMap.forEach((element) => {
         const [attr, m] = element;
+
         // For now, attribute mappings are just integer column offsets
         const inputColumn = range.start.column + m;
         if (inputColumn >= row.length) {
@@ -504,7 +514,29 @@ exports.SessionPerformMappingJob = (sid, range, mapping) => {
     }
 
     if (foundSomeValues) {
-      records.push(record);
+      // Only if we found something do we validate and map the types
+      mappedRecord = {};
+
+      Object.entries(record).forEach((element) => {
+        const [attr, inputVal] = element;
+        const attrType = attrTypes[attr] || attributeTypes.basicStringType;
+
+        const [warnings, errors] = attrType.validate(inputVal);
+        warnings.forEach((text) => {
+          rowWarnings.push([attr,text]);
+        });
+        errors.forEach((text) => {
+          rowErrors.push([attr,text]);
+        });
+
+        if(errors.length == 0) {
+          mappedRecord[attr] = attrType.translate(inputVal);
+        }
+      });
+
+      if (rowErrors.length == 0) {
+        records.push(record);
+      }
     }
 
     if(rowWarnings.length > 0) {

lib/importer/backend.test.js

@@ -1,5 +1,5 @@
 const backend = require('./backend');
-
+const attributeTypes = require('./attribute-types');
 
 const testFiles = new Map([
   ["test", [