Using guessed formats to guide the user in picking formats.

Includes a few bug fixes to the guesser!

Author: alaric-rd

lib/importer/nunjucks/importer/macros/format_picker.njk

@@ -59,21 +59,26 @@
   </thead>
   <tbody class="govuk-table__body">
     {% set mappingsLen = mapping | length %}
+    {% set possibleFormatsByColumn = importerPossibleColumnFormats(params.data) %}
     {% for h in headings.data %}
       {% set hIndex = loop.index0 %}
       {% set currentValue = importerErrorMappingData(error, hIndex) %}
-      {% set possibleFormats = importerPossibleColumnFormats(params.data, hIndex) %}
+      {% set possibleFormats = possibleFormatsByColumn[hIndex] %}
 
     <tr class="govuk-table__row" id="field-{{ h.name | slugify }}">
       <th scope="row" class="govuk-table__header">{{ h.name }}</th>
       <td class="govuk-table__cell">{{ h.examples }}</td>
       <td class="govuk-table__cell govuk-table__cell--numeric">
          {% if possibleFormats %}
          <select class="govuk-select" style="float: right;" name="format-{{h.index}}">
-                <option name=""></option>
                 {% for format in possibleFormats %}
-                    <option value="{{format.name}}" {% if format.name == currentValue %}selected{% endif %}>
-                      {{format.displayName}}
+                    <option value="{{format.name}}"
+                            {% if format.name == currentValue %}selected{% endif %}>
+                      {% if format.likely %}
+                         {{format.displayName}} (*)
+                      {% else %}
+                         {{format.displayName}}
+                      {% endif %}
                     </option>
                 {% endfor %}
          </select>

lib/importer/nunjucks/views/error-handling/server-error.njk

@@ -45,11 +45,6 @@
 {% endblock %}
 
 
-{% block footer %}
-  {{ govukFooter({}) }}
-{% endblock %}
-
-
 {% block pageScripts %}
 <script>
   ;(() => {

lib/importer/src/dudk/backend.js

@@ -70,7 +70,7 @@ exports.SessionSetFile = (sid, filename) => {
 ///
 
 exports.SessionSetAttributes = (sid, attrDefs) => {
-  sessionStore.apply(sid, (s) => { s.attribudeDefs = attrDefs })
+  sessionStore.apply(sid, (s) => { s.attributeDefs = attrDefs })
 }
 
 exports.SessionSetHeaderRange = (sid, range) => {
@@ -124,25 +124,57 @@ exports.SessionSetFormatChoices = (sid, choices) => {
 // have the required information.
 exports.SessionGetColumnsNeedingFormats = (sid) => {
   const session = sessionStore.get(sid);
-  const attributes = session.attribudeDefs;
+  const attributes = session.attributeDefs;
   const mappings = session.mappingRules;
   const supportedTypes = attributeTypes.supportedTypes;
 
   let columnsNeedingFormats = new Array();
 
   Object.entries(mappings).forEach((entry) => {
     const [colIdx, columnAttribute] = entry;
-    const columnDef = attributes.find((attr) => attr.name == columnAttribute);
-    const columnTypeName = columnDef.type;
-    const columnType = supportedTypes.get(columnTypeName);
-    if (columnType.formats) {
-      columnsNeedingFormats.push(colIdx);
+    if(columnAttribute != "") {
+      const columnDef = attributes.find((attr) => attr.name == columnAttribute);
+      const columnTypeName = columnDef.type;
+      const columnType = supportedTypes.get(columnTypeName);
+      if (columnType.formats) {
+        columnsNeedingFormats.push(colIdx);
+      }
     }
   });
 
   return columnsNeedingFormats;
 };
 
+// Given guesses as returned by SessionGuessTypes and a list of domain-model
+// fields (with name and type fields), returns the type guesses with each column
+// augmented with a `fields` field - a Map mapping likely field names for this
+// column to an array of possible formats for that field.
+
+exports.SessionSuggestFields = (sid, typeGuesses, domainModelFields) => {
+  let result = [];
+
+  typeGuesses.forEach((colTypeGuess) => {
+    let fieldFormats = new Map(); // Map from field names to likely formats
+    // Examine every field, and check if they have a type that this column might
+    // contain
+    domainModelFields.forEach((field) => {
+      if(colTypeGuess.types.has(field.type)) {
+        // Add this field to the possibilities for the column, using the guessed
+        // formats
+        fieldFormats.set(field.name, colTypeGuess.types.get(field.type));
+      }
+    });
+
+    // Extend the column type guess with the "fields" field
+    let typeGuessesAndFields = Object.assign({}, colTypeGuess);
+    typeGuessesAndFields.fields = fieldFormats;
+    result.push(typeGuessesAndFields);
+  });
+
+  return result;
+};
+
+
 ///
 /// Information about the loaded input file
 ///
@@ -514,7 +546,7 @@ exports.SessionGuessTypes = (sid, range) => {
   }
 
   // How many columns do we have?
-  const columns = range.end.column-range.start.column;
+  const columns = (range.end.column-range.start.column)+1;
 
   let guesses = new Array(columns);
   if (range.end.row < range.start.row) {
@@ -535,7 +567,7 @@ exports.SessionGuessTypes = (sid, range) => {
   const merges = sheet["!merges"];
 
   // Examine each row in turn, looking at the columns in parallel
-  for (let row = range.start.row; row < range.end.row; row++) {
+  for (let row = range.start.row; row <= range.end.row; row++) {
     let rowData = extractColsFromRow(getMergedRow(data, merges, row),
                                      range.start.column, range.end.column + 1);
     for(let col = 0; col < columns; col++) {
@@ -590,34 +622,6 @@ exports.SessionGetSupportedTypes = (sid) => {
   return attributeTypes.supportedTypes;
 };
 
-// Given guesses as returned by SessionGuessTypes and a list of domain-model
-// fields (with name and type fields), returns the type guesses with each column
-// augmented with a `fields` field - a Map mapping likely field names for this
-// column to an array of possible formats for that field.
-
-exports.SessionSuggestFields = (sid, typeGuesses, domainModelFields) => {
-  let result = [];
-
-  typeGuesses.forEach((colTypeGuess) => {
-    let fieldFormats = new Map(); // Map from field names to likely formats
-    // Examine every field, and check if they have a type that this column might
-    // contain
-    domainModelFields.forEach((field) => {
-      if(colTypeGuess.types.has(field.type)) {
-        // Add this field to the possibilities for the column, using the guessed
-        // formats
-        fieldFormats.set(field.name, colTypeGuess.types.get(field.type));
-      }
-    });
-
-    // Extend the column type guess with the "fields" field
-    let typeGuessesAndFields = Object.assign({}, colTypeGuess);
-    typeGuessesAndFields.fields = fieldFormats;
-    result.push(typeGuessesAndFields);
-  });
-
-  return result;
-};
 
 // Return the unique values in each column in the range. Return no more than
 // maxValues values for any given column. Return format is an array, one entry

lib/importer/src/dudk/backend.test.js

@@ -603,7 +603,7 @@ test('guessing types', () => {
   const dataRange = {
     sheet: 'Sheet1',
     start: { row: 1, column: 0 },
-    end: { row: 6, column: 9 }
+    end: { row: 6, column: 8 }
   };
 
   const guesses = backend.SessionGuessTypes(sid, dataRange);

lib/importer/src/dudk/types/attribute-types.js

@@ -111,7 +111,7 @@ exports.supportedTypes = new Map([
   ["boolean", {displayName: "Y/N", description: "A 'Yes' or 'No'"}],
   ["date", {displayName: "Date", description: "A full date (eg Year, Month and Day number)",
             formats: new Map([
-              ["native", {displayName: "Spreadsheet", description: "A date cell in a spreadsheet"}],
+              ["native", {displayName: "Spreadsheet Date", description: "A date cell in a spreadsheet"}],
               ["ymd", {displayName: "Y/M/D", description: "A date written in Year, Month, Day order"}],
               ["ydm", {displayName: "Y/D/M", description: "A date written in Year, Day, Month order"}],
               ["dmy", {displayName: "D/M/Y", description: "A date written in Day, Month, Year order"}],

lib/importer/src/dudk/types/guesser.js

@@ -35,6 +35,18 @@ const TYPE_PATTERNS = [
 
 exports.TYPE_PATTERNS = TYPE_PATTERNS;
 
+// Does the format string look like a date?  See "Common Date-Time Formats" at
+// https://docs.sheetjs.com/docs/csf/features/dates for the source of these
+// patterns.
+const formatStringPartRE = new RegExp("[^a-zA-Z]+");
+// Note that m and mm can also represent minutes in time formats
+const formatStringDateParts = ["yy","yyyy","m","mm","mmm","mmmm","mmmmm","d","dd","ddd","dddd"];
+function isDateFormat(fmt) {
+  const parts = fmt.split(formatStringPartRE);
+  // Is every token in the format string valid for a date format?
+  return parts.every((part) => formatStringDateParts.includes(part));
+}
+
 // Given a sheet.js field, return two values. First is a list of possible types
 // for it, as a map from type names to either:
 
@@ -64,6 +76,13 @@ exports.ListPossibleTypes = (field) => {
   case "z": // Empty cell
     // Either way, we can't narrow the set of possible types, but it's a blank cell
     return [false, true];
+  case "d": // Date or time or both
+    if(isDateFormat(field.z)) {
+      possibleTypes.set("date", ["native"]);
+    }
+    // FIXME: Add cases for time formats, eg "h:mm", when we support them
+    return [possibleTypes, isBlank];
+    break;
   case "s":
     // String
     if(isBlank) {

lib/importer/src/functions.js

@@ -56,30 +56,54 @@ const importerErrorMappingData = (error, key) => {
 
 //--------------------------------------------------------------------
 // As long as column->field mappings have been set up, this function will return
-// a list of possible formats for a specified column (by index). The return
-// value will be false if the column is mapped to a field whose type does not
-// require formats, otherwise it will be a list of objects with 'name' (internal
-// code), 'displayName' (human-facing name) and 'description' (longer description) fields.
+// an array indexed on column, with each element being list of possible formats
+// for that column. The array element for a column will be false if the column is mapped to a
+// field whose type does not require formats, otherwise it will be a list of
+// objects with 'name' (internal code), 'displayName' (human-facing name) and
+// 'description' (longer description) fields.
 // --------------------------------------------------------------------
-const importerPossibleColumnFormats = (data, index) => {
+const importerPossibleColumnFormats = (data) => {
+    // FIXME: This should probably move into the backend, as a SessionSuggestFormats function
     const session_data = data[IMPORTER_SESSION_KEY];
     const session = new session_lib.Session(session_data);
     const supportedTypes = backend_lib.SessionGetSupportedTypes(session.backendSid);
     const mappings = backend_lib.SessionGetMappingRules(session.backendSid);
-    const columnField = mappings[index];
-    const columnDef = session.fields.find((field) => field.name == columnField);
-    const columnTypeName = columnDef.type;
-    const columnType = supportedTypes.get(columnTypeName);
-    if (columnType.formats) {
-        const options = Array.from(columnType.formats.entries()).map((fmtEntry) => ({
-            name: fmtEntry[0],
-            displayName: fmtEntry[1].displayName,
-            description: fmtEntry[1].description
-        }));
-        return options;
-    } else {
-        return false;
-    }
+    const hRange = backend_lib.SessionGetHeaderRange(session.backendSid, session.sheet)
+    const fRange = backend_lib.SessionGetFooterRange(session.backendSid, session.sheet)
+    const dataRange = backend_lib.SessionSuggestDataRange(session.backendSid, hRange, fRange);
+    const typeGuesses = backend_lib.SessionGuessTypes(session.backendSid, dataRange);
+
+    return Object.entries(mappings).map((entry) => {
+        const [colIndex, columnField] = entry;
+        if (!columnField) return false; // Skip unmapped columns
+        const columnGuesses = typeGuesses[colIndex];
+        const columnDef = session.fields.find((field) => field.name == columnField);
+        const columnTypeName = columnDef.type;
+        const columnType = supportedTypes.get(columnTypeName);
+        let likelyFormats = new Set(); // Names of formats this column is likely to have, according to the guesser
+        if (columnGuesses.types && columnGuesses.types.has(columnTypeName)) {
+            const guessedFormats = columnGuesses.types.get(columnTypeName);
+            if(guessedFormats && columnType.formats) {
+                likelyFormats = new Set(Array.from(columnType.formats.keys()).filter((fmtName) => {
+                    return guessedFormats.includes(fmtName);
+                }));
+            }
+        }
+
+        if (columnType.formats) {
+            const options = Array.from(columnType.formats.entries()).map((fmtEntry) => {
+                return {
+                    name: fmtEntry[0],
+                    displayName: fmtEntry[1].displayName,
+                    description: fmtEntry[1].description,
+                    likely: likelyFormats.has(fmtEntry[0])
+                };
+            });
+            return options;
+        } else {
+            return false;
+        }
+    });
 }
 
 //--------------------------------------------------------------------

lib/importer/templates/format.html

@@ -13,7 +13,10 @@
     <div class="govuk-grid-column-full">
         <h1 class="govuk-heading-l">Identify columns</h1>
         <p class="govuk-body">
-            Select a format for any fields that need it. FIXME: Write better text here...
+            One or more columns may contain data represented in one of a number
+            of formats, such as dates. Please select the format your file uses
+            in the table below. Formats that appear to match the data found in
+            your file are marked with an asterisk (*).
         </p>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>

prototypes/basic/app/views/format.html

@@ -46,7 +46,10 @@
     <div class="govuk-grid-column-full">
         <h1 class="govuk-heading-l">Choose formats</h1>
         <p class="govuk-body">
-            Select a format for any fields that need it. FIXME: Write better text here...
+            One or more columns may contain data represented in one of a number
+            of formats, such as dates. Please select the format your file uses
+            in the table below. Formats that appear to match the data found in
+            your file are marked with an asterisk (*).
         </p>
 
         <h2 class="govuk-heading-m">{{sheet}}</h2>