User defined tables (#345)

* merge changes from OwainJones-patch-1 into new branch recreated from main (#317)

* Update backend.js

Updating for tables

* Added for table testing

* Some minor updates to testing

* Some updates to tables_test and added a metadata file

* Installed our locally "vendored" patched copy of sheet.js and used it to get defined tables

Excel.js only offered an async API to load files, so we couldn't use it :'-(

# Conflicts:
#	lib/importer/package.json
#	lib/importer/src/dudk/backend.js

* Licence compliance requirements for SheetJS fork

* Added html and nkj files, as well as the associated functions in function.js and sheet.js for selecting user defined tables.

* Added njk and html files as well as functions in functions.js and sheets.js to select and preview tables.

* A range of edits to fix errros that campe up when testing the table selector and to extract the header row automatically

* Several tweaks to code to get table previews to work.

* Add files via upload

* Delete backend.js

* Delete sheets.js

* Delete functions.js

* Add files via upload

* Add files via upload

* Add files via upload

* Add files via upload

---------

Co-authored-by: Alaric Snell-Pym <alaric@register-dynamics.co.uk>

* Delete table_selector.njk

* Updates for tables attempting to pass multiple urls to route to

* Minor updates

* Edits to ensure that header identification in tables is working correctly, and to ensure we preserve the new header range.

* Added a get and set table function, updated index.js, and various other functions to accomodate tables into the header and footer pages

* Added some tweaks to review page to accomodate tables,
moved the check for footer rows in SessionPerformMappingJob from if(row) to its
parent for loop, as we were having an issue where an empty row in the table footer
(i.e. the row immediately below the table) was giving an error.

* Tidied up code removing several console.debug lines that were used for debugging

* Some updates to how we handle table header and use it in the final mapData step so that we can import data without visible headers

* Quick comit to ensure any minor local changes are accounted for

* Changes to fix table preview for no table and to fix merge issues

---------

Co-authored-by: Alaric Snell-Pym <alaric@register-dynamics.co.uk>

Author: OwainJones

fixtures/tables_test.xlsx

@@ -0,0 +1,9 @@
+Table,Sheet,Name,Range ,Columns
+table1,Standard,Soldier,A1:I21,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table2,Standard,Scout,A23:I43,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table3,Standard,Scoundrel,A45:I65,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table4,Jedi,Guardian,A1:K21,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+table5,Jedi ,Sentinel,A23:K23,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+table6,Jedi ,Consular,A45:K65,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+,,,,
+There is also a third sheet called 'No tables' with no tables in it just a block of data,,,,

fixtures/tables_test_metadata.csv

@@ -7,15 +7,18 @@ window.addEventListener("load", () => {
             return;
         }
 
-        let index = parseInt(elem.dataset.previewIndex ?? -1);
-        if (index < 0) {
-            return;
-        }
-
         previews.forEach((p) => {
             p.classList.remove("selected");
             p.ariaHidden = true;
         });
+
+        let index = parseInt(elem.dataset.previewIndex ?? -1);      
+
+        if (index < 0) {
+            return;
+        }
+
+        
         previews[index].classList.add("selected");
         previews[index].ariaHidden = false;
     }

fixtures/tables_test_metadata.xlsx

@@ -15,6 +15,11 @@
       "path": "/templates/select_sheet.html",
       "type": "nunjucks"
     },
+    {
+      "name": "Select a table",
+      "path": "/templates/select_table.html",
+      "type": "nunjucks"
+    },
     {
       "name": "Select header rows",
       "path": "/templates/select_header_row.html",
@@ -64,6 +69,10 @@
       "macroName": "importerSheetSelector",
       "importFrom": "importer/macros/sheet_selector.njk"
     },
+    {
+      "macroName": "importerTableSelector",
+      "importFrom": "importer/macros/table_selector.njk"
+    },
     {
       "macroName": "importerFieldMapper",
       "importFrom": "importer/macros/field_mapper.njk"

lib/importer/assets/js/sheet_selector.js

@@ -0,0 +1,83 @@
+
+{#
+    importerTableSelector generates a radio button list which allows the
+    user to choose which table is to be used (if any). 
+
+    The list of table names is retrieved from the current spreadsheet
+    being uploaded.
+
+    It accepts a data object which is taken from the prototype kit's
+    session data which is made available on every page, and contains the
+    data submitted from forms to the backend, and also the current
+    data import session.
+
+    legend is the text that should be used for the legend part of the
+    radio buttons, if none is supplied, then a legend is not added.
+#}
+{% from "importer/macros/table_view.njk" import tableView %}
+
+{% macro importerTableSelector(data, legend) %}
+  {% set selectedTable = data['importer.session'].table %}
+  {% set tables = importTablePreview(data) %}
+  {% set tableRowIndex = tables.length + 2 %}
+  {% set error = importerError(data) %}
+
+  <div class="govuk-form {% if error %}govuk-form-group--error{% endif %}">
+    <div class="govuk-form-group ">
+      {% if legend %}
+      <legend class="govuk-fieldset__legend govuk-fieldset__legend--l">
+        <h1 class="govuk-fieldset__heading">
+          {{ legend }}
+        </h1>
+      </legend>
+      {% endif %}
+
+      {% if error %}
+      <p id="upload-error" class="govuk-error-message">
+        <span class="govuk-visually-hidden">Error:</span> {{ error.text }}
+      </p>
+      {% endif %}
+
+      <div class="govuk-radios rd-sheet-preview" data-module="govuk-radios">
+        {% for table in tables %}
+        <div class="govuk-radios__item">
+          <input class="govuk-radios__input" id="{{table.name}}" name="table" type="radio" value="{{table.name}}" {% if selectedTable==table.name %}checked="checked"{% endif %} data-preview-index="{{loop.index0}}">
+          <label class="govuk-label govuk-radios__label" for="{{table.name}}">
+          {{table.name}}
+          </label>
+        </div> <!-- .govuk-radios__item  -->
+        {% endfor %}
+        <div class="govuk-radios__divider">or</div>
+        <div class="govuk-radios__item">
+        <input class="govuk-radios__input" id="no_table" name="table" type="radio" value="no_table" data-preview-index=-1>
+        <label class="govuk-label govuk-radios__label" for="no_table">
+          I don't want to use a predefined table
+        </label>
+      </div> <!-- .govuk-radios__item  -->
+    </div>
+
+    <div class="rd-sheet-selector-previews">
+      {% for table in tables %}
+        <div class="hidden">
+          {% if table.data.rows == null %}
+          <div class="govuk-body">
+            Table '{{ table.name }}' is empty
+            </div>
+          {% else %}
+            {% set caption = importerGetTableCaption(data, "First", 10, sheet.name, table.name) %}
+            {{ tableView({
+              caption: caption,
+              showHeaders: true,
+              headers: table.data.headers,
+              showRowNumbers: false,
+              rows: table.data.rows,
+              moreRowsCount: 0
+            } )}}
+
+            
+          {% endif %}
+        </div>
+      {% endfor %}
+    </div>
+  </div>
+{% endmacro %}

lib/importer/govuk-prototype-kit.config.json

@@ -20,16 +20,15 @@
   "license": "ISC",
   "description": "The Data Upload Design Kit makes it easier for users to upload data about many things at once by uploading a file without the need for hard-coded Excel spreadsheet templates. Designed to work with the GOV.UK Prototype Kit, it makes it easier for service designers and developers to work with spreadsheets.",
   "dependencies": {
-    "multer": "2.0.1",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
-    "fs-extra": "11.3.0"
+    "fs-extra": "11.3.0",
+    "multer": "2.0.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.32.0",
     "eslint": "^9.32.0",
     "eslint-plugin-jest": "^29.0.1",
     "globals": "^16.3.0",
-    "mock-fs": "^5.5.0",
-    "jest": "^30.0.5"
+    "jest": "^30.0.5",
+    "mock-fs": "^5.5.0"
   }
 }

lib/importer/nunjucks/importer/macros/table_selector.njk

@@ -1,4 +1,4 @@
-const xlsx = require("xlsx");
+const xlsx = require("./sheetjs/xlsx");
 const crypto = require("crypto");
 const assert = require('node:assert').strict;
 const attributeTypes = require('./types/attribute-types');
@@ -17,6 +17,8 @@ let sessionStore = function () {
 }();
 
 let dimensionsCache = new Map();
+let tablesCache = new Map();
+let tableRangesCache = new Map();
 
 // Maximum number of rows/records to return in one go. While we're all
 // in-memory, this merely restricts how much we allocate in one go for a shallow
@@ -45,6 +47,8 @@ exports.CreateSession = () => {
 /// Session Interface
 ///
 
+
+
 // Set the filename of the file for an import session.
 exports.SessionSetFile = (sid, filename) => {
   const wb = xlsx.readFile(filename, { dense: true, cellStyles: true, cellDates: true, raw: true });
@@ -75,6 +79,11 @@ exports.SessionGetSheets = (sid) => {
   return session.sheetNames
 };
 
+exports.SessionGetTables = (sid, sheetName) => {
+  let session = sessionStore.get(sid);
+  return session.tables[sheetName]
+};
+
 exports.SessionSetHeaderRange = (sid, range) => {
   assert(range != null, "Null range passed to SessionSetHeaderRange");
   assert(range.sheet != null, "Range with null sheet passed to SessionSetHeaderRange");
@@ -159,6 +168,16 @@ exports.SessionGetInputDimensions = (sid) => {
   return getDimensions(sid);
 };
 
+exports.SessionGetInputTables = (sid) => {
+  assert(sessionStore.has(sid));
+  return getTables(sid);
+};
+
+exports.SessionGetInputTableRanges = (sid, sheet) => {
+  assert(sessionStore.has(sid));
+  return getTableRanges(sid, sheet);
+};
+
 function randInRange(min, max) {
   return Math.floor(Math.random() * (max - min + 1) + min);
 }
@@ -375,6 +394,69 @@ function cellsToSamples(row) {
   return row.map(cellToSample);
 }
 
+function getTables(sid) {
+  let session = sessionStore.get(sid);
+  let sheetTables = new Map();
+
+  let cached = tablesCache.get(sid);
+  if (cached) {
+    return cached
+  }
+  Object.keys(session.wb.Sheets).forEach((sheetName) => {
+    const sheet = session.wb.Sheets[sheetName];
+
+    if(sheet["!tables"]) {
+      const tables = sheet["!tables"];
+      sheetTables.set(sheetName, {
+        tableCount: tables.length,
+        tableNames: tables.map((t) => t.name)
+      })
+    }
+  })
+
+  tablesCache.set(sid, { sheetTables: sheetTables })
+
+  return sheetTables;
+}
+
+function decodeRange(rangeStr) {
+  const parts = xlsx.utils.decode_range(rangeStr);
+  return {
+    start: {
+      row: parts.s.r,
+      column: parts.s.c
+    },
+    end: {
+      row: parts.e.r,
+      column: parts.e.c
+    }
+  };
+}
+
+function getTableRanges(sid, sheetName) {
+  let session = sessionStore.get(sid);
+  let tableRanges = new Map();
+
+  //let cached = tableRangesCache.get(sid);
+  //if (cached) {
+  //  return cached
+  //}
+  const sheet = session.wb.Sheets[sheetName];
+  if (sheet["!tables"]) {
+    sheet["!tables"].forEach((table) => {
+      tableRanges.set(table.name, {
+        displayName: table.displayname,
+        range: decodeRange(table.ref),
+        columns: table.columns
+      });
+    });
+  }
+
+  //tableRangesCache.set(sid, tableRanges)
+
+  return tableRanges;
+}
+
 // Returns a sample of rows in a range. range is of the form {sheet: 'Foo', start:{row: X, column: Y}, end:{row: X, column: Y}}.
 
 // Returns three arrays - one with startCount rows from the top of the range,
@@ -635,6 +717,8 @@ exports.SessionDelete = (sid) => {
   assert(sessionStore.get(sid), `No such session ${sid} when deleting session`);
   sessionStore.delete(sid);
   dimensionsCache.delete(sid);
+  tablesCache.delete(sid);
+  tableRangesCache.delete(sid);
 }
 
 
@@ -701,13 +785,13 @@ exports.SessionPerformMappingJob = (sid, range, mapping, includeErrorRow = false
     let row = getMergedRow(data, merges, rowIdx);
     let recordCells = {};
     let foundSomeValues = false;
+    // Ensure we exclude footer rows from the validation as they may not
+    // have values in them.
+    if (rowIdx == range.end.row && footerRange) {
+      continue;
+    }
 
     if (row) {
-      // Ensure we exclude footer rows from the validation as they may not
-      // have the same number of columns as the header row.
-      if (rowIdx == range.end.row && footerRange) {
-        continue;
-      }
 
       if (row.length < expectedColumnCount) {
         warnings.push({row: rowIdx, field: "", type: errorTypes.ValidationError.ShortRow({ expected: expectedColumnCount, actual: row.length})});