User defined tables

fixtures/tables_test_metadata.csv

@@ -0,0 +1,9 @@
+Table,Sheet,Name,Range ,Columns
+table1,Standard,Soldier,A1:I21,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table2,Standard,Scout,A23:I43,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table3,Standard,Scoundrel,A45:I65,Level; Vitality; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat
+table4,Jedi,Guardian,A1:K21,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+table5,Jedi ,Sentinel,A23:K23,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+table6,Jedi ,Consular,A45:K65,Level; Vitality; Force Points; Attack; Defence; Fortitude; Reflex; Will; Feats granted; Feat; Force Powers
+,,,,
+There is also a third sheet called 'No tables' with no tables in it just a block of data,,,,

lib/importer/assets/js/sheet_selector.js

@@ -7,15 +7,18 @@ window.addEventListener("load", () => {
             return;
         }
 
-        let index = parseInt(elem.dataset.previewIndex ?? -1);
-        if (index < 0) {
-            return;
-        }
-
         previews.forEach((p) => {
             p.classList.remove("selected");
             p.ariaHidden = true;
         });
+
+        let index = parseInt(elem.dataset.previewIndex ?? -1);      
+
+        if (index < 0) {
+            return;
+        }
+
+
         previews[index].classList.add("selected");
         previews[index].ariaHidden = false;
     }

lib/importer/govuk-prototype-kit.config.json

@@ -15,6 +15,11 @@
       "path": "/templates/select_sheet.html",
       "type": "nunjucks"
     },
+    {
+      "name": "Select a table",
+      "path": "/templates/select_table.html",
+      "type": "nunjucks"
+    },
     {
       "name": "Select header rows",
       "path": "/templates/select_header_row.html",
@@ -64,6 +69,10 @@
       "macroName": "importerSheetSelector",
       "importFrom": "importer/macros/sheet_selector.njk"
     },
+    {
+      "macroName": "importerTableSelector",
+      "importFrom": "importer/macros/table_selector.njk"
+    },
     {
       "macroName": "importerFieldMapper",
       "importFrom": "importer/macros/field_mapper.njk"

lib/importer/nunjucks/importer/macros/table_selector.njk

@@ -0,0 +1,83 @@
+
+{#
+    importerTableSelector generates a radio button list which allows the
+    user to choose which table is to be used (if any). 
+
+    The list of table names is retrieved from the current spreadsheet
+    being uploaded.
+
+    It accepts a data object which is taken from the prototype kit's
+    session data which is made available on every page, and contains the
+    data submitted from forms to the backend, and also the current
+    data import session.
+
+    legend is the text that should be used for the legend part of the
+    radio buttons, if none is supplied, then a legend is not added.
+#}
+{% from "importer/macros/table_view.njk" import tableView %}
+
+{% macro importerTableSelector(data, legend) %}
+  {% set selectedTable = data['importer.session'].table %}
+  {% set tables = importTablePreview(data) %}
+  {% set tableRowIndex = tables.length + 2 %}
+  {% set error = importerError(data) %}
+
+  <div class="govuk-form {% if error %}govuk-form-group--error{% endif %}">
+    <div class="govuk-form-group ">
+      {% if legend %}
+      <legend class="govuk-fieldset__legend govuk-fieldset__legend--l">
+        <h1 class="govuk-fieldset__heading">
+          {{ legend }}
+        </h1>
+      </legend>
+      {% endif %}
+
+      {% if error %}
+      <p id="upload-error" class="govuk-error-message">
+        <span class="govuk-visually-hidden">Error:</span> {{ error.text }}
+      </p>
+      {% endif %}
+
+      <div class="govuk-radios rd-sheet-preview" data-module="govuk-radios">
+        {% for table in tables %}
+        <div class="govuk-radios__item">
+          <input class="govuk-radios__input" id="{{table.name}}" name="table" type="radio" value="{{table.name}}" {% if selectedTable==table.name %}checked="checked"{% endif %} data-preview-index="{{loop.index0}}">
+          <label class="govuk-label govuk-radios__label" for="{{table.name}}">
+          {{table.name}}
+          </label>
+        </div> <!-- .govuk-radios__item  -->
+        {% endfor %}
+        <div class="govuk-radios__divider">or</div>
+        <div class="govuk-radios__item">
+        <input class="govuk-radios__input" id="no_table" name="table" type="radio" value="no_table" data-preview-index=-1>
+        <label class="govuk-label govuk-radios__label" for="no_table">
+          I don't want to use a predefined table
+        </label>
+      </div> <!-- .govuk-radios__item  -->
+    </div>
+
+    <div class="rd-sheet-selector-previews">
+      {% for table in tables %}
+        <div class="hidden">
+          {% if table.data.rows == null %}
+          <div class="govuk-body">
+            Table '{{ table.name }}' is empty
+            </div>
+          {% else %}
+            {% set caption = importerGetTableCaption(data, "First", 10, sheet.name, table.name) %}
+            {{ tableView({
+              caption: caption,
+              showHeaders: true,
+              headers: table.data.headers,
+              showRowNumbers: false,
+              rows: table.data.rows,
+              moreRowsCount: 0
+            } )}}
+
+
+          {% endif %}
+        </div>
+      {% endfor %}
+    </div>
+  </div>
+{% endmacro %}

lib/importer/package.json

@@ -20,16 +20,15 @@
   "license": "ISC",
   "description": "The Data Upload Design Kit makes it easier for users to upload data about many things at once by uploading a file without the need for hard-coded Excel spreadsheet templates. Designed to work with the GOV.UK Prototype Kit, it makes it easier for service designers and developers to work with spreadsheets.",
   "dependencies": {
-    "multer": "2.0.1",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
-    "fs-extra": "11.3.0"
+    "fs-extra": "11.3.0",
+    "multer": "2.0.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.32.0",
     "eslint": "^9.32.0",
     "eslint-plugin-jest": "^29.0.1",
     "globals": "^16.3.0",
-    "mock-fs": "^5.5.0",
-    "jest": "^30.0.5"
+    "jest": "^30.0.5",
+    "mock-fs": "^5.5.0"
   }
 }

lib/importer/src/dudk/backend.js

@@ -1,4 +1,4 @@
-const xlsx = require("xlsx");
+const xlsx = require("./sheetjs/xlsx");
 const crypto = require("crypto");
 const assert = require('node:assert').strict;
 const attributeTypes = require('./types/attribute-types');
@@ -17,6 +17,8 @@ let sessionStore = function () {
 }();
 
 let dimensionsCache = new Map();
+let tablesCache = new Map();
+let tableRangesCache = new Map();
 
 // Maximum number of rows/records to return in one go. While we're all
 // in-memory, this merely restricts how much we allocate in one go for a shallow
@@ -45,6 +47,8 @@ exports.CreateSession = () => {
 /// Session Interface
 ///
 
+
+
 // Set the filename of the file for an import session.
 exports.SessionSetFile = (sid, filename) => {
   const wb = xlsx.readFile(filename, { dense: true, cellStyles: true, cellDates: true, raw: true });
@@ -75,6 +79,11 @@ exports.SessionGetSheets = (sid) => {
   return session.sheetNames
 };
 
+exports.SessionGetTables = (sid, sheetName) => {
+  let session = sessionStore.get(sid);
+  return session.tables[sheetName]
+};
+
 exports.SessionSetHeaderRange = (sid, range) => {
   assert(range != null, "Null range passed to SessionSetHeaderRange");
   assert(range.sheet != null, "Range with null sheet passed to SessionSetHeaderRange");
@@ -159,6 +168,16 @@ exports.SessionGetInputDimensions = (sid) => {
   return getDimensions(sid);
 };
 
+exports.SessionGetInputTables = (sid) => {
+  assert(sessionStore.has(sid));
+  return getTables(sid);
+};
+
+exports.SessionGetInputTableRanges = (sid, sheet) => {
+  assert(sessionStore.has(sid));
+  return getTableRanges(sid, sheet);
+};
+
 function randInRange(min, max) {
   return Math.floor(Math.random() * (max - min + 1) + min);
 }
@@ -375,6 +394,69 @@ function cellsToSamples(row) {
   return row.map(cellToSample);
 }
 
+function getTables(sid) {
+  let session = sessionStore.get(sid);
+  let sheetTables = new Map();
+
+  let cached = tablesCache.get(sid);
+  if (cached) {
+    return cached
+  }
+  Object.keys(session.wb.Sheets).forEach((sheetName) => {
+    const sheet = session.wb.Sheets[sheetName];
+
+    if(sheet["!tables"]) {
+      const tables = sheet["!tables"];
+      sheetTables.set(sheetName, {
+        tableCount: tables.length,
+        tableNames: tables.map((t) => t.name)
+      })
+    }
+  })
+
+  tablesCache.set(sid, { sheetTables: sheetTables })
+
+  return sheetTables;
+}
+
+function decodeRange(rangeStr) {
+  const parts = xlsx.utils.decode_range(rangeStr);
+  return {
+    start: {
+      row: parts.s.r,
+      column: parts.s.c
+    },
+    end: {
+      row: parts.e.r,
+      column: parts.e.c
+    }
+  };
+}
+
+function getTableRanges(sid, sheetName) {
+  let session = sessionStore.get(sid);
+  let tableRanges = new Map();
+
+  //let cached = tableRangesCache.get(sid);
+  //if (cached) {
+  //  return cached
+  //}
+  const sheet = session.wb.Sheets[sheetName];
+  if (sheet["!tables"]) {
+    sheet["!tables"].forEach((table) => {
+      tableRanges.set(table.name, {
+        displayName: table.displayname,
+        range: decodeRange(table.ref),
+        columns: table.columns
+      });
+    });
+  }
+
+  //tableRangesCache.set(sid, tableRanges)
+
+  return tableRanges;
+}
+
 // Returns a sample of rows in a range. range is of the form {sheet: 'Foo', start:{row: X, column: Y}, end:{row: X, column: Y}}.
 
 // Returns three arrays - one with startCount rows from the top of the range,
@@ -635,6 +717,8 @@ exports.SessionDelete = (sid) => {
   assert(sessionStore.get(sid), `No such session ${sid} when deleting session`);
   sessionStore.delete(sid);
   dimensionsCache.delete(sid);
+  tablesCache.delete(sid);
+  tableRangesCache.delete(sid);
 }
 
 
@@ -701,13 +785,13 @@ exports.SessionPerformMappingJob = (sid, range, mapping, includeErrorRow = false
     let row = getMergedRow(data, merges, rowIdx);
     let recordCells = {};
     let foundSomeValues = false;
+    // Ensure we exclude footer rows from the validation as they may not
+    // have values in them.
+    if (rowIdx == range.end.row && footerRange) {
+      continue;
+    }
 
     if (row) {
-      // Ensure we exclude footer rows from the validation as they may not
-      // have the same number of columns as the header row.
-      if (rowIdx == range.end.row && footerRange) {
-        continue;
-      }
 
       if (row.length < expectedColumnCount) {
         warnings.push({row: rowIdx, field: "", type: errorTypes.ValidationError.ShortRow({ expected: expectedColumnCount, actual: row.length})});