TD-5154 Implements basic pre-process logic and view and adds validation to exported sheet

Author: kevwhitt-hee

DigitalLearningSolutions.Data/Models/Frameworks/Import/CompetencyTableRow.cs

@@ -1,6 +1,8 @@
 namespace DigitalLearningSolutions.Data.Models.Frameworks.Import
 {
     using ClosedXML.Excel;
+    using Org.BouncyCastle.Asn1.X509;
+
     public enum RowStatus
     {
         NotYetProcessed,
@@ -18,16 +20,16 @@ public CompetencyTableRow(IXLTable table, IXLRangeRow row)
         {
             string? FindFieldValue(string name)
             {
-                var colNumber = table.FindColumn(col => col.FirstCell().Value.ToString()?.ToLower() == name).ColumnNumber();
+                var colNumber = table.FindColumn(col => col.FirstCell().Value.ToString()?.ToLower() == name.ToLower()).ColumnNumber();
                 return row.Cell(colNumber).GetValue<string?>();
             }
 
             RowNumber = row.RowNumber();
-            id = int.Parse(FindFieldValue("ID"));
+            id = row.Cell(1).GetValue<int?>();
             CompetencyGroup = FindFieldValue("CompetencyGroup");
             Competency = FindFieldValue("Competency");
             CompetencyDescription = FindFieldValue("CompetencyDescription");
-            GroupDescription = FindFieldValue("CompetencyGroupDescription");
+            GroupDescription = FindFieldValue("GroupDescription");
             FlagsCsv = FindFieldValue("FlagsCSV");
             RowStatus = RowStatus.NotYetProcessed;
         }

DigitalLearningSolutions.Data/Models/Frameworks/Import/ImportCompetenciesResult.cs

@@ -8,10 +8,11 @@ public class ImportCompetenciesResult
     {
         public enum ErrorReason
         {
+            InvalidId,
             MissingCompetencyName,
-            TooLongCompetencyGroupName,
             TooLongCompetencyName,
-            AlreadyExists
+            TooLongCompetencyGroupName,
+            InvalidAlwaysShowDescription
         }
         public ImportCompetenciesResult() { }
 
@@ -20,16 +21,18 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
         )
         {
             ProcessedCount = competencyTableRows.Count;
-            CompetenciesInsertedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
-            CompetencyGroupsInsertedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
+            CompetencyAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
+            GroupAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
             SkippedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.Skipped);
             Errors = competencyTableRows.Where(dr => dr.Error.HasValue).Select(dr => (dr.RowNumber, dr.Error!.Value));
         }
 
         public IEnumerable<(int RowNumber, ErrorReason Reason)>? Errors { get; set; }
         public int ProcessedCount { get; set; }
-        public int CompetenciesInsertedCount { get; set; }
-        public int CompetencyGroupsInsertedCount { get; set; }
+        public int CompetencyAddedCount { get; set; }
+        public int CompetencyUpdatedCount { get; set; }
+        public int GroupAddedCount { get; set; }
+        public int GroupUpdatedCount { get; set; }
         public int SkippedCount { get; set; }
     }
 }

DigitalLearningSolutions.Web/Controllers/FrameworksController/ImportCompetencies.cs

@@ -1,29 +1,28 @@
 using ClosedXML.Excel;
 using DigitalLearningSolutions.Data.Exceptions;
-using DigitalLearningSolutions.Data.Migrations;
-using DigitalLearningSolutions.Data.Utilities;
 using DigitalLearningSolutions.Web.Helpers;
 using DigitalLearningSolutions.Web.Models;
 using DigitalLearningSolutions.Web.Services;
 using DigitalLearningSolutions.Web.ViewModels.Frameworks;
 using GDS.MultiPageFormData.Enums;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
+using System.IO;
 
 namespace DigitalLearningSolutions.Web.Controllers.FrameworksController
 {
     public partial class FrameworksController
     {
         [Route("/Framework/{frameworkId}/{tabname}/Import")]
-        public IActionResult ImportCompetencies(int frameworkId, string tabname)
+        public IActionResult ImportCompetencies(int frameworkId, string tabname, bool isNotBlank)
         {
             var adminId = GetAdminId();
             var userRole = frameworkService.GetAdminUserRoleForFrameworkId(adminId, frameworkId);
             if (userRole < 2)
                 return StatusCode(403);
             var model = new ImportCompetenciesViewModel()
             {
-                FrameworkId = frameworkId
+                FrameworkId = frameworkId,
+                IsNotBlank = isNotBlank
             };
             return View("Developer/ImportCompetencies", model);
         }
@@ -72,7 +71,25 @@ public IActionResult StartImport(ImportCompetenciesViewModel model, string tabna
         [Route("/Framework/{frameworkId}/{tabname}/ImportCompleted")]
         public IActionResult ImportCompleted()
         {
-            return View("Developer/ImportCompleted");
+            var data = GetBulkUploadData();
+            var uploadDir = Path.Combine(webHostEnvironment.WebRootPath, "Uploads\\");
+            var filePath = Path.Combine(uploadDir, data.CompetenciesFileName);
+            var workbook = new XLWorkbook(filePath);
+            try
+            {
+                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook);
+                var resultsModel = new ImportCompetenciesPreProcessViewModel(results);
+                data.CompetenciesToProcessCount = resultsModel.ToProcessCount;
+                data.CompetenciesToAddCount = resultsModel.CompetenciesToAddCount;
+                data.CompetenciesToUpdateCount = resultsModel.CompetenciesToUpdateCount;
+                setBulkUploadData(data);
+                return View("Developer/ImportCompleted", resultsModel);
+            }
+            catch (InvalidHeadersException)
+            {
+                FileHelper.DeleteFile(webHostEnvironment, data.CompetenciesFileName);
+                return View("ImportFailed");
+            }
         }
 
         private void setupBulkUploadData(int frameworkId, int adminUserID, string competenciessFileName, string tabName)

DigitalLearningSolutions.Web/Models/BulkCompetenciesData.cs

@@ -19,9 +19,9 @@ public BulkCompetenciesData(int frameworkId, int adminUserId, string competencie
         public string CompetenciesFileName { get; set; }
         public List<int> AssessmentQuestionIDs { get; set; }
         public int? AddAssessmentQuestionOption { get; set; }
-        public int ToProcessCount { get; set; }
-        public int ToAddCount { get; set; }
-        public int ToUpdateCount { get; set; }
+        public int CompetenciesToProcessCount { get; set; }
+        public int CompetenciesToAddCount { get; set; }
+        public int CompetenciesToUpdateCount { get; set; }
         public int LastRowProcessed { get; set; }
         public int SubtotalCompetenciesAdded { get; set; }
         public int SubtotalCompetenciesUpdated { get; set; }

DigitalLearningSolutions.Web/Models/BulkCompetenciesResult.cs

@@ -12,12 +12,14 @@ namespace DigitalLearningSolutions.Web.Services
     using DigitalLearningSolutions.Data.Exceptions;
     using DigitalLearningSolutions.Data.Helpers;
     using DigitalLearningSolutions.Data.Models.Frameworks.Import;
+    using DigitalLearningSolutions.Web.Models;
     using Microsoft.AspNetCore.Http;
 
     public interface IImportCompetenciesFromFileService
     {
         byte[] GetCompetencyFileForFramework(int frameworkId, bool v);
-        public ImportCompetenciesResult ProcessCompetenciesFromFile(IFormFile file, int adminUserId, int frameworkId);
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook);
+        public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId);
     }
     public class ImportCompetenciesFromFileService : IImportCompetenciesFromFileService
     {
@@ -30,17 +32,38 @@ IFrameworkService frameworkService
         {
             this.frameworkService = frameworkService;
         }
-        public ImportCompetenciesResult ProcessCompetenciesFromFile(IFormFile file, int adminUserId, int frameworkId)
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook)
+        {
+            var table = OpenCompetenciesTable(workbook);
+            var competencyRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
+            foreach (var competencyRow in competencyRows)
+            {
+                PreProcessCompetencyRow(competencyRow);
+            }
+            return new ImportCompetenciesResult(competencyRows);
+        }
+        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow)
+        {
+            if (competencyRow.id == null)
+            {
+                competencyRow.RowStatus = RowStatus.CompetencyInserted;
+            }
+            else
+            {
+                competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+            }
+        }
+        public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId)
         {
             int maxFrameworkCompetencyId = frameworkService.GetMaxFrameworkCompetencyID();
             int maxFrameworkCompetencyGroupId = frameworkService.GetMaxFrameworkCompetencyGroupID();
-            var table = OpenCompetenciesTable(file);
+            var table = OpenCompetenciesTable(workbook);
             return ProcessCompetenciesTable(table, adminUserId, frameworkId, maxFrameworkCompetencyId, maxFrameworkCompetencyGroupId);
         }
-        internal IXLTable OpenCompetenciesTable(IFormFile file)
+        internal IXLTable OpenCompetenciesTable(IXLWorkbook workbook)
         {
-            var workbook = new XLWorkbook(file.OpenReadStream());
             var worksheet = workbook.Worksheet(1);
+            worksheet.Columns(1, 15).Unhide();
             if (worksheet.Tables.Count() == 0)
             {
                 throw new InvalidHeadersException();
@@ -112,11 +135,15 @@ private static bool ValidateHeaders(IXLTable table)
         {
             var expectedHeaders = new List<string>
             {
-                "competency group",
-                "competency name",
-                "competency description"
+                "ID",
+                "CompetencyGroup",
+                "GroupDescription",
+                "Competency",
+                "CompetencyDescription",
+                "AlwaysShowDescription",
+                "FlagsCSV"
             }.OrderBy(x => x);
-            var actualHeaders = table.Fields.Select(x => x.Name.ToLower()).OrderBy(x => x);
+            var actualHeaders = table.Fields.Select(x => x.Name).OrderBy(x => x);
             return actualHeaders.SequenceEqual(expectedHeaders);
         }
 
@@ -128,6 +155,13 @@ public byte[] GetCompetencyFileForFramework(int frameworkId, bool blank)
             {
                 ClosedXmlHelper.HideWorkSheetColumn(workbook, "ID");
             }
+            var options = new List<string> { "TRUE", "FALSE" };
+            ClosedXmlHelper.AddValidationListToWorksheetColumn(workbook, 6, options);
+            var rowCount = workbook.Worksheet(1).RangeUsed().RowCount();
+            ClosedXmlHelper.AddValidationRangeToWorksheetColumn(workbook, 1, 1, rowCount, 1);
+            // Calculate the workbook
+            workbook.CalculateMode = XLCalculateMode.Auto;
+            workbook.RecalculateAllFormulas();
             using var stream = new MemoryStream();
             workbook.SaveAs(stream);
             return stream.ToArray();

DigitalLearningSolutions.Web/Services/ImportCompetenciesFromFileService.cs

@@ -2,12 +2,13 @@
 using System;
 using DigitalLearningSolutions.Web.Models;
 using System.Linq;
+using DigitalLearningSolutions.Data.Models.Frameworks.Import;
 
 namespace DigitalLearningSolutions.Web.ViewModels.Frameworks
 {
     public class ImportCompetenciesPreProcessViewModel
     {
-        public ImportCompetenciesPreProcessViewModel(BulkCompetenciesResult bulkCompetenciesResult)
+        public ImportCompetenciesPreProcessViewModel(ImportCompetenciesResult bulkCompetenciesResult)
         {
             ToProcessCount = bulkCompetenciesResult.ProcessedCount;
             CompetenciesToAddCount = bulkCompetenciesResult.CompetencyAddedCount;
@@ -25,20 +26,21 @@ public ImportCompetenciesPreProcessViewModel(BulkCompetenciesResult bulkCompeten
         public int CompetencyGroupsToAddCount { get; set; }
         public int CompetencyGroupsToUpdateCount { get; set; }
         public int ToUpdateOrSkipCount { get; set; }
+        public string? CompetenciesFileName { get; set; }
 
-        private static string MapReasonToErrorMessage(BulkCompetenciesResult.ErrorReason reason)
+        private static string MapReasonToErrorMessage(ImportCompetenciesResult.ErrorReason reason)
         {
             return reason switch
             {
-                BulkCompetenciesResult.ErrorReason.TooLongCompetencyGroupName =>
+                ImportCompetenciesResult.ErrorReason.TooLongCompetencyGroupName =>
                     "Group name must be 255 characters or less.",
-                BulkCompetenciesResult.ErrorReason.MissingCompetencyName =>
+                ImportCompetenciesResult.ErrorReason.MissingCompetencyName =>
                     "Competency is blank. Competency is a required field and cannot be left blank",
-                BulkCompetenciesResult.ErrorReason.InvalidId =>
+                ImportCompetenciesResult.ErrorReason.InvalidId =>
                     "The ID provided does not match a Competency ID in this Framework",
-                BulkCompetenciesResult.ErrorReason.TooLongCompetencyName =>
+                ImportCompetenciesResult.ErrorReason.TooLongCompetencyName =>
                     "Competency must be 255 characters or less.",
-                BulkCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription =>
+                ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription =>
                     "Always show description is invalid. The Always show description  field must contain 'TRUE' or 'FALSE'",
                 _ => "Unspecified error.",
             };

DigitalLearningSolutions.Web/ViewModels/Frameworks/ImportCompetenciesPreProcessViewModel.cs

@@ -9,8 +9,8 @@ public class ImportCompetenciesResultsViewModel
         public ImportCompetenciesResultsViewModel(ImportCompetenciesResult importCompetenciesResult)
         {
             ProcessedCount = importCompetenciesResult.ProcessedCount;
-            CompetenciesInsertedCount = importCompetenciesResult.CompetenciesInsertedCount;
-            CompetencyGroupsInsertedCount = importCompetenciesResult.CompetencyGroupsInsertedCount;
+            CompetenciesInsertedCount = importCompetenciesResult.CompetencyAddedCount;
+            CompetencyGroupsInsertedCount = importCompetenciesResult.GroupAddedCount;
             SkippedCount = importCompetenciesResult.SkippedCount;
             Errors = importCompetenciesResult.Errors.Select(x => (x.RowNumber, MapReasonToErrorMessage(x.Reason)));
         }

DigitalLearningSolutions.Web/ViewModels/Frameworks/ImportCompetenciesResultsViewModel.cs

@@ -7,6 +7,7 @@
     public class ImportCompetenciesViewModel
     {
         public int FrameworkId { get; set; }
+        public bool IsNotBlank { get; set; }
         [Required(ErrorMessage = "Import competencies file is required")]
         [AllowedExtensions(new[] { ".xlsx" }, "Import competencies file must be in xlsx format")]
         [MaxFileSize(5 * 1024 * 1024, "Maximum allowed file size is 5MB")]