TD-5154 Implements competencies uploaded page

DigitalLearningSolutions.Data/Models/Frameworks/Import/CompetencyTableRow.cs

@@ -1,6 +1,7 @@
 namespace DigitalLearningSolutions.Data.Models.Frameworks.Import
 {
     using ClosedXML.Excel;
+
     public enum RowStatus
     {
         NotYetProcessed,
@@ -10,28 +11,32 @@ public enum RowStatus
         CompetencyUpdated,
         CompetencyGroupInserted,
         CompetencyGroupUpdated,
-        CompetencyGroupAndCompetencyUpdated
+        CompetencyGroupAndCompetencyUpdated,
+        InvalidAlwaysShowDescription
     }
     public class CompetencyTableRow : BulkCompetency
     {
         public CompetencyTableRow(IXLTable table, IXLRangeRow row)
         {
             string? FindFieldValue(string name)
             {
-                var colNumber = table.FindColumn(col => col.FirstCell().Value.ToString()?.ToLower() == name).ColumnNumber();
+                var colNumber = table.FindColumn(col => col.FirstCell().Value.ToString()?.ToLower() == name.ToLower()).ColumnNumber();
                 return row.Cell(colNumber).GetValue<string?>();
             }
 
             RowNumber = row.RowNumber();
-            id = int.Parse(FindFieldValue("ID"));
+            id = row.Cell(1).GetValue<int?>();
             CompetencyGroup = FindFieldValue("CompetencyGroup");
             Competency = FindFieldValue("Competency");
             CompetencyDescription = FindFieldValue("CompetencyDescription");
-            GroupDescription = FindFieldValue("CompetencyGroupDescription");
+            GroupDescription = FindFieldValue("GroupDescription");
+            AlwaysShowDescriptionRaw = FindFieldValue("AlwaysShowDescription");
+            AlwaysShowDescription = bool.TryParse(AlwaysShowDescriptionRaw, out var hasPrn) ? hasPrn : (bool?)null;
             FlagsCsv = FindFieldValue("FlagsCSV");
             RowStatus = RowStatus.NotYetProcessed;
         }
         public int RowNumber { get; set; }
+        public string? AlwaysShowDescriptionRaw { get; set; }
         public ImportCompetenciesResult.ErrorReason? Error { get; set; }
         public RowStatus RowStatus { get; set; }
         public bool Validate()
@@ -48,6 +53,10 @@ public bool Validate()
             {
                 Error = ImportCompetenciesResult.ErrorReason.TooLongCompetencyName;
             }
+            else if (!string.IsNullOrWhiteSpace(AlwaysShowDescriptionRaw) && !bool.TryParse(AlwaysShowDescriptionRaw, out _))
+            {
+                Error = ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription;
+            }
 
             return !Error.HasValue;
         }

DigitalLearningSolutions.Data/Models/Frameworks/Import/ImportCompetenciesResult.cs

@@ -8,10 +8,11 @@ public class ImportCompetenciesResult
     {
         public enum ErrorReason
         {
+            InvalidId,
             MissingCompetencyName,
-            TooLongCompetencyGroupName,
             TooLongCompetencyName,
-            AlreadyExists
+            TooLongCompetencyGroupName,
+            InvalidAlwaysShowDescription
         }
         public ImportCompetenciesResult() { }
 
@@ -20,16 +21,18 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
         )
         {
             ProcessedCount = competencyTableRows.Count;
-            CompetenciesInsertedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
-            CompetencyGroupsInsertedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
+            CompetencyAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
+            GroupAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
             SkippedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.Skipped);
             Errors = competencyTableRows.Where(dr => dr.Error.HasValue).Select(dr => (dr.RowNumber, dr.Error!.Value));
         }
 
         public IEnumerable<(int RowNumber, ErrorReason Reason)>? Errors { get; set; }
         public int ProcessedCount { get; set; }
-        public int CompetenciesInsertedCount { get; set; }
-        public int CompetencyGroupsInsertedCount { get; set; }
+        public int CompetencyAddedCount { get; set; }
+        public int CompetencyUpdatedCount { get; set; }
+        public int GroupAddedCount { get; set; }
+        public int GroupUpdatedCount { get; set; }
         public int SkippedCount { get; set; }
     }
 }

DigitalLearningSolutions.Web/Controllers/FrameworksController/ImportCompetencies.cs

@@ -1,29 +1,28 @@
 using ClosedXML.Excel;
 using DigitalLearningSolutions.Data.Exceptions;
-using DigitalLearningSolutions.Data.Migrations;
-using DigitalLearningSolutions.Data.Utilities;
 using DigitalLearningSolutions.Web.Helpers;
 using DigitalLearningSolutions.Web.Models;
 using DigitalLearningSolutions.Web.Services;
 using DigitalLearningSolutions.Web.ViewModels.Frameworks;
 using GDS.MultiPageFormData.Enums;
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc;
+using System.IO;
 
 namespace DigitalLearningSolutions.Web.Controllers.FrameworksController
 {
     public partial class FrameworksController
     {
         [Route("/Framework/{frameworkId}/{tabname}/Import")]
-        public IActionResult ImportCompetencies(int frameworkId, string tabname)
+        public IActionResult ImportCompetencies(int frameworkId, string tabname, bool isNotBlank)
         {
             var adminId = GetAdminId();
             var userRole = frameworkService.GetAdminUserRoleForFrameworkId(adminId, frameworkId);
             if (userRole < 2)
                 return StatusCode(403);
             var model = new ImportCompetenciesViewModel()
             {
-                FrameworkId = frameworkId
+                FrameworkId = frameworkId,
+                IsNotBlank = isNotBlank
             };
             return View("Developer/ImportCompetencies", model);
         }
@@ -41,7 +40,8 @@ public IActionResult DownloadCompetencies(int frameworkId, int DownloadOption)
         }
         [HttpPost]
         [Route("/Framework/{frameworkId}/{tabname}/Import")]
-        public IActionResult StartImport(ImportCompetenciesViewModel model, string tabname)
+        [Route("/Framework/{frameworkId}/{tabname}/ImportCompleted")]
+        public IActionResult StartImport(ImportCompetenciesViewModel model, string tabname, bool isNotBlank)
         {
             if (!ModelState.IsValid)
                 return View("Developer/ImportCompetencies", model);
@@ -51,11 +51,11 @@ public IActionResult StartImport(ImportCompetenciesViewModel model, string tabna
                 var workbook = new XLWorkbook(model.ImportFile.OpenReadStream());
                 if (!workbook.Worksheets.Contains(ImportCompetenciesFromFileService.CompetenciesSheetName))
                 {
-                    ModelState.AddModelError("InvalidWorksheet", CommonValidationErrorMessages.InvalidCompetenciesUploadExcelFile);
+                    ModelState.AddModelError("ImportFile", CommonValidationErrorMessages.InvalidCompetenciesUploadExcelFile);
                     return View("Developer/ImportCompetencies", model);
                 }
                 var competenciesFileName = FileHelper.UploadFile(webHostEnvironment, model.ImportFile);
-                setupBulkUploadData(model.FrameworkId, adminUserID, competenciesFileName, tabname);
+                setupBulkUploadData(model.FrameworkId, adminUserID, competenciesFileName, tabname, isNotBlank);
 
                 return RedirectToAction("ImportCompleted", "Frameworks", new { frameworkId = model.FrameworkId, tabname });
             }
@@ -72,15 +72,33 @@ public IActionResult StartImport(ImportCompetenciesViewModel model, string tabna
         [Route("/Framework/{frameworkId}/{tabname}/ImportCompleted")]
         public IActionResult ImportCompleted()
         {
-            return View("Developer/ImportCompleted");
+            var data = GetBulkUploadData();
+            var uploadDir = Path.Combine(webHostEnvironment.WebRootPath, "Uploads\\");
+            var filePath = Path.Combine(uploadDir, data.CompetenciesFileName);
+            var workbook = new XLWorkbook(filePath);
+            try
+            {
+                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook);
+                var resultsModel = new ImportCompetenciesPreProcessViewModel(results) { IsNotBlank = data.IsNotBlank, TabName = data.TabName };
+                data.CompetenciesToProcessCount = resultsModel.ToProcessCount;
+                data.CompetenciesToAddCount = resultsModel.CompetenciesToAddCount;
+                data.CompetenciesToUpdateCount = resultsModel.CompetenciesToUpdateCount;
+                setBulkUploadData(data);
+                return View("Developer/ImportCompleted", resultsModel);
+            }
+            catch (InvalidHeadersException)
+            {
+                FileHelper.DeleteFile(webHostEnvironment, data.CompetenciesFileName);
+                return View("ImportFailed");
+            }
         }
 
-        private void setupBulkUploadData(int frameworkId, int adminUserID, string competenciessFileName, string tabName)
+        private void setupBulkUploadData(int frameworkId, int adminUserID, string competenciessFileName, string tabName, bool isNotBlank)
         {
             TempData.Clear();
             multiPageFormService.ClearMultiPageFormData(MultiPageFormDataFeature.AddCustomWebForm("BulkCompetencyDataCWF"), TempData);
             var today = clockUtility.UtcToday;
-            var bulkUploadData = new BulkCompetenciesData(frameworkId, adminUserID, competenciessFileName, tabName);
+            var bulkUploadData = new BulkCompetenciesData(frameworkId, adminUserID, competenciessFileName, tabName, isNotBlank);
             setBulkUploadData(bulkUploadData);
         }
         private void setBulkUploadData(BulkCompetenciesData bulkUploadData)

DigitalLearningSolutions.Web/Helpers/CommonValidationErrorMessages.cs

@@ -36,7 +36,7 @@ public static class CommonValidationErrorMessages
         public const string CentreNameAlreadyExist = "The centre name you have entered already exists, please enter a different centre name";
         public const string MaxBulkUploadRowsLimit = "File must contain no more than {0} rows";
         public const string InvalidBulkUploadExcelFile = "The uploaded file must contain a \"DelegatesBulkUpload\" worksheet. Use the \"Download delegates\" button to generate a template.";
-        public const string InvalidCompetenciesUploadExcelFile = "The uploaded file must contain a \"CompetenciesBulkUpload\" worksheet. Use the \"Download competencies\" button to generate a template.";
+        public const string InvalidCompetenciesUploadExcelFile = "The uploaded file must contain a \"CompetenciesBulkUpload\" worksheet. Use the \"Download template\" button to generate a template.";
         public const string ReportFilterReturnsTooManyRows = "The report frequency is too high for the date range. Choose a lower report frequency (or shorten the date range)";
     }
 }

DigitalLearningSolutions.Web/Models/BulkCompetenciesData.cs

@@ -6,22 +6,24 @@ namespace DigitalLearningSolutions.Web.Models
     public class BulkCompetenciesData
     {
         public BulkCompetenciesData() { }
-        public BulkCompetenciesData(int frameworkId, int adminUserId, string competenciesFileName, string tabName)
+        public BulkCompetenciesData(int frameworkId, int adminUserId, string competenciesFileName, string tabName, bool isNotBlank)
         {
             FrameworkId = frameworkId;
             AdminUserId = adminUserId;
             CompetenciesFileName = competenciesFileName;
             TabName = tabName;
+            IsNotBlank = isNotBlank;
         }
         public int FrameworkId { get; set; }
         public string TabName { get; set; }
         public int AdminUserId { get; set; }
+        public bool IsNotBlank { get; set; }
         public string CompetenciesFileName { get; set; }
         public List<int> AssessmentQuestionIDs { get; set; }
         public int? AddAssessmentQuestionOption { get; set; }
-        public int ToProcessCount { get; set; }
-        public int ToAddCount { get; set; }
-        public int ToUpdateCount { get; set; }
+        public int CompetenciesToProcessCount { get; set; }
+        public int CompetenciesToAddCount { get; set; }
+        public int CompetenciesToUpdateCount { get; set; }
         public int LastRowProcessed { get; set; }
         public int SubtotalCompetenciesAdded { get; set; }
         public int SubtotalCompetenciesUpdated { get; set; }

DigitalLearningSolutions.Web/Services/ImportCompetenciesFromFileService.cs

@@ -12,12 +12,14 @@ namespace DigitalLearningSolutions.Web.Services
     using DigitalLearningSolutions.Data.Exceptions;
     using DigitalLearningSolutions.Data.Helpers;
     using DigitalLearningSolutions.Data.Models.Frameworks.Import;
+    using DigitalLearningSolutions.Web.Models;
     using Microsoft.AspNetCore.Http;
 
     public interface IImportCompetenciesFromFileService
     {
         byte[] GetCompetencyFileForFramework(int frameworkId, bool v);
-        public ImportCompetenciesResult ProcessCompetenciesFromFile(IFormFile file, int adminUserId, int frameworkId);
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook);
+        public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId);
     }
     public class ImportCompetenciesFromFileService : IImportCompetenciesFromFileService
     {
@@ -30,17 +32,39 @@ IFrameworkService frameworkService
         {
             this.frameworkService = frameworkService;
         }
-        public ImportCompetenciesResult ProcessCompetenciesFromFile(IFormFile file, int adminUserId, int frameworkId)
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook)
+        {
+            var table = OpenCompetenciesTable(workbook);
+            var competencyRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
+            foreach (var competencyRow in competencyRows)
+            {
+                PreProcessCompetencyRow(competencyRow);
+            }
+            return new ImportCompetenciesResult(competencyRows);
+        }
+        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow)
+        {
+            competencyRow.Validate();
+            if (competencyRow.id == null)
+            {
+                competencyRow.RowStatus = RowStatus.CompetencyInserted;
+            }
+            else
+            {
+                competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+            }
+        }
+        public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId)
         {
             int maxFrameworkCompetencyId = frameworkService.GetMaxFrameworkCompetencyID();
             int maxFrameworkCompetencyGroupId = frameworkService.GetMaxFrameworkCompetencyGroupID();
-            var table = OpenCompetenciesTable(file);
+            var table = OpenCompetenciesTable(workbook);
             return ProcessCompetenciesTable(table, adminUserId, frameworkId, maxFrameworkCompetencyId, maxFrameworkCompetencyGroupId);
         }
-        internal IXLTable OpenCompetenciesTable(IFormFile file)
+        internal IXLTable OpenCompetenciesTable(IXLWorkbook workbook)
         {
-            var workbook = new XLWorkbook(file.OpenReadStream());
             var worksheet = workbook.Worksheet(1);
+            worksheet.Columns(1, 15).Unhide();
             if (worksheet.Tables.Count() == 0)
             {
                 throw new InvalidHeadersException();
@@ -112,11 +136,15 @@ private static bool ValidateHeaders(IXLTable table)
         {
             var expectedHeaders = new List<string>
             {
-                "competency group",
-                "competency name",
-                "competency description"
+                "ID",
+                "CompetencyGroup",
+                "GroupDescription",
+                "Competency",
+                "CompetencyDescription",
+                "AlwaysShowDescription",
+                "FlagsCSV"
             }.OrderBy(x => x);
-            var actualHeaders = table.Fields.Select(x => x.Name.ToLower()).OrderBy(x => x);
+            var actualHeaders = table.Fields.Select(x => x.Name).OrderBy(x => x);
             return actualHeaders.SequenceEqual(expectedHeaders);
         }
 
@@ -128,6 +156,13 @@ public byte[] GetCompetencyFileForFramework(int frameworkId, bool blank)
             {
                 ClosedXmlHelper.HideWorkSheetColumn(workbook, "ID");
             }
+            var options = new List<string> { "TRUE", "FALSE" };
+            ClosedXmlHelper.AddValidationListToWorksheetColumn(workbook, 6, options);
+            var rowCount = workbook.Worksheet(1).RangeUsed().RowCount();
+            ClosedXmlHelper.AddValidationRangeToWorksheetColumn(workbook, 1, 1, rowCount, 1);
+            // Calculate the workbook
+            workbook.CalculateMode = XLCalculateMode.Auto;
+            workbook.RecalculateAllFormulas();
             using var stream = new MemoryStream();
             workbook.SaveAs(stream);
             return stream.ToArray();