TD-5219 Improvements to pre-processing summary and errors

Author: kevwhitt-hee

DigitalLearningSolutions.Data/Models/Frameworks/Import/BulkCompetency.cs

@@ -8,7 +8,7 @@ namespace DigitalLearningSolutions.Data.Models.Frameworks.Import
 {
     public class BulkCompetency
     {
-        public int? id { get; set; }
+        public int? ID { get; set; }
         public string? CompetencyGroup { get; set; }
         public string? GroupDescription { get; set; }
         public string? Competency { get; set; }

DigitalLearningSolutions.Data/Models/Frameworks/Import/CompetencyTableRow.cs

@@ -12,7 +12,8 @@ public enum RowStatus
         CompetencyGroupInserted,
         CompetencyGroupUpdated,
         CompetencyGroupAndCompetencyUpdated,
-        InvalidAlwaysShowDescription
+        InvalidAlwaysShowDescription,
+        InvalidId
     }
     public class CompetencyTableRow : BulkCompetency
     {
@@ -25,7 +26,7 @@ public CompetencyTableRow(IXLTable table, IXLRangeRow row)
             }
 
             RowNumber = row.RowNumber();
-            id = row.Cell(1).GetValue<int?>();
+            ID = row.Cell(1).GetValue<int?>();
             CompetencyGroup = row.Cell(2).GetValue<string?>();
             GroupDescription = row.Cell(3).GetValue<string?>();
             Competency = row.Cell(4).GetValue<string?>();
@@ -57,6 +58,10 @@ public bool Validate()
             {
                 Error = ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription;
             }
+            else if (RowStatus == RowStatus.InvalidId)
+            {
+                Error = ImportCompetenciesResult.ErrorReason.InvalidId;
+            }
 
             return !Error.HasValue;
         }

DigitalLearningSolutions.Data/Models/Frameworks/Import/ImportCompetenciesResult.cs

@@ -26,6 +26,21 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
             GroupAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
             SkippedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.Skipped);
             Errors = competencyTableRows.Where(dr => dr.Error.HasValue).Select(dr => (dr.RowNumber, dr.Error!.Value));
+            FlagCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.FlagsCsv))
+                .SelectMany(static row => row.FlagsCsv.Split(',', StringSplitOptions.RemoveEmptyEntries))
+                .Count();
+            DistinctFlagsCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.FlagsCsv))
+                .SelectMany(row => row.FlagsCsv.Split(',', StringSplitOptions.RemoveEmptyEntries))
+                .Select(flag => flag.Trim())
+                .Distinct()
+                .Count();
+            CompetencyGroupCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.CompetencyGroup))
+                .Select(static row => row.CompetencyGroup)
+                .Distinct()
+                .Count();
         }
 
         public IEnumerable<(int RowNumber, ErrorReason Reason)>? Errors { get; set; }
@@ -35,5 +50,8 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
         public int GroupAddedCount { get; set; }
         public int GroupUpdatedCount { get; set; }
         public int SkippedCount { get; set; }
+        public int FlagCount { get; set; }
+        public int DistinctFlagsCount { get; set; }
+        public int CompetencyGroupCount { get; set; }
     }
 }

DigitalLearningSolutions.Web/Controllers/FrameworksController/ImportCompetencies.cs

@@ -79,7 +79,7 @@ public IActionResult ImportCompleted()
             var workbook = new XLWorkbook(filePath);
             try
             {
-                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary);
+                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary, data.FrameworkId);
                 var resultsModel = new ImportCompetenciesPreProcessViewModel(results, data) { IsNotBlank = data.IsNotBlank, TabName = data.TabName };
                 data.CompetenciesToProcessCount = resultsModel.ToProcessCount;
                 data.CompetenciesToAddCount = resultsModel.CompetenciesToAddCount;

DigitalLearningSolutions.Web/Services/ImportCompetenciesFromFileService.cs

@@ -16,7 +16,7 @@ namespace DigitalLearningSolutions.Web.Services
     public interface IImportCompetenciesFromFileService
     {
         byte[] GetCompetencyFileForFramework(int frameworkId, bool isBlank, string vocabulary);
-        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary);
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId);
         public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary);
     }
     public class ImportCompetenciesFromFileService : IImportCompetenciesFromFileService
@@ -30,27 +30,36 @@ IFrameworkService frameworkService
         {
             this.frameworkService = frameworkService;
         }
-        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary)
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId)
         {
             var table = OpenCompetenciesTable(workbook, vocabulary);
             var competencyRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
+            var existingCompetencies = frameworkService.GetBulkCompetenciesForFramework(frameworkId);
+            var existingIds = existingCompetencies.Select(bc => (int)bc.ID).ToList();
             foreach (var competencyRow in competencyRows)
             {
-                PreProcessCompetencyRow(competencyRow);
+                PreProcessCompetencyRow(competencyRow, existingIds);
             }
             return new ImportCompetenciesResult(competencyRows);
         }
-        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow)
+        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow, List<int> existingIds)
         {
-            competencyRow.Validate();
-            if (competencyRow.id == null)
+            if (competencyRow.ID == null)
             {
                 competencyRow.RowStatus = RowStatus.CompetencyInserted;
             }
             else
             {
-                competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+                if (!existingIds.Contains((int)(competencyRow?.ID)))
+                {
+                    competencyRow.RowStatus = RowStatus.InvalidId;
+                }
+                else
+                {
+                    competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+                }
             }
+            competencyRow.Validate();
         }
         public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary)
         {
@@ -77,7 +86,12 @@ internal IXLTable OpenCompetenciesTable(IXLWorkbook workbook, string vocabulary)
         internal ImportCompetenciesResult ProcessCompetenciesTable(IXLTable table, int adminUserId, int frameworkId, int maxFrameworkCompetencyId, int maxFrameworkCompetencyGroupId)
         {
             var competenciesRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
-
+            
+            var competencyGroupCount = competenciesRows
+            .Where(row => !string.IsNullOrWhiteSpace(row.CompetencyGroup))
+            .Select(row => row.CompetencyGroup)
+            .Distinct()
+            .Count();
             foreach (var competencyRow in competenciesRows)
             {
                 maxFrameworkCompetencyGroupId = ProcessCompetencyRow(adminUserId, frameworkId, maxFrameworkCompetencyId, maxFrameworkCompetencyGroupId, competencyRow);
@@ -167,13 +181,11 @@ public byte[] GetCompetencyFileForFramework(int frameworkId, bool blank, string
         }
         private void PopulateCompetenciesSheet(IXLWorkbook workbook, int frameworkId, bool blank, string vocabulary)
         {
-
-
             var competencyRecords = frameworkService.GetBulkCompetenciesForFramework(blank ? 0 : frameworkId);
             var competencies = competencyRecords.Select(
                 x => new
                 {
-                    ID = x.id,
+                    x.ID,
                     x.CompetencyGroup,
                     x.GroupDescription,
                     x.Competency,

DigitalLearningSolutions.Web/ViewModels/Frameworks/Import/ImportCompetenciesPreProcessViewModel.cs

@@ -19,6 +19,9 @@ public ImportCompetenciesPreProcessViewModel(ImportCompetenciesResult bulkCompet
             CompetenciesToAddCount = bulkCompetenciesResult.CompetencyAddedCount;
             ToUpdateOrSkipCount = bulkCompetenciesResult.CompetencyUpdatedCount;
             Errors = bulkCompetenciesResult.Errors.Select(x => (x.RowNumber, MapReasonToErrorMessage(x.Reason, FrameworkVocabularyHelper.VocabularySingular(bulkCompetenciesData.FrameworkVocubulary))));
+            FlagCount = bulkCompetenciesResult.FlagCount;
+            DistinctFlagsCount = bulkCompetenciesResult.DistinctFlagsCount;
+            CompetencyGroupCount = bulkCompetenciesResult.CompetencyGroupCount;
         }
         public string? FrameworkName { get; set; }
         public int PublishStatusID { get; set; }
@@ -32,6 +35,9 @@ public ImportCompetenciesPreProcessViewModel(ImportCompetenciesResult bulkCompet
         public string? ImportFile { get; set; }
         public bool IsNotBlank { get; set; }
         public string TabName { get; set; }
+        public int FlagCount { get; set; }
+        public int DistinctFlagsCount { get; set; }
+        public int CompetencyGroupCount { get; set; }
 
         private static string MapReasonToErrorMessage(ImportCompetenciesResult.ErrorReason reason, string vocabularySingular)
         {
@@ -42,7 +48,7 @@ private static string MapReasonToErrorMessage(ImportCompetenciesResult.ErrorReas
                 ImportCompetenciesResult.ErrorReason.MissingCompetencyName =>
                     vocabularySingular + " is blank. " + vocabularySingular + " is a required field and cannot be left blank",
                 ImportCompetenciesResult.ErrorReason.InvalidId =>
-                    "The ID provided does not match a " + vocabularySingular + " ID in this Framework",
+                    "The ID provided does not match a " + vocabularySingular + " ID in this Framework. Leave the ID column blank for new competencies.",
                 ImportCompetenciesResult.ErrorReason.TooLongCompetencyName =>
                     vocabularySingular + " must be 500 characters or less.",
                 ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription =>

DigitalLearningSolutions.Web/Views/Frameworks/Developer/Import/ImportCompleted.cshtml

@@ -35,6 +35,8 @@
                 <li>@Model.ToProcessCount @(Model.ToProcessCount == 1 ? "row" : "rows") to process</li>
                 <li>@Model.CompetenciesToAddCount new @(Model.CompetenciesToAddCount == 1 ? Model.FrameworkVocabularySingular.ToLower() : Model.FrameworkVocabularyPlural.ToLower()) to add</li>
                 <li>@Model.ToUpdateOrSkipCount @Model.FrameworkVocabularySingular.ToLower() @(Model.ToUpdateOrSkipCount == 1 ? "record" : "records") to update (or skip if unchanged)</li>
+                <li>In @Model.CompetencyGroupCount @Model.FrameworkVocabularySingular.ToLower() groups</li>
+                <li>With a total of @Model.FlagCount flags assigned to Model.FrameworkVocabularyPlural.ToLower() (@Model.DistinctFlagsCount distinct flags)</li>
                 @if (Model.ErrorCount > 0)
                 {
                     <li>@Model.ErrorCount @(Model.ErrorCount == 1 ? "row" : "rows") containing errors that cannot be processed</li>