Merge pull request #3049 from TechnologyEnhancedLearning/Develop/Features/TD-5219-ImproveFileUploaded

kevwhitt-hee · web-flow · commit 28853e3c8c26 · 2025-01-03T12:11:54.000Z
TD-5219 Improvements file uploaded summary and error messages
diff --git a/DigitalLearningSolutions.Data/Models/Frameworks/Import/BulkCompetency.cs b/DigitalLearningSolutions.Data/Models/Frameworks/Import/BulkCompetency.cs
@@ -8,7 +8,7 @@ namespace DigitalLearningSolutions.Data.Models.Frameworks.Import
 {
     public class BulkCompetency
     {
-        public int? id { get; set; }
+        public int? ID { get; set; }
         public string? CompetencyGroup { get; set; }
         public string? GroupDescription { get; set; }
         public string? Competency { get; set; }
diff --git a/DigitalLearningSolutions.Data/Models/Frameworks/Import/CompetencyTableRow.cs b/DigitalLearningSolutions.Data/Models/Frameworks/Import/CompetencyTableRow.cs
@@ -12,7 +12,8 @@ public enum RowStatus
         CompetencyGroupInserted,
         CompetencyGroupUpdated,
         CompetencyGroupAndCompetencyUpdated,
-        InvalidAlwaysShowDescription
+        InvalidAlwaysShowDescription,
+        InvalidId
     }
     public class CompetencyTableRow : BulkCompetency
     {
@@ -25,7 +26,7 @@ public CompetencyTableRow(IXLTable table, IXLRangeRow row)
             }
 
             RowNumber = row.RowNumber();
-            id = row.Cell(1).GetValue<int?>();
+            ID = row.Cell(1).GetValue<int?>();
             CompetencyGroup = row.Cell(2).GetValue<string?>();
             GroupDescription = row.Cell(3).GetValue<string?>();
             Competency = row.Cell(4).GetValue<string?>();
@@ -57,6 +58,10 @@ public bool Validate()
             {
                 Error = ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription;
             }
+            else if (RowStatus == RowStatus.InvalidId)
+            {
+                Error = ImportCompetenciesResult.ErrorReason.InvalidId;
+            }
 
             return !Error.HasValue;
         }
diff --git a/DigitalLearningSolutions.Data/Models/Frameworks/Import/ImportCompetenciesResult.cs b/DigitalLearningSolutions.Data/Models/Frameworks/Import/ImportCompetenciesResult.cs
@@ -26,6 +26,21 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
             GroupAddedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.CompetencyGroupInserted | dr.RowStatus == RowStatus.CompetencyGroupAndCompetencyInserted);
             SkippedCount = competencyTableRows.Count(dr => dr.RowStatus == RowStatus.Skipped);
             Errors = competencyTableRows.Where(dr => dr.Error.HasValue).Select(dr => (dr.RowNumber, dr.Error!.Value));
+            FlagCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.FlagsCsv))
+                .SelectMany(static row => row.FlagsCsv.Split(',', StringSplitOptions.RemoveEmptyEntries))
+                .Count();
+            DistinctFlagsCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.FlagsCsv))
+                .SelectMany(row => row.FlagsCsv.Split(',', StringSplitOptions.RemoveEmptyEntries))
+                .Select(flag => flag.Trim())
+                .Distinct()
+                .Count();
+            CompetencyGroupCount = competencyTableRows
+                .Where(row => !string.IsNullOrWhiteSpace(row.CompetencyGroup))
+                .Select(static row => row.CompetencyGroup)
+                .Distinct()
+                .Count();
         }
 
         public IEnumerable<(int RowNumber, ErrorReason Reason)>? Errors { get; set; }
@@ -35,5 +50,8 @@ IReadOnlyCollection<CompetencyTableRow> competencyTableRows
         public int GroupAddedCount { get; set; }
         public int GroupUpdatedCount { get; set; }
         public int SkippedCount { get; set; }
+        public int FlagCount { get; set; }
+        public int DistinctFlagsCount { get; set; }
+        public int CompetencyGroupCount { get; set; }
     }
 }
diff --git a/DigitalLearningSolutions.Web/Controllers/FrameworksController/ImportCompetencies.cs b/DigitalLearningSolutions.Web/Controllers/FrameworksController/ImportCompetencies.cs
@@ -79,7 +79,7 @@ public IActionResult ImportCompleted()
             var workbook = new XLWorkbook(filePath);
             try
             {
-                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary);
+                var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary, data.FrameworkId);
                 var resultsModel = new ImportCompetenciesPreProcessViewModel(results, data) { IsNotBlank = data.IsNotBlank, TabName = data.TabName };
                 data.CompetenciesToProcessCount = resultsModel.ToProcessCount;
                 data.CompetenciesToAddCount = resultsModel.CompetenciesToAddCount;
diff --git a/DigitalLearningSolutions.Web/Services/ImportCompetenciesFromFileService.cs b/DigitalLearningSolutions.Web/Services/ImportCompetenciesFromFileService.cs
@@ -16,7 +16,7 @@ namespace DigitalLearningSolutions.Web.Services
     public interface IImportCompetenciesFromFileService
     {
         byte[] GetCompetencyFileForFramework(int frameworkId, bool isBlank, string vocabulary);
-        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary);
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId);
         public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary);
     }
     public class ImportCompetenciesFromFileService : IImportCompetenciesFromFileService
@@ -30,27 +30,36 @@ IFrameworkService frameworkService
         {
             this.frameworkService = frameworkService;
         }
-        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary)
+        public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId)
         {
             var table = OpenCompetenciesTable(workbook, vocabulary);
             var competencyRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
+            var existingCompetencies = frameworkService.GetBulkCompetenciesForFramework(frameworkId);
+            var existingIds = existingCompetencies.Select(bc => (int)bc.ID).ToList();
             foreach (var competencyRow in competencyRows)
             {
-                PreProcessCompetencyRow(competencyRow);
+                PreProcessCompetencyRow(competencyRow, existingIds);
             }
             return new ImportCompetenciesResult(competencyRows);
         }
-        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow)
+        private void PreProcessCompetencyRow(CompetencyTableRow competencyRow, List<int> existingIds)
         {
-            competencyRow.Validate();
-            if (competencyRow.id == null)
+            if (competencyRow.ID == null)
             {
                 competencyRow.RowStatus = RowStatus.CompetencyInserted;
             }
             else
             {
-                competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+                if (!existingIds.Contains((int)(competencyRow?.ID)))
+                {
+                    competencyRow.RowStatus = RowStatus.InvalidId;
+                }
+                else
+                {
+                    competencyRow.RowStatus = RowStatus.CompetencyUpdated;
+                }
             }
+            competencyRow.Validate();
         }
         public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary)
         {
@@ -77,7 +86,12 @@ internal IXLTable OpenCompetenciesTable(IXLWorkbook workbook, string vocabulary)
         internal ImportCompetenciesResult ProcessCompetenciesTable(IXLTable table, int adminUserId, int frameworkId, int maxFrameworkCompetencyId, int maxFrameworkCompetencyGroupId)
         {
             var competenciesRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();
-
+            
+            var competencyGroupCount = competenciesRows
+            .Where(row => !string.IsNullOrWhiteSpace(row.CompetencyGroup))
+            .Select(row => row.CompetencyGroup)
+            .Distinct()
+            .Count();
             foreach (var competencyRow in competenciesRows)
             {
                 maxFrameworkCompetencyGroupId = ProcessCompetencyRow(adminUserId, frameworkId, maxFrameworkCompetencyId, maxFrameworkCompetencyGroupId, competencyRow);
@@ -167,13 +181,11 @@ public byte[] GetCompetencyFileForFramework(int frameworkId, bool blank, string
         }
         private void PopulateCompetenciesSheet(IXLWorkbook workbook, int frameworkId, bool blank, string vocabulary)
         {
-
-
             var competencyRecords = frameworkService.GetBulkCompetenciesForFramework(blank ? 0 : frameworkId);
             var competencies = competencyRecords.Select(
                 x => new
                 {
-                    ID = x.id,
+                    x.ID,
                     x.CompetencyGroup,
                     x.GroupDescription,
                     x.Competency,
diff --git a/DigitalLearningSolutions.Web/ViewModels/Frameworks/Import/ImportCompetenciesPreProcessViewModel.cs b/DigitalLearningSolutions.Web/ViewModels/Frameworks/Import/ImportCompetenciesPreProcessViewModel.cs
@@ -19,6 +19,9 @@ public ImportCompetenciesPreProcessViewModel(ImportCompetenciesResult bulkCompet
             CompetenciesToAddCount = bulkCompetenciesResult.CompetencyAddedCount;
             ToUpdateOrSkipCount = bulkCompetenciesResult.CompetencyUpdatedCount;
             Errors = bulkCompetenciesResult.Errors.Select(x => (x.RowNumber, MapReasonToErrorMessage(x.Reason, FrameworkVocabularyHelper.VocabularySingular(bulkCompetenciesData.FrameworkVocubulary))));
+            FlagCount = bulkCompetenciesResult.FlagCount;
+            DistinctFlagsCount = bulkCompetenciesResult.DistinctFlagsCount;
+            CompetencyGroupCount = bulkCompetenciesResult.CompetencyGroupCount;
         }
         public string? FrameworkName { get; set; }
         public int PublishStatusID { get; set; }
@@ -32,6 +35,9 @@ public ImportCompetenciesPreProcessViewModel(ImportCompetenciesResult bulkCompet
         public string? ImportFile { get; set; }
         public bool IsNotBlank { get; set; }
         public string TabName { get; set; }
+        public int FlagCount { get; set; }
+        public int DistinctFlagsCount { get; set; }
+        public int CompetencyGroupCount { get; set; }
 
         private static string MapReasonToErrorMessage(ImportCompetenciesResult.ErrorReason reason, string vocabularySingular)
         {
@@ -42,7 +48,7 @@ private static string MapReasonToErrorMessage(ImportCompetenciesResult.ErrorReas
                 ImportCompetenciesResult.ErrorReason.MissingCompetencyName =>
                     vocabularySingular + " is blank. " + vocabularySingular + " is a required field and cannot be left blank",
                 ImportCompetenciesResult.ErrorReason.InvalidId =>
-                    "The ID provided does not match a " + vocabularySingular + " ID in this Framework",
+                    "The ID provided does not match a " + vocabularySingular + " ID in this Framework. Leave the ID column blank for new competencies.",
                 ImportCompetenciesResult.ErrorReason.TooLongCompetencyName =>
                     vocabularySingular + " must be 500 characters or less.",
                 ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription =>
diff --git a/DigitalLearningSolutions.Web/Views/Frameworks/Developer/Import/ImportCompleted.cshtml b/DigitalLearningSolutions.Web/Views/Frameworks/Developer/Import/ImportCompleted.cshtml
@@ -35,6 +35,8 @@
                 <li>@Model.ToProcessCount @(Model.ToProcessCount == 1 ? "row" : "rows") to process</li>
                 <li>@Model.CompetenciesToAddCount new @(Model.CompetenciesToAddCount == 1 ? Model.FrameworkVocabularySingular.ToLower() : Model.FrameworkVocabularyPlural.ToLower()) to add</li>
                 <li>@Model.ToUpdateOrSkipCount @Model.FrameworkVocabularySingular.ToLower() @(Model.ToUpdateOrSkipCount == 1 ? "record" : "records") to update (or skip if unchanged)</li>
+                <li>In @Model.CompetencyGroupCount @Model.FrameworkVocabularySingular.ToLower() groups</li>
+                <li>With a total of @Model.FlagCount flags assigned to @Model.FrameworkVocabularyPlural.ToLower() (@Model.DistinctFlagsCount distinct flags)</li>
                 @if (Model.ErrorCount > 0)
                 {
                     <li>@Model.ErrorCount @(Model.ErrorCount == 1 ? "row" : "rows") containing errors that cannot be processed</li>

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ namespace DigitalLearningSolutions.Data.Models.Frameworks.Import`
`8`	`8`	`{`
`9`	`9`	`public class BulkCompetency`
`10`	`10`	`{`
`11`		`- public int? id { get; set; }`
	`11`	`+ public int? ID { get; set; }`
`12`	`12`	`public string? CompetencyGroup { get; set; }`
`13`	`13`	`public string? GroupDescription { get; set; }`
`14`	`14`	`public string? Competency { get; set; }`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,8 @@ public enum RowStatus`
`12`	`12`	`CompetencyGroupInserted,`
`13`	`13`	`CompetencyGroupUpdated,`
`14`	`14`	`CompetencyGroupAndCompetencyUpdated,`
`15`		`- InvalidAlwaysShowDescription`
	`15`	`+ InvalidAlwaysShowDescription,`
	`16`	`+ InvalidId`
`16`	`17`	`}`
`17`	`18`	`public class CompetencyTableRow : BulkCompetency`
`18`	`19`	`{`
`@@ -25,7 +26,7 @@ public CompetencyTableRow(IXLTable table, IXLRangeRow row)`
`25`	`26`	`}`
`26`	`27`
`27`	`28`	`RowNumber = row.RowNumber();`
`28`		`- id = row.Cell(1).GetValue<int?>();`
	`29`	`+ ID = row.Cell(1).GetValue<int?>();`
`29`	`30`	`CompetencyGroup = row.Cell(2).GetValue<string?>();`
`30`	`31`	`GroupDescription = row.Cell(3).GetValue<string?>();`
`31`	`32`	`Competency = row.Cell(4).GetValue<string?>();`
`@@ -57,6 +58,10 @@ public bool Validate()`
`57`	`58`	`{`
`58`	`59`	`Error = ImportCompetenciesResult.ErrorReason.InvalidAlwaysShowDescription;`
`59`	`60`	`}`
	`61`	`+ else if (RowStatus == RowStatus.InvalidId)`
	`62`	`+ {`
	`63`	`+ Error = ImportCompetenciesResult.ErrorReason.InvalidId;`
	`64`	`+ }`
`60`	`65`
`61`	`66`	`return !Error.HasValue;`
`62`	`67`	`}`
Original file line number	Diff line number	Diff line change
`@@ -79,7 +79,7 @@ public IActionResult ImportCompleted()`
`79`	`79`	`var workbook = new XLWorkbook(filePath);`
`80`	`80`	`try`
`81`	`81`	`{`
`82`		`- var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary);`
	`82`	`+ var results = importCompetenciesFromFileService.PreProcessCompetenciesTable(workbook, data.FrameworkVocubulary, data.FrameworkId);`
`83`	`83`	`var resultsModel = new ImportCompetenciesPreProcessViewModel(results, data) { IsNotBlank = data.IsNotBlank, TabName = data.TabName };`
`84`	`84`	`data.CompetenciesToProcessCount = resultsModel.ToProcessCount;`
`85`	`85`	`data.CompetenciesToAddCount = resultsModel.CompetenciesToAddCount;`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ namespace DigitalLearningSolutions.Web.Services`
`16`	`16`	`public interface IImportCompetenciesFromFileService`
`17`	`17`	`{`
`18`	`18`	`byte[] GetCompetencyFileForFramework(int frameworkId, bool isBlank, string vocabulary);`
`19`		`- public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary);`
	`19`	`+ public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId);`
`20`	`20`	`public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary);`
`21`	`21`	`}`
`22`	`22`	`public class ImportCompetenciesFromFileService : IImportCompetenciesFromFileService`
`@@ -30,27 +30,36 @@ IFrameworkService frameworkService`
`30`	`30`	`{`
`31`	`31`	`this.frameworkService = frameworkService;`
`32`	`32`	`}`
`33`		`- public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary)`
	`33`	`+ public ImportCompetenciesResult PreProcessCompetenciesTable(IXLWorkbook workbook, string vocabulary, int frameworkId)`
`34`	`34`	`{`
`35`	`35`	`var table = OpenCompetenciesTable(workbook, vocabulary);`
`36`	`36`	`var competencyRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();`
	`37`	`+ var existingCompetencies = frameworkService.GetBulkCompetenciesForFramework(frameworkId);`
	`38`	`+ var existingIds = existingCompetencies.Select(bc => (int)bc.ID).ToList();`
`37`	`39`	`foreach (var competencyRow in competencyRows)`
`38`	`40`	`{`
`39`		`- PreProcessCompetencyRow(competencyRow);`
	`41`	`+ PreProcessCompetencyRow(competencyRow, existingIds);`
`40`	`42`	`}`
`41`	`43`	`return new ImportCompetenciesResult(competencyRows);`
`42`	`44`	`}`
`43`		`- private void PreProcessCompetencyRow(CompetencyTableRow competencyRow)`
	`45`	`+ private void PreProcessCompetencyRow(CompetencyTableRow competencyRow, List<int> existingIds)`
`44`	`46`	`{`
`45`		`- competencyRow.Validate();`
`46`		`- if (competencyRow.id == null)`
	`47`	`+ if (competencyRow.ID == null)`
`47`	`48`	`{`
`48`	`49`	`competencyRow.RowStatus = RowStatus.CompetencyInserted;`
`49`	`50`	`}`
`50`	`51`	`else`
`51`	`52`	`{`
`52`		`- competencyRow.RowStatus = RowStatus.CompetencyUpdated;`
	`53`	`+ if (!existingIds.Contains((int)(competencyRow?.ID)))`
	`54`	`+ {`
	`55`	`+ competencyRow.RowStatus = RowStatus.InvalidId;`
	`56`	`+ }`
	`57`	`+ else`
	`58`	`+ {`
	`59`	`+ competencyRow.RowStatus = RowStatus.CompetencyUpdated;`
	`60`	`+ }`
`53`	`61`	`}`
	`62`	`+ competencyRow.Validate();`
`54`	`63`	`}`
`55`	`64`	`public ImportCompetenciesResult ProcessCompetenciesFromFile(IXLWorkbook workbook, int adminUserId, int frameworkId, string vocabulary)`
`56`	`65`	`{`
`@@ -77,7 +86,12 @@ internal IXLTable OpenCompetenciesTable(IXLWorkbook workbook, string vocabulary)`
`77`	`86`	`internal ImportCompetenciesResult ProcessCompetenciesTable(IXLTable table, int adminUserId, int frameworkId, int maxFrameworkCompetencyId, int maxFrameworkCompetencyGroupId)`
`78`	`87`	`{`
`79`	`88`	`var competenciesRows = table.Rows().Skip(1).Select(row => new CompetencyTableRow(table, row)).ToList();`
`80`		`-`
	`89`	`+`
	`90`	`+ var competencyGroupCount = competenciesRows`
	`91`	`+ .Where(row => !string.IsNullOrWhiteSpace(row.CompetencyGroup))`
	`92`	`+ .Select(row => row.CompetencyGroup)`
	`93`	`+ .Distinct()`
	`94`	`+ .Count();`
`81`	`95`	`foreach (var competencyRow in competenciesRows)`
`82`	`96`	`{`
`83`	`97`	`maxFrameworkCompetencyGroupId = ProcessCompetencyRow(adminUserId, frameworkId, maxFrameworkCompetencyId, maxFrameworkCompetencyGroupId, competencyRow);`
`@@ -167,13 +181,11 @@ public byte[] GetCompetencyFileForFramework(int frameworkId, bool blank, string`
`167`	`181`	`}`
`168`	`182`	`private void PopulateCompetenciesSheet(IXLWorkbook workbook, int frameworkId, bool blank, string vocabulary)`
`169`	`183`	`{`
`170`		`-`
`171`		`-`
`172`	`184`	`var competencyRecords = frameworkService.GetBulkCompetenciesForFramework(blank ? 0 : frameworkId);`
`173`	`185`	`var competencies = competencyRecords.Select(`
`174`	`186`	`x => new`
`175`	`187`	`{`
`176`		`- ID = x.id,`
	`188`	`+ x.ID,`
`177`	`189`	`x.CompetencyGroup,`
`178`	`190`	`x.GroupDescription,`
`179`	`191`	`x.Competency,`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,8 @@`
`35`	`35`	`<li>@Model.ToProcessCount @(Model.ToProcessCount == 1 ? "row" : "rows") to process</li>`
`36`	`36`	`<li>@Model.CompetenciesToAddCount new @(Model.CompetenciesToAddCount == 1 ? Model.FrameworkVocabularySingular.ToLower() : Model.FrameworkVocabularyPlural.ToLower()) to add</li>`
`37`	`37`	`<li>@Model.ToUpdateOrSkipCount @Model.FrameworkVocabularySingular.ToLower() @(Model.ToUpdateOrSkipCount == 1 ? "record" : "records") to update (or skip if unchanged)</li>`
	`38`	`+ <li>In @Model.CompetencyGroupCount @Model.FrameworkVocabularySingular.ToLower() groups</li>`
	`39`	`+ <li>With a total of @Model.FlagCount flags assigned to @Model.FrameworkVocabularyPlural.ToLower() (@Model.DistinctFlagsCount distinct flags)</li>`
`38`	`40`	`@if (Model.ErrorCount > 0)`
`39`	`41`	`{`
`40`	`42`	`<li>@Model.ErrorCount @(Model.ErrorCount == 1 ? "row" : "rows") containing errors that cannot be processed</li>`