Merge branch 'rc' of https://github.com/bexis2/core into rc

Author: sventhiel

.github/workflows/codeql.yml

@@ -13,10 +13,12 @@ name: "CodeQL Advanced"
 
 on:
   push:
-    branches: 
+    branches:
+      - master
       - rc
   pull_request:
     branches: 
+      - master
       - rc
   schedule:
     - cron: '38 14 * * 4'

Components/IO/BExIS.IO.Tests/Transform/Input/AsciiReaderTest.cs

@@ -121,6 +121,29 @@ public void rowToList_RowAsQuotesAndSeperatorInQuotes_ReturnExpectedListOfString
             Assert.That(values, Is.EquivalentTo(expectedOutcome));
         }
 
+        [Test]
+        public void rowToList_RowAsOneVariablewithQuotesAndSeperatorInQuotes_ReturnExpectedListOfStrings()
+        {
+            //Arrange
+            string row = "V1,V2,'V3,V4'";
+            List<string> expectedOutcome = new List<string> { "V1", "V2", "V3,V4" };
+
+            AsciiFileReaderInfo info = new AsciiFileReaderInfo();
+            info.Seperator = TextSeperator.comma;
+
+            AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
+
+            //Act
+
+            List<string> values = reader.rowToList(row,
+                AsciiFileReaderInfo.GetSeperator(TextSeperator.comma));
+
+            //Assert
+            Assert.That(values.Count, Is.EqualTo(expectedOutcome.Count));
+            Assert.That(values, Is.EquivalentTo(expectedOutcome));
+        }
+
+
         [Test]
         public void ValidateRow_runNotValid_LimitErrors()
         {

Components/IO/BExIS.IO.Tests/Transform/Input/StructureAnalyzerTest.cs

@@ -249,7 +249,7 @@ public void SuggestSystemTypes_Valid_ResultWithCorrectTypes(int n)
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n),TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -294,7 +294,7 @@ public void SuggestSystemTypes_WithTestData_ResultWithCorrectTypes()
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows, TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -321,7 +321,7 @@ public void SuggestSystemTypes_ValidDateTypes_ResultWithCorrectTypes(int n)
             dateValues.Add("2022-12-24"); // yyyy-MM-dd
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -334,7 +334,7 @@ public void SuggestSystemTypes_ValidWithMissingValues_ResultWithCorrectTypes()
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rowWithMissingValues, TextSeperator.semicolon, DecimalCharacter.comma, missingValueList);
+            var result = structureAnalyser.SuggestSystemTypes(rowWithMissingValues, TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, missingValueList);
 
             //Assert
             Assert.NotNull(result);

Components/IO/BExIS.IO.Transform.Input/AsciiReader.cs

@@ -988,6 +988,28 @@ public List<string> rowToList(string line, char seperator)
             return line.Split(seperator).ToList();
         }
 
+        /// <summary>
+        /// Convert a row as a string to a list of strings based on offset,seperator and textmarker
+        /// </summary>
+        /// <remarks></remarks>
+        /// <seealso cref=""/>
+        /// <param name="line">Row as a string</param>
+        /// <param name="seperator">Character used as TextSeparator</param>
+        /// <param name="textMarker">Character used as TextMarker</param>
+        /// <param name="offset">offset to skip values from begin</param>
+        /// <returns>List of values</returns>
+        public List<string> RowToList(string line, char seperator, char textMarker, int offset = 0)
+        {
+            List<string> tempRow = new List<string>();
+
+            tempRow = new List<string>();
+            tempRow = TextMarkerHandling(line, seperator,textMarker);
+
+            //if a offset is marked in the file reader information the offset needs to skip from the complete string array
+            return tempRow.Skip(offset).ToList();
+            //return line.Split(seperator).ToList();
+        }
+
         private List<string> values;
         private List<string> temp;
 

Components/IO/BExIS.IO.Transform.Input/StructureAnalyser.cs

@@ -457,14 +457,20 @@ public List<string> GetRandowRows(List<string> rows, int numberOfRows)
             throw new NotImplementedException();
         }
 
-        public Dictionary<int, Type> SuggestSystemTypes(List<string> rows, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues)
+        public Dictionary<int, Type> SuggestSystemTypes(List<string> rows,TextMarker textMarker, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues)
         {
             Dictionary<int, List<Type>> source = new Dictionary<int, List<Type>>();
             Dictionary<int, Type> result = new Dictionary<int, Type>();
 
             // get type checks
             checks = getDataTypeChecks(decimalCharacter);
 
+            // validate input
+            AsciiFileReaderInfo info = new AsciiFileReaderInfo();
+            info.Seperator = delimeter;
+            info.TextMarker = textMarker;
+            AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
+
             // create dictionary with types per column
             char seperator = AsciiFileReaderInfo.GetSeperator(delimeter);
             int numberOfRow = rows.First().Split(seperator).Count();
@@ -477,8 +483,11 @@ public Dictionary<int, Type> SuggestSystemTypes(List<string> rows, TextSeperator
             // go throw each row
             foreach (var row in rows)
             {
+                // 
+                var cells = reader.rowToList(row, seperator);
+
                 //split row based on sepeartor
-                foreach (var cell in row.Split(seperator).Select((x, i) => new { Value = x, Index = i }))
+                foreach (var cell in cells.Select((x, i) => new { Value = x, Index = i }))
                 {
                     //update possible list of types by check the value against the existing list
                     source[cell.Index] = checkValue(cell.Value, source[cell.Index], missingValues);

Console/BExIS.Web.Shell/Areas/DDM/BExIS.Modules.Ddm.UI/Controllers/DataController.cs

@@ -469,7 +469,10 @@ public async Task<ActionResult> GetCitationOrTitle(long Id, long datasetVersionI
                     var citationSettings = settingsHelper.GetCitationSettings();
 
                     var errors = new List<string>();
-                    if (citationSettings == null || !citationSettings.ShowCitation || !MappingUtils.IsMapped(datasetVersion.Dataset.MetadataStructure.Id, LinkElementType.MetadataStructure, conceptManager.FindByName("citation").Id, LinkElementType.MappingConcept, out errors))
+                    string conceptName = "Citation_" + citationSettings.ReadCitationFormat;
+                    var concept = conceptManager.FindByName(conceptName);
+
+                    if (citationSettings == null || !citationSettings.ShowCitation || concept == null || !MappingUtils.IsMapped(datasetVersion.Dataset.MetadataStructure.Id, LinkElementType.MetadataStructure, concept.Id, LinkElementType.MappingConcept, out errors))
                     {
                         return PartialView("_Title", datasetVersion.Title);
                     }

Console/BExIS.Web.Shell/Areas/RPM/BExIS.Modules.Rpm.UI/Controllers/DataStructureController.cs

@@ -441,6 +441,7 @@ public JsonResult Generate(DataStructureCreationModel model)
                 path,
                 AsciiFileReaderInfo.GetSeperator((char)model.Delimeter),
                 AsciiFileReaderInfo.GetDecimalCharacter((char)model.Decimal),
+                AsciiFileReaderInfo.GetTextMarker((char)model.TextMarker),
                 missingValues,
                 startdataIndex + 1
                 );
@@ -844,7 +845,7 @@ public JsonResult GetDWCRequirements()
         /// <param name="missingValues"></param>
         /// <param name="datastart">row not index!</param>
         /// <returns></returns>
-        private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues, int datastart)
+        private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator delimeter, DecimalCharacter decimalCharacter, TextMarker textMarker, List<string> missingValues, int datastart)
         {
             var settings = ModuleManager.GetModuleSettings("Rpm");
             int min = Convert.ToInt32(settings.GetValueByKey("minToAnalyse"));
@@ -863,7 +864,7 @@ private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator deli
 
             List<string> rows = AsciiReader.GetRandowRows(file, total, selection, datastart);
 
-            return structureAnalyser.SuggestSystemTypes(rows, delimeter, decimalCharacter, missingValues);
+            return structureAnalyser.SuggestSystemTypes(rows, textMarker, delimeter, decimalCharacter, missingValues);
         }
 
         private string getValueFromMarkedRow(List<string> rows, List<Marker> markers, string type, char delimeter, int position, char textMarker)

Release Notes/Release_Notes.md

@@ -21,8 +21,6 @@ Set up if user need to agree the data agreements before download and where the a
 
 
 
-
-
 # Bugfixes and enhancements
 ### New features
 
@@ -38,6 +36,7 @@ Set up if user need to agree the data agreements before download and where the a
 - **Entity Template:** Fix Wrong (trash) icon for cancel button in edit mode ([#2228](https://github.com/BEXIS2/Core/issues/2228))
 - **Core UI**: UX improvements across the core UI. ([#2223](https://github.com/BEXIS2/Core/issues/2223))
 - **Accessibility**: Accessibility improvements for Dataset Creation / Edit / Data Structure flows. ([#2271](https://github.com/BEXIS2/Core/issues/2271))
+- **JWT Token**: Add a copy to clipboard button ([#2259](https://github.com/BEXIS2/Core/issues/2259))
 
 
 ### Search
@@ -68,6 +67,11 @@ Set up if user need to agree the data agreements before download and where the a
 - **Complex types**: Random reuse of metadata elements with the same name in complex types. ([#2189](https://github.com/BEXIS2/Core/issues/2189))
 - **Concept Output**: Multi complex → one complex conversion does not work. ([#2224](https://github.com/BEXIS2/Core/issues/2224))
 - **Structure Mapping**: Allow mapping multiple default sources to one target. ([#2227](https://github.com/BEXIS2/Core/issues/2227))
+- **Concept mapping**: False outcome when one field is mapped to multiple target fields with default values. ([#2202](https://github.com/BEXIS2/Core/issues/2202))
+- **Dataset Details**: Creating links fails with HTTP 500 error. ([#2269](https://github.com/BEXIS2/Core/issues/2269))
+- **Set links**: Set links cannot be deleted — fix deletion handling. ([#2272](https://github.com/BEXIS2/Core/issues/2272))
+- **Metadata API**: Formats missing in JSON schemas — add missing format entries. ([#2275](https://github.com/BEXIS2/Core/issues/2275))
+
 
 
 ### Upload & parsing
@@ -81,6 +85,9 @@ Set up if user need to agree the data agreements before download and where the a
 - **Data Structure Edit**: Fix Selecting a meaning opens the select template overlay ([#2222](https://github.com/BEXIS2/Core/issues/2222))
 - **External Link**: Fix Missing validation and required fields indicator ([#2218](https://github.com/BEXIS2/Core/issues/2218))
 - **Manage Meaning**: Fix Crashes when saving empty relation([#2215](https://github.com/BEXIS2/Core/issues/2215))
+- **Data Structure From File**: Fix Escaped delimiter not ignored during data structure creation ([#283](https://github.com/BEXIS2/Core/issues/222832835))
 
 ### DOI & Citation
 - **DOI Overview**: Fix links not working. ([#2261](https://github.com/BEXIS2/Core/issues/2261))
+- **Citation API**: Add and add generic creation of citation formats 
+ ([#2258](https://github.com/BEXIS2/Core/issues/2258)) ([#2104](https://github.com/BEXIS2/Core/issues/2104))