#2283 fix issue with textmarker during data structure suggestion

Author: DavidBlaa

Components/IO/BExIS.IO.Tests/Transform/Input/AsciiReaderTest.cs

@@ -121,6 +121,29 @@ public void rowToList_RowAsQuotesAndSeperatorInQuotes_ReturnExpectedListOfString
             Assert.That(values, Is.EquivalentTo(expectedOutcome));
         }
 
+        [Test]
+        public void rowToList_RowAsOneVariablewithQuotesAndSeperatorInQuotes_ReturnExpectedListOfStrings()
+        {
+            //Arrange
+            string row = "V1,V2,'V3,V4'";
+            List<string> expectedOutcome = new List<string> { "V1", "V2", "V3,V4" };
+
+            AsciiFileReaderInfo info = new AsciiFileReaderInfo();
+            info.Seperator = TextSeperator.comma;
+
+            AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
+
+            //Act
+
+            List<string> values = reader.rowToList(row,
+                AsciiFileReaderInfo.GetSeperator(TextSeperator.comma));
+
+            //Assert
+            Assert.That(values.Count, Is.EqualTo(expectedOutcome.Count));
+            Assert.That(values, Is.EquivalentTo(expectedOutcome));
+        }
+
+
         [Test]
         public void ValidateRow_runNotValid_LimitErrors()
         {

Components/IO/BExIS.IO.Tests/Transform/Input/StructureAnalyzerTest.cs

@@ -249,7 +249,7 @@ public void SuggestSystemTypes_Valid_ResultWithCorrectTypes(int n)
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n),TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -294,7 +294,7 @@ public void SuggestSystemTypes_WithTestData_ResultWithCorrectTypes()
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows, TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -321,7 +321,7 @@ public void SuggestSystemTypes_ValidDateTypes_ResultWithCorrectTypes(int n)
             dateValues.Add("2022-12-24"); // yyyy-MM-dd
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
+            var result = structureAnalyser.SuggestSystemTypes(rows.GetRange(0, n), TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, new List<string>());
 
             //Assert
             Assert.NotNull(result);
@@ -334,7 +334,7 @@ public void SuggestSystemTypes_ValidWithMissingValues_ResultWithCorrectTypes()
             StructureAnalyser structureAnalyser = new StructureAnalyser();
 
             //Act
-            var result = structureAnalyser.SuggestSystemTypes(rowWithMissingValues, TextSeperator.semicolon, DecimalCharacter.comma, missingValueList);
+            var result = structureAnalyser.SuggestSystemTypes(rowWithMissingValues, TextMarker.doubleQuotes, TextSeperator.semicolon, DecimalCharacter.comma, missingValueList);
 
             //Assert
             Assert.NotNull(result);

Components/IO/BExIS.IO.Transform.Input/AsciiReader.cs

@@ -988,6 +988,28 @@ public List<string> rowToList(string line, char seperator)
             return line.Split(seperator).ToList();
         }
 
+        /// <summary>
+        /// Convert a row as a string to a list of strings based on offset,seperator and textmarker
+        /// </summary>
+        /// <remarks></remarks>
+        /// <seealso cref=""/>
+        /// <param name="line">Row as a string</param>
+        /// <param name="seperator">Character used as TextSeparator</param>
+        /// <param name="textMarker">Character used as TextMarker</param>
+        /// <param name="offset">offset to skip values from begin</param>
+        /// <returns>List of values</returns>
+        public List<string> RowToList(string line, char seperator, char textMarker, int offset = 0)
+        {
+            List<string> tempRow = new List<string>();
+
+            tempRow = new List<string>();
+            tempRow = TextMarkerHandling(line, seperator,textMarker);
+
+            //if a offset is marked in the file reader information the offset needs to skip from the complete string array
+            return tempRow.Skip(offset).ToList();
+            //return line.Split(seperator).ToList();
+        }
+
         private List<string> values;
         private List<string> temp;
 

Components/IO/BExIS.IO.Transform.Input/StructureAnalyser.cs

@@ -457,14 +457,20 @@ public List<string> GetRandowRows(List<string> rows, int numberOfRows)
             throw new NotImplementedException();
         }
 
-        public Dictionary<int, Type> SuggestSystemTypes(List<string> rows, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues)
+        public Dictionary<int, Type> SuggestSystemTypes(List<string> rows,TextMarker textMarker, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues)
         {
             Dictionary<int, List<Type>> source = new Dictionary<int, List<Type>>();
             Dictionary<int, Type> result = new Dictionary<int, Type>();
 
             // get type checks
             checks = getDataTypeChecks(decimalCharacter);
 
+            // validate input
+            AsciiFileReaderInfo info = new AsciiFileReaderInfo();
+            info.Seperator = delimeter;
+            info.TextMarker = textMarker;
+            AsciiReader reader = new AsciiReader(new StructuredDataStructure(), info);
+
             // create dictionary with types per column
             char seperator = AsciiFileReaderInfo.GetSeperator(delimeter);
             int numberOfRow = rows.First().Split(seperator).Count();
@@ -477,8 +483,11 @@ public Dictionary<int, Type> SuggestSystemTypes(List<string> rows, TextSeperator
             // go throw each row
             foreach (var row in rows)
             {
+                // 
+                var cells = reader.rowToList(row, seperator);
+
                 //split row based on sepeartor
-                foreach (var cell in row.Split(seperator).Select((x, i) => new { Value = x, Index = i }))
+                foreach (var cell in cells.Select((x, i) => new { Value = x, Index = i }))
                 {
                     //update possible list of types by check the value against the existing list
                     source[cell.Index] = checkValue(cell.Value, source[cell.Index], missingValues);

Console/BExIS.Web.Shell/Areas/RPM/BExIS.Modules.Rpm.UI/Controllers/DataStructureController.cs

@@ -441,6 +441,7 @@ public JsonResult Generate(DataStructureCreationModel model)
                 path,
                 AsciiFileReaderInfo.GetSeperator((char)model.Delimeter),
                 AsciiFileReaderInfo.GetDecimalCharacter((char)model.Decimal),
+                AsciiFileReaderInfo.GetTextMarker((char)model.TextMarker),
                 missingValues,
                 startdataIndex + 1
                 );
@@ -844,7 +845,7 @@ public JsonResult GetDWCRequirements()
         /// <param name="missingValues"></param>
         /// <param name="datastart">row not index!</param>
         /// <returns></returns>
-        private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator delimeter, DecimalCharacter decimalCharacter, List<string> missingValues, int datastart)
+        private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator delimeter, DecimalCharacter decimalCharacter, TextMarker textMarker, List<string> missingValues, int datastart)
         {
             var settings = ModuleManager.GetModuleSettings("Rpm");
             int min = Convert.ToInt32(settings.GetValueByKey("minToAnalyse"));
@@ -863,7 +864,7 @@ private Dictionary<int, Type> suggestSystemTypes(string file, TextSeperator deli
 
             List<string> rows = AsciiReader.GetRandowRows(file, total, selection, datastart);
 
-            return structureAnalyser.SuggestSystemTypes(rows, delimeter, decimalCharacter, missingValues);
+            return structureAnalyser.SuggestSystemTypes(rows, textMarker, delimeter, decimalCharacter, missingValues);
         }
 
         private string getValueFromMarkedRow(List<string> rows, List<Marker> markers, string type, char delimeter, int position, char textMarker)