Skip to content

Commit 5b68094

Browse files
jehugaleahsajehugaleahsa
committed
FlatFileDataReader not correctly ignoring ignored columns in several places.
1 parent 480cdf2 commit 5b68094

File tree

5 files changed

+117
-16
lines changed

5 files changed

+117
-16
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
1+
## 4.12.0 (2020-10-21)
2+
**Summary** - `FlatFileDataReader` not correctly ignoring ignored columns in several places.
3+
4+
The ADO.NET classes didn't receive the same level of love that the rest of the library received when introducing ignored columns. When getting the column names, their ordinal positions, etc., the `FlatFileDataReader` was returning information for ignored columns. This caused the `DataTable` extensions to see too many column names and yet receive too few record values in the table (with the data shifted to the left for each missing column). Fixing the `FlatFileDataReader` fixed the `DataTable` problems, as well.
5+
6+
Technically this is a breaking change that might warrant a major version change; however, as the previous behavior could not possibly be desired and few people actually use the ADO.NET classes, I am going to include this in the next minor version, treating it as just a bug fix.
7+
18
## 4.11.0 (2020-10-09)
29
**Summary* - Allow handling unrecognized rows when using schema selectors.
310

‎FlatFiles.Test/DataTableExtensionsTester.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
using System;
2+
using System.Collections.Generic;
3+
using System.ComponentModel.DataAnnotations;
24
using System.Data;
35
using System.Globalization;
46
using System.IO;
57
using System.Linq;
8+
using System.Text;
69
using System.Threading;
710
using Microsoft.VisualStudio.TestTools.UnitTesting;
811

@@ -413,6 +416,39 @@ public void TestWriteFlatFile_ExtraColumn_Ignores()
413416
", output);
414417
}
415418

419+
[TestMethod]
420+
public void TestReadFlatFile_IgnoredColumns2()
421+
{
422+
const string data =
423+
@"A,B,C
424+
1,2,3
425+
4,5,6";
426+
var schema = new SeparatedValueSchema();
427+
schema.AddColumn(new StringColumn("A"));
428+
schema.AddColumn(new IgnoredColumn("Ignored"));
429+
schema.AddColumn(new StringColumn("C"));
430+
431+
var options = new SeparatedValueOptions()
432+
{
433+
IsFirstRecordSchema = true
434+
};
435+
436+
var textReader = new StringReader(data);
437+
var csvReader = new SeparatedValueReader(textReader, schema, options);
438+
439+
DataTable dataTable = new DataTable();
440+
dataTable.ReadFlatFile(csvReader);
441+
string[] columnNames = dataTable.Columns.OfType<DataColumn>()
442+
.Select(r => r.ColumnName)
443+
.ToArray();
444+
CollectionAssert.AreEqual(new[] { "A", "C" }, columnNames);
445+
Assert.AreEqual(2, dataTable.Rows.Count);
446+
object[] values1 = dataTable.Rows[0].ItemArray;
447+
CollectionAssert.AreEqual(new[] { "1", "3" }, values1);
448+
object[] values2 = dataTable.Rows[1].ItemArray;
449+
CollectionAssert.AreEqual(new[] { "4", "6" }, values2);
450+
}
451+
416452
[TestMethod]
417453
public void TestWriteFlatFile_IgnoredColumns()
418454
{

‎FlatFiles.Test/FlatFileDataReaderTester.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,5 +146,48 @@ private static FlatFileDataReader GetFlatFileReader()
146146
var dataReader = new FlatFileDataReader(csvReader);
147147
return dataReader;
148148
}
149+
150+
[TestMethod]
151+
public void TestFlatFileReader_IgnoreIgnoredColumns()
152+
{
153+
const string data =
154+
@"A,B,C
155+
1,2,3
156+
4,5,6";
157+
var schema = new SeparatedValueSchema();
158+
schema.AddColumn(new StringColumn("A"));
159+
schema.AddColumn(new IgnoredColumn("Ignored"));
160+
schema.AddColumn(new StringColumn("C"));
161+
162+
var options = new SeparatedValueOptions()
163+
{
164+
IsFirstRecordSchema = true
165+
};
166+
167+
var textReader = new StringReader(data);
168+
var csvReader = new SeparatedValueReader(textReader, schema, options);
169+
using (var dataReader = new FlatFileDataReader(csvReader))
170+
{
171+
Assert.AreEqual("A", dataReader.GetName(0));
172+
Assert.AreEqual("C", dataReader.GetName(1));
173+
Assert.AreEqual(0, dataReader.GetOrdinal("A"));
174+
Assert.AreEqual(-1, dataReader.GetOrdinal("B"));
175+
Assert.AreEqual(1, dataReader.GetOrdinal("C"));
176+
177+
var schemaTable = dataReader.GetSchemaTable();
178+
string[] columnNames = schemaTable.Rows.OfType<DataRow>()
179+
.Select(r => r.Field<string>("ColumnName"))
180+
.ToArray();
181+
CollectionAssert.AreEqual(new[] { "A", "C" }, columnNames);
182+
183+
Assert.IsTrue(dataReader.Read());
184+
object[] values1 = dataReader.GetValues();
185+
CollectionAssert.AreEqual(new[] { "1", "3" }, values1);
186+
Assert.IsTrue(dataReader.Read());
187+
object[] values2 = dataReader.GetValues();
188+
CollectionAssert.AreEqual(new[] { "4", "6" }, values2);
189+
Assert.IsFalse(dataReader.Read());
190+
}
191+
}
149192
}
150193
}

‎FlatFiles/FlatFileDataReader.cs‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Data;
44
using System.Data.Common;
55
using System.Globalization;
6+
using System.Linq;
67

78
namespace FlatFiles
89
{
@@ -60,6 +61,7 @@ public interface IFlatFileDataRecord : IDataRecord
6061
public sealed class FlatFileDataReader : IDataReader, IFlatFileDataRecord
6162
{
6263
private ISchema schema; // cached
64+
private ColumnCollection columns; // cached
6365
private object[] values; // cached
6466

6567
/// <summary>
@@ -124,9 +126,9 @@ public DataTable GetSchemaTable()
124126
{
125127
var schema = GetSchema();
126128
var schemaTable = GetEmptySchemaDataTable(schema);
127-
for (int index = 0; index != schema.ColumnDefinitions.Count; ++index)
129+
for (int index = 0, count = columns.Count; index != count; ++index)
128130
{
129-
var column = schema.ColumnDefinitions[index];
131+
var column = columns[index];
130132
object[] values = new object[]
131133
{
132134
true, // AllowDBNull
@@ -165,7 +167,7 @@ private static DataTable GetEmptySchemaDataTable(ISchema schema)
165167
{
166168
DataTable schemaTable = new DataTable();
167169
schemaTable.Locale = CultureInfo.InvariantCulture;
168-
schemaTable.MinimumCapacity = schema.ColumnDefinitions.Count;
170+
schemaTable.MinimumCapacity = schema.ColumnDefinitions.PhysicalCount;
169171
schemaTable.Columns.AddRange(new[]
170172
{
171173
new DataColumn(SchemaTableColumn.AllowDBNull, typeof(Boolean)),
@@ -229,7 +231,12 @@ public bool Read()
229231
/// <summary>
230232
/// Gets the number of fields in the current record.
231233
/// </summary>
232-
public int FieldCount => GetSchema().ColumnDefinitions.Count;
234+
public int FieldCount {
235+
get {
236+
GetSchema();
237+
return columns.Count;
238+
}
239+
}
233240

234241
/// <summary>
235242
/// Gets the boolean value from the current record at the given index.
@@ -322,8 +329,8 @@ IDataReader IDataRecord.GetData(int i)
322329
/// <returns>The type name.</returns>
323330
public string GetDataTypeName(int i)
324331
{
325-
var schema = GetSchema();
326-
return schema.ColumnDefinitions[i].ColumnType.Name;
332+
GetSchema();
333+
return columns[i].ColumnType.Name;
327334
}
328335

329336
/// <summary>
@@ -377,8 +384,8 @@ public double GetDouble(int i)
377384
/// <returns>The type of the value at the given index.</returns>
378385
public Type GetFieldType(int i)
379386
{
380-
var schema = GetSchema();
381-
return schema.ColumnDefinitions[i].ColumnType;
387+
GetSchema();
388+
return columns[i].ColumnType;
382389
}
383390

384391
/// <summary>
@@ -443,8 +450,8 @@ public long GetInt64(int i)
443450
/// <returns>The name of the column at the given index.</returns>
444451
public string GetName(int i)
445452
{
446-
var schema = GetSchema();
447-
return schema.ColumnDefinitions[i].ColumnName;
453+
GetSchema();
454+
return columns[i].ColumnName;
448455
}
449456

450457
/// <summary>
@@ -454,8 +461,8 @@ public string GetName(int i)
454461
/// <returns>The index of the column with the given name.</returns>
455462
public int GetOrdinal(string name)
456463
{
457-
var schema = GetSchema();
458-
return schema.GetOrdinal(name);
464+
GetSchema();
465+
return columns.GetOrdinal(name);
459466
}
460467

461468
/// <summary>
@@ -609,6 +616,14 @@ private ISchema GetSchema()
609616
if (schema == null)
610617
{
611618
schema = Reader.GetSchema();
619+
columns = new ColumnCollection();
620+
foreach (ColumnDefinition column in schema.ColumnDefinitions)
621+
{
622+
if (!column.IsIgnored)
623+
{
624+
columns.AddColumn(column);
625+
}
626+
}
612627
}
613628
return schema;
614629
}

‎FlatFiles/FlatFiles.csproj‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,17 @@
1010
<RepositoryUrl>https://github.com/jehugaleahsa/FlatFiles.git</RepositoryUrl>
1111
<RepositoryType>git</RepositoryType>
1212
<PackageTags>csv;comma;tab;separated;value;delimited;flat;file;fixed;width;fixed-width;length;fixed-length;parser;parsing;parse</PackageTags>
13-
<PackageReleaseNotes>Allow handling unrecognized rows when using schema selectors.</PackageReleaseNotes>
13+
<PackageReleaseNotes>FlatFileDataReader not correctly ignoring ignored columns in several places.</PackageReleaseNotes>
1414
<SignAssembly>true</SignAssembly>
1515
<AssemblyOriginatorKeyFile>FlatFiles.snk</AssemblyOriginatorKeyFile>
16-
<Version>4.11.0</Version>
16+
<Version>4.12.0</Version>
1717
</PropertyGroup>
1818

1919
<PropertyGroup>
2020
<LangVersion>8.0</LangVersion>
2121
<PackageIconUrl></PackageIconUrl>
22-
<AssemblyVersion>4.11.0.0</AssemblyVersion>
23-
<FileVersion>4.11.0.0</FileVersion>
22+
<AssemblyVersion>4.12.0.0</AssemblyVersion>
23+
<FileVersion>4.12.0.0</FileVersion>
2424
<PackageLicenseFile>UNLICENSE.txt</PackageLicenseFile>
2525
<PackageIcon>icon.png</PackageIcon>
2626
</PropertyGroup>

0 commit comments

Comments
 (0)