- [Bug] Fixed QueryAsDataTable read big file will throw NotImplementedException #360

shps951023 · shps951023 · commit f9d3a2b370bd · 2022-04-03T13:17:50.000+08:00
- [Doc] Reading big file by disk-base cache
diff --git a/README.md b/README.md
@@ -293,6 +293,32 @@ support variable length and width multi-row and column filling
 
 ![image](https://user-images.githubusercontent.com/12729184/117973820-6d2f1800-b35f-11eb-88d8-555063938108.png)
 
+#### 12. Reading big file by disk-base cache (Disk-Base Cache - SharedString)
+
+If the SharedStrings size exceeds 5 MB, MiniExcel default will use local disk cache, e.g, [10x100000.xlsx](https://github.com/MiniExcel/MiniExcel/files/8403819/NotDuplicateSharedStrings_10x100000.xlsx)(one million rows data), when disable disk cache the maximum memory usage is 195MB, but able disk cache only needs 65MB. Note, this optimization needs some efficiency cost, so this case will increase reading time from 7.4 seconds to 27.2 seconds, If you don't need it that you can disable disk cache with the following code:  
+
+```csharp
+var config = new OpenXmlConfiguration { EnableSharedStringCache = false };
+MiniExcel.Query(path,configuration: config)
+```
+
+You can use `SharedStringCacheSize ` to change the sharedString file size beyond the specified size for disk caching
+```csharp
+var config = new OpenXmlConfiguration { SharedStringCacheSize=500*1024*1024 };
+MiniExcel.Query(path, configuration: config);
+```
+
+
+![image](https://user-images.githubusercontent.com/12729184/161411851-1c3f72a7-33b3-4944-84dc-ffc1d16747dd.png)
+
+![image](https://user-images.githubusercontent.com/12729184/161411825-17f53ec7-bef4-4b16-b234-e24799ea41b0.png)
+
+
+
+
+
+
+
 
 
 ### Create/Export Excel  <a name="getstart2"></a>
diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -296,6 +296,30 @@ MiniExcel.Query(path,useHeaderRow:true,startCell:"B3")
 
 
 
+#### 12. 读取大文件硬盘缓存 (Disk-Base Cache - SharedString)
+
+概念 : MiniExcel 当判断文件 SharedString 大小超过 5MB，预设会使用本地缓存，如 [10x100000.xlsx](https://github.com/MiniExcel/MiniExcel/files/8403819/NotDuplicateSharedStrings_10x100000.xlsx)(一百万笔数据)，读取不开启本地缓存需要最高内存使用约195MB，开启后降为65MB。但要特别注意，此优化是以`时间换取内存减少`，所以读取效率会变慢，此例子读取时间从 7.4 秒提高到 27.2 秒，假如不需要能用以下代码关闭硬盘缓存
+
+```csharp
+var config = new OpenXmlConfiguration { EnableSharedStringCache = false };
+MiniExcel.Query(path,configuration: config)
+```
+
+也能使用 SharedStringCacheSize 调整 sharedString 文件大小超过指定大小才做硬盘缓存
+```csharp
+var config = new OpenXmlConfiguration { SharedStringCacheSize=500*1024*1024 };
+MiniExcel.Query(path, configuration: config);
+```
+
+
+![image](https://user-images.githubusercontent.com/12729184/161411851-1c3f72a7-33b3-4944-84dc-ffc1d16747dd.png)
+
+![image](https://user-images.githubusercontent.com/12729184/161411825-17f53ec7-bef4-4b16-b234-e24799ea41b0.png)
+
+
+
+
+
 ### 写/导出 Excel  <a name="getstart2"></a>
 
 1. 必须是非abstract 类别有公开无参数构造函数
diff --git a/README.zh-Hant.md b/README.zh-Hant.md
@@ -295,6 +295,25 @@ MiniExcel.Query(path,useHeaderRow:true,startCell:"B3")
 ![image](https://user-images.githubusercontent.com/12729184/117973820-6d2f1800-b35f-11eb-88d8-555063938108.png)
 
 
+#### 12. 讀取大文件硬碟緩存 (Disk-Base Cache - SharedString)
+
+概念 : MiniExcel 當判斷文件 SharedString 大小超過 5MB，預設會使用本地緩存，如 [10x100000.xlsx](https://github.com/MiniExcel/MiniExcel/files/8403819/NotDuplicateSharedStrings_10x100000.xlsx)(一百萬筆數據)，讀取不開啟本地緩存需要最高記憶體使用約195MB，開啟後降為65MB。但要特別注意，此優化是以`時間換取記憶體減少`，所以讀取效率會變慢，此例子讀取時間從 7.4 秒提高到 27.2 秒，假如不需要能用以下代碼關閉硬碟緩存
+
+```csharp
+var config = new OpenXmlConfiguration { EnableSharedStringCache = false };
+MiniExcel.Query(path,configuration: config)
+```
+
+也能使用 SharedStringCacheSize 調整 sharedString 文件大小超過指定大小才做硬碟緩存
+```csharp
+var config = new OpenXmlConfiguration { SharedStringCacheSize=500*1024*1024 };
+MiniExcel.Query(path, configuration: config);
+```
+
+
+![image](https://user-images.githubusercontent.com/12729184/161411851-1c3f72a7-33b3-4944-84dc-ffc1d16747dd.png)
+
+![image](https://user-images.githubusercontent.com/12729184/161411825-17f53ec7-bef4-4b16-b234-e24799ea41b0.png)
 
 
 
diff --git a/docs/README.md b/docs/README.md
@@ -22,6 +22,9 @@
 
 ---
 
+### 1.25.1
+- [Bug] Fixed QueryAsDataTable read big file will throw NotImplementedException #360
+
 ### 1.25.0
 - [New] Support SharingStrings disk cache (when this file size >= 5 MB), it can reduce reading 2GB SharingStrings only needs 1~13 MB memory #117
 - [New] SaveAs support overwriteFile parameter for enable/unable overwriting exist file #307
diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md
@@ -32,8 +32,10 @@
 
 ---
 
-### 1.25.0
+### 1.25.1
+- [Bug] 修正 QueryAsDataTable 读取大文件会抛出 NotImplementedException #360
 
+### 1.25.0
 - [New] 支持 SharingStrings disk cache (文件大小 >= 5 MB)，现在读取 2GB SharingStrings 只需要使用 1~13MB 内存 #117
 - [New] SaveAs 支持 overwriteFile 参数，方便调整是否要覆盖已存在文件。 #307
 - [Bug] SaveAs by datareader， 有时会多一个 autoFilter column #352
diff --git a/docs/README.zh-Hant.md b/docs/README.zh-Hant.md
@@ -24,6 +24,9 @@
 
 ---
 
+### 1.25.1
+- [Bug] 修正 QueryAsDataTable 讀取大文件會拋出 NotImplementedException #360
+
 ### 1.25.0
 - [New] 支持 SharingStrings disk cache (當該文件大小 >= 5 MB)，現在讀取 2GB SharingStrings 只需要使用 1~13MB 記憶體 #117
 - [New] SaveAs 支持 overwriteFile 參數，方便調整是否要覆蓋已存在文件。 #307
diff --git a/samples/xlsx/NotDuplicateSharedStrings_10x100.xlsx b/samples/xlsx/NotDuplicateSharedStrings_10x100.xlsx
diff --git a/samples/xlsx/NotDuplicateSharedStrings_10x100000.xlsx b/samples/xlsx/NotDuplicateSharedStrings_10x100000.xlsx
diff --git a/src/MiniExcel/MiniExcelLibs.csproj b/src/MiniExcel/MiniExcelLibs.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 	<PropertyGroup>
 		<TargetFrameworks>net45;netstandard2.0;net5.0</TargetFrameworks>
-		<Version>1.25.0</Version>
+		<Version>1.25.1</Version>
 	</PropertyGroup>
 	<PropertyGroup>
 		<AssemblyName>MiniExcel</AssemblyName>
diff --git a/src/MiniExcel/OpenXml/OpenXmlConfiguration.cs b/src/MiniExcel/OpenXml/OpenXmlConfiguration.cs
@@ -12,6 +12,6 @@ public class OpenXmlConfiguration : Configuration
         public bool EnableConvertByteArray { get; set; } = true;
         public bool IgnoreTemplateParameterMissing { get; set; } = true;
         public bool EnableSharedStringCache { get; set; } = true;
-        public int SharedStringCacheSize { get; set; } = 5 * 1024 * 1024;
+        public long SharedStringCacheSize { get; set; } = 5 * 1024 * 1024;
     }
 }
diff --git a/src/MiniExcel/OpenXml/SharedStringsDiskCache.cs b/src/MiniExcel/OpenXml/SharedStringsDiskCache.cs
@@ -17,12 +17,12 @@ internal class SharedStringsDiskCache : IDictionary<int, string>,IDisposable
 
         public ICollection<string> Values => throw new NotImplementedException();
 
-        public int Count => throw new NotImplementedException();
+        public int Count => (int)(_maxIndx+1); 
 
         public bool IsReadOnly => throw new NotImplementedException();
 
         public string this[int key] { get => GetValue(key); set => Add(key,value); }
-        private long _maxIndx = 0;
+        private long _maxIndx = -1;
         public bool ContainsKey(int key)
         {
             return key <= _maxIndx;
@@ -134,12 +134,14 @@ public bool Remove(KeyValuePair<int, string> item)
 
         public IEnumerator<KeyValuePair<int, string>> GetEnumerator()
         {
-            throw new NotImplementedException();
+            for (int i = 0; i < _maxIndx; i++)
+                yield return new KeyValuePair<int, string>(i, this[i]);
         }
 
         IEnumerator IEnumerable.GetEnumerator()
         {
-            throw new NotImplementedException();
+            for (int i = 0; i < _maxIndx; i++)
+                yield return this[i];
         }
 
         void IDictionary<int, string>.Add(int key, string value)
diff --git a/tests/MiniExcelTests/MiniExcelIssueTests.cs b/tests/MiniExcelTests/MiniExcelIssueTests.cs
@@ -34,19 +34,36 @@ public MiniExcelIssueTests(ITestOutputHelper output)
             this.output = output;
         }
 
+        [Fact]
+        public void TestIssue360()
+        {
+            var path = PathHelper.GetFile("xlsx/NotDuplicateSharedStrings_10x100.xlsx");
+            var config = new OpenXmlConfiguration { SharedStringCacheSize = 1 };
+            var sheets = MiniExcel.GetSheetNames(path);
+            foreach (var sheetName in sheets)
+            {
+                var dt = MiniExcel.QueryAsDataTable(path, useHeaderRow: true, sheetName: sheetName, configuration: config);
+            }
+        }
+
         [Fact]
         public void TestIssue117()
         {
             {
                 var cache = new SharedStringsDiskCache();
-                for (int i = 0; i < 1000000; i++)
+                for (int i = 0; i < 100; i++)
                 {
-                    cache[i]= i.ToString();
+                    cache[i] = i.ToString();
                 }
-                for (int i = 0; i < 1000000; i++)
+                for (int i = 0; i < 100; i++)
                 {
                     Assert.Equal(i.ToString(), cache[i]);
                 }
+                Assert.Equal(100, cache.Count);
+            }
+            {
+                var cache = new SharedStringsDiskCache();
+                Assert.Empty(cache);
             }
         }
 
@@ -77,7 +94,7 @@ public void TestIssue352()
                 }
                 var path = Path.GetTempPath() + Guid.NewGuid() + ".xlsx";
                 DataTableReader reader = table.CreateDataReader();
-                MiniExcel.SaveAs(path, reader,false);
+                MiniExcel.SaveAs(path, reader, false);
                 var xml = Helpers.GetZipFileContent(path, "xl/worksheets/sheet1.xml");
                 var cnt = Regex.Matches(xml, "<x:autoFilter ref=\"A1:B2\" />").Count;
             }

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,6 @@ public class OpenXmlConfiguration : Configuration`
`12`	`12`	`public bool EnableConvertByteArray { get; set; } = true;`
`13`	`13`	`public bool IgnoreTemplateParameterMissing { get; set; } = true;`
`14`	`14`	`public bool EnableSharedStringCache { get; set; } = true;`
`15`		`- public int SharedStringCacheSize { get; set; } = 5 * 1024 * 1024;`
	`15`	`+ public long SharedStringCacheSize { get; set; } = 5 * 1024 * 1024;`
`16`	`16`	`}`
`17`	`17`	`}`