Merge pull request #68 from rzymek/inmem-zipfile

drastically improve reader memory consumption when using InputStream

Author: Olivier Chédru

fastexcel-reader/src/main/java/org/dhatim/fastexcel/reader/ReadableWorkbook.java

@@ -27,13 +27,18 @@
 import java.util.stream.StreamSupport;
 import javax.xml.stream.XMLInputFactory;
 import javax.xml.stream.XMLStreamException;
+
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.utils.SeekableInMemoryByteChannel;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
 import org.apache.poi.openxml4j.exceptions.NotOfficeXmlFileException;
 import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
 import org.apache.poi.openxml4j.opc.OPCPackage;
 import org.apache.poi.openxml4j.opc.PackageAccess;
+import org.apache.poi.openxml4j.util.ZipFileZipEntrySource;
 import org.apache.poi.poifs.common.POIFSConstants;
 import org.apache.poi.poifs.storage.HeaderBlockConstants;
+import org.apache.poi.util.IOUtils;
 import org.apache.poi.util.LittleEndian;
 import org.apache.poi.xssf.eventusermodel.XSSFReader;
 import org.apache.poi.xssf.model.SharedStringsTable;
@@ -53,7 +58,8 @@ public ReadableWorkbook(File inputFile) throws IOException {
     }
 
     /**
-     * Note: when working with huge sheets (e.g. 500_000 rows) use {@link #ReadableWorkbook(File)}
+     * Note: will load the whole xlsx file into memory,
+     * (but will not uncompress it in memory)
      */
     public ReadableWorkbook(InputStream inputStream) throws IOException {
         this(open(inputStream));
@@ -197,7 +203,9 @@ private static OPCPackage open(File file){
 
     private static OPCPackage open(InputStream in) throws IOException {
         try {
-            return OPCPackage.open(in);
+            byte[] compressedBytes = IOUtils.toByteArray(in);
+            ZipFile zipFile = new ZipFile(new SeekableInMemoryByteChannel(compressedBytes));
+            return OPCPackage.open(new ZipFileZipEntrySource(zipFile));
         } catch (InvalidFormatException e) {
             throw new ExcelReaderException(e);
         }

fastexcel-reader/src/test/java/org/dhatim/fastexcel/reader/MemoryUsageTest.java

@@ -6,21 +6,17 @@
 import org.apache.poi.xssf.streaming.SXSSFSheet;
 import org.apache.poi.xssf.streaming.SXSSFWorkbook;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
+import java.io.*;
 import java.util.logging.Logger;
 import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class MemoryUsageTest {
     private static final Logger LOG = Logger.getLogger(MemoryUsageTest.class.getName());
-    private static final int ROWS = 600_000;
+    private static final int ROWS = 600_001;
     private static final int COLS = 200;
     private static File testFile = new File("target/memtest" + ROWS + "x" + COLS + ".xlsx");
 
@@ -48,33 +44,42 @@ public static void generateBig() throws IOException {
         LOG.info("Size: " + testFile.length());
     }
 
-    @BeforeEach
-    public void disableZipBombDetection() {
+    @Test
+    public void readFile() throws Exception {
         ZipSecureFileWorkaround.disableZipBombDetection();
+        try (ReadableWorkbook wb = new ReadableWorkbook(testFile)) {
+            fastexcelReader(wb);
+        }
     }
 
     @Test
-    public void read() throws Exception {
-        try (ReadableWorkbook wb = new ReadableWorkbook(testFile)) {
-            org.dhatim.fastexcel.reader.Sheet sheet = wb.getFirstSheet();
-            try (Stream<org.dhatim.fastexcel.reader.Row> rows = sheet.openStream()) {
-                rows.forEach(r -> {
-                    printProgress("reading", r.getRowNum() - 1);
-                    for (int c = 0; c < r.getCellCount(); c++) {
-                        assertEquals(
-                                valueFor(r.getRowNum() - 1, c),
-                                r.getCell(c).asNumber().doubleValue(),
-                                1e-5);
+    public void readInputStream() throws Exception {
+        try (InputStream in = new FileInputStream(testFile);
+             ReadableWorkbook wb = new ReadableWorkbook(in)
+        ) {
+            fastexcelReader(wb);
+        }
+    }
 
-                    }
-                });
-            }
+    private void fastexcelReader(ReadableWorkbook wb) throws IOException {
+        Sheet sheet = wb.getFirstSheet();
+        try (Stream<Row> rows = sheet.openStream()) {
+            rows.forEach(r -> {
+                printProgress("reading", r.getRowNum() - 1);
+                for (int c = 0; c < r.getCellCount(); c++) {
+                    assertEquals(
+                            valueFor(r.getRowNum() - 1, c),
+                            r.getCell(c).asNumber().doubleValue(),
+                            1e-5);
+
+                }
+            });
         }
     }
 
     private static void printProgress(String prefix, int r) {
         if (r % (ROWS / 100) == 0) {
-            LOG.info(prefix+": "+(100 * r / ROWS) + "%");
+            LOG.info(prefix + ": " + (100 * r / ROWS) + "%");
         }
     }