Merge pull request #66 from rzymek/support-sheet.xml-over4gb

writer support for xlsx with sheets.xml over 4GB

Author: Olivier Chédru

fastexcel-writer/pom.xml

@@ -1,18 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
-    
+
     <parent>
         <groupId>org.dhatim</groupId>
         <artifactId>fastexcel-parent</artifactId>
         <version>0-SNAPSHOT</version>
     </parent>
-    
+
     <artifactId>fastexcel</artifactId>
     <name>Fastexcel Writer</name>
     <packaging>jar</packaging>
 
     <dependencies>
+        <dependency>
+            <groupId>com.github.rzymek</groupId>
+            <artifactId>opczip</artifactId>
+            <version>1.0.0</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.poi</groupId>
             <artifactId>poi-ooxml</artifactId>
@@ -52,7 +57,7 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-    
+
     <build>
         <plugins>
             <plugin>

fastexcel-writer/src/main/java/org/dhatim/fastexcel/Workbook.java

@@ -15,15 +15,15 @@
  */
 package org.dhatim.fastexcel;
 
+import com.github.rzymek.opczip.OpcOutputStream;
+
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.charset.Charset;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 /**
  * A {@link Workbook} contains one or more {@link Worksheet} objects.
@@ -35,7 +35,7 @@ public class Workbook {
     private final List<Worksheet> worksheets = new ArrayList<>();
     private final StringCache stringCache = new StringCache();
     private final StyleCache styleCache = new StyleCache();
-    private final ZipOutputStream os;
+    private final OpcOutputStream os;
     private final Writer writer;
 
     /**
@@ -50,7 +50,7 @@ public class Workbook {
      * page</a> for details.
      */
     public Workbook(OutputStream os, String applicationName, String applicationVersion) {
-        this.os = new ZipOutputStream(os, Charset.forName("UTF-8"));
+        this.os = new OpcOutputStream(os);
         /* Tests showed that:
          * The default (-1) is level 6
          * Level 4 gives best size and very good time

fastexcel-writer/src/test/java/org/apache/poi/openxml4j/util/ZipSecureFileGateway.java

@@ -0,0 +1,15 @@
+package org.apache.poi.openxml4j.util;
+
+public class ZipSecureFileGateway {
+    /**
+     * Suppresses the following exception:
+     * "java.io.IOException: Zip bomb detected! The file would exceed the max size of the expanded data in the zip-file.
+     * This may indicates that the file is used to inflate memory usage and thus could pose a security risk.
+     * You can adjust this limit via ZipSecureFile.setMaxEntrySize() if you need to work with files which are very large.
+     * Uncompressed size: 4294970702, Raw/compressed size: 492446720
+     * Limits: MAX_ENTRY_SIZE: 4294967295, Entry: xl/worksheets/sheet1.xml"
+     */
+    public static void disableZipBombDetection() {
+        ZipSecureFile.MAX_ENTRY_SIZE = Long.MAX_VALUE;
+    }
+}

fastexcel-writer/src/test/java/org/dhatim/fastexcel/MemoryUsageTest.java

@@ -1,13 +1,16 @@
 package org.dhatim.fastexcel;
 
 import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.openxml4j.opc.PackageAccess;
+import org.apache.poi.openxml4j.util.ZipSecureFileGateway;
 import org.apache.poi.ss.usermodel.DataFormatter;
 import org.apache.poi.ss.util.CellReference;
 import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
 import org.apache.poi.xssf.eventusermodel.XSSFReader;
 import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler;
 import org.apache.poi.xssf.model.StylesTable;
 import org.apache.poi.xssf.usermodel.XSSFComment;
+import org.junit.Before;
 import org.junit.Test;
 import org.xml.sax.ContentHandler;
 import org.xml.sax.InputSource;
@@ -26,9 +29,9 @@
 
 public class MemoryUsageTest {
     private static final Logger LOG = Logger.getLogger(MemoryUsageTest.class.getName());
-    private static final int ROWS = 500_000;
-    private static final int COLS = 100;
-    private static final int SHEETS = 2;
+    private static final int ROWS = 600_000;
+    private static final int COLS = 200;
+    private static final int SHEETS = 1;
     private static final File FILE = new File("target/memtest" + ROWS + "x" + COLS + ".xlsx");
 
     private class SheetContentHandler implements XSSFSheetXMLHandler.SheetContentsHandler {
@@ -43,7 +46,7 @@ public SheetContentHandler(int sheetIndex) {
 
         @Override
         public void startRow(int rowNum) {
-            printProgress("validating", sheetIndex * ROWS + rowNum, SHEETS * ROWS);
+            printProgress("validating poi", sheetIndex * ROWS + rowNum, SHEETS * ROWS);
         }
 
         @Override
@@ -59,16 +62,21 @@ public void cell(String cellReference, String formattedValue, XSSFComment commen
         }
     }
 
+    @Before
+    public void disableZipBombDetection() {
+        ZipSecureFileGateway.disableZipBombDetection();
+    }
+
     @Test
     public void run() throws Exception {
         try (OutputStream out = new FileOutputStream(FILE)) {
             generate(out);
         }
-        validate();
+        validateWithPOI();
     }
 
-    private void validate() throws Exception {
-        try (OPCPackage pkg = OPCPackage.open(FILE)) {
+    private void validateWithPOI() throws Exception {
+        try (OPCPackage pkg = OPCPackage.open(FILE, PackageAccess.READ)) {
             ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(pkg);
             XSSFReader reader = new XSSFReader(pkg);
             StylesTable styles = reader.getStylesTable();