Stream WriteTo directly to writer and fix ZIP64 local file header sizes (B4)

excelize.go

@@ -15,6 +15,7 @@ package excelize
 import (
 	"archive/zip"
 	"bytes"
+	"compress/flate"
 	"encoding/xml"
 	"io"
 	"io/fs"
@@ -65,6 +66,27 @@ type File struct {
 	ZipWriter        func(io.Writer) ZipWriter
 }
 
+// Compression defines the compression level for the ZIP archive used to store
+// the spreadsheet.
+type Compression int
+
+const (
+	// CompressionDefault uses standard deflate compression (the default).
+	// This produces the smallest files but uses more CPU and memory during
+	// Save/WriteTo.
+	CompressionDefault Compression = iota
+	// CompressionNone disables ZIP compression entirely. The spreadsheet
+	// parts are stored uncompressed. This significantly reduces CPU time and
+	// memory usage during Save/WriteTo at the cost of larger output files
+	// (typically 5-10x larger). Recommended for memory-constrained
+	// environments or when the output will be compressed by another layer.
+	CompressionNone
+	// CompressionBestSpeed uses the fastest deflate compression level. This
+	// is a good middle ground: roughly 2x faster than default compression
+	// with only moderately larger output.
+	CompressionBestSpeed
+)
+
 // ZipWriter defines an interface for writing files to a ZIP archive. It
 // provides methods to create new files within the archive, add files from a
 // filesystem, and close the archive when writing is complete.
@@ -122,6 +144,26 @@ type Options struct {
 	LongDatePattern   string
 	LongTimePattern   string
 	CultureInfo       CultureName
+	// StreamingChunkSize is the number of bytes of XML data accumulated in
+	// memory before a streaming worksheet spills to a temp file. A smaller
+	// value reduces peak memory usage at the cost of more disk I/O. Zero
+	// means use the default (StreamChunkSize = 16 MiB). Set to -1 to
+	// disable temp files entirely (all data stays in memory); this
+	// eliminates disk I/O overhead and can be significantly faster when
+	// sufficient memory is available.
+	StreamingChunkSize int
+	// StreamingBufSize is the size of the bufio.Writer used for all disk
+	// writes after the StreamingChunkSize threshold is crossed. Larger values
+	// reduce write syscall counts at the cost of slightly more memory. The
+	// measured inflection point on NVMe and HDD alike is 128 KiB. Zero means
+	// use the default (StreamingBufSizeDefault = 128 KiB).
+	StreamingBufSize int
+	// Compression specifies the compression level for the output ZIP
+	// archive. The default (CompressionDefault) uses standard deflate. Use
+	// CompressionNone in memory-constrained environments like AWS Lambda to
+	// eliminate compressor overhead, or CompressionBestSpeed for a balance
+	// of speed and size.
+	Compression Compression
 }
 
 // OpenFile take the name of a spreadsheet file and returns a populated
@@ -261,6 +303,31 @@ func (f *File) CharsetTranscoder(fn func(charset string, input io.Reader) (rdr i
 // SetZipWriter set user defined zip writer function for saving the workbook.
 func (f *File) SetZipWriter(fn func(io.Writer) ZipWriter) *File { f.ZipWriter = fn; return f }
 
+// configureZipCompression applies the Compression option to the zip writer.
+// It is a no-op for the default compression level or for custom ZipWriter
+// implementations that are not *zip.Writer.
+func (f *File) configureZipCompression(zw ZipWriter) {
+	if f.options == nil || f.options.Compression == CompressionDefault {
+		return
+	}
+	zipW, ok := zw.(*zip.Writer)
+	if !ok {
+		return
+	}
+	var level int
+	switch f.options.Compression {
+	case CompressionNone:
+		level = flate.NoCompression
+	case CompressionBestSpeed:
+		level = flate.BestSpeed
+	default:
+		return
+	}
+	zipW.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
+		return flate.NewWriter(out, level)
+	})
+}
+
 // Creates new XML decoder with charset reader.
 func (f *File) xmlNewDecoder(rdr io.Reader) (ret *xml.Decoder) {
 	ret = xml.NewDecoder(rdr)

file.go

@@ -113,7 +113,10 @@ func (f *File) Write(w io.Writer, opts ...Options) error {
 	return err
 }
 
-// WriteTo implements io.WriterTo to write the file.
+// WriteTo implements io.WriterTo to write the file. When no password
+// encryption is required, the ZIP archive is streamed directly to w without
+// buffering the entire compressed output in memory. When password encryption
+// is required, a temporary file is used to reduce memory usage.
 func (f *File) WriteTo(w io.Writer, opts ...Options) (int64, error) {
 	for i := range opts {
 		f.options = &opts[i]
@@ -127,18 +130,92 @@ func (f *File) WriteTo(w io.Writer, opts ...Options) (int64, error) {
 			return 0, err
 		}
 	}
-	buf, err := f.WriteToBuffer()
+	// Password encryption requires post-processing the entire output.
+	// Use a temporary file to reduce peak memory usage.
+	if f.options != nil && f.options.Password != "" {
+		return f.writeToWithEncryption(w)
+	}
+	// Stream the ZIP directly to w. This avoids holding the full compressed
+	// archive in a bytes.Buffer, which can be 50-200 MB+ for large reports.
+	cw := &countWriter{w: w}
+	f.zip64Entries = nil
+	zw := f.ZipWriter(cw)
+	f.configureZipCompression(zw)
+	if err := f.writeToZip(zw); err != nil {
+		_ = zw.Close()
+		return cw.n, err
+	}
+	return cw.n, zw.Close()
+}
+
+// writeToWithEncryption writes an encrypted file using a temporary file to
+// reduce memory usage.
+func (f *File) writeToWithEncryption(w io.Writer) (int64, error) {
+	var tmpDir string
+	if f.options != nil {
+		tmpDir = f.options.TmpDir
+	}
+	tmpFile, err := os.CreateTemp(tmpDir, "excelize-encrypt-*.zip")
+	if err != nil {
+		return 0, err
+	}
+	tmpPath := tmpFile.Name()
+	defer func() {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+	}()
+
+	f.zip64Entries = nil
+	zw := f.ZipWriter(tmpFile)
+	f.configureZipCompression(zw)
+	if err := f.writeToZip(zw); err != nil {
+		_ = zw.Close()
+		return 0, err
+	}
+	if err := zw.Close(); err != nil {
+		return 0, err
+	}
+
+	if len(f.zip64Entries) > 0 {
+		if err := f.writeZip64LFHFile(tmpFile); err != nil {
+			return 0, err
+		}
+	}
+
+	rawZip, err := os.ReadFile(tmpPath)
+	if err != nil {
+		return 0, err
+	}
+
+	encrypted, err := Encrypt(rawZip, f.options)
 	if err != nil {
 		return 0, err
 	}
-	return buf.WriteTo(w)
+	n, err := w.Write(encrypted)
+	return int64(n), err
+}
+
+// countWriter wraps an io.Writer and counts bytes written.
+type countWriter struct {
+	w io.Writer
+	n int64
+}
+
+func (cw *countWriter) Write(p []byte) (int, error) {
+	n, err := cw.w.Write(p)
+	cw.n += int64(n)
+	return n, err
 }
 
 // WriteToBuffer provides a function to get bytes.Buffer from the saved file,
 // and it allocates space in memory. Be careful when the file size is large.
+// Consider using WriteTo with a file for large password-protected files to
+// reduce memory usage.
 func (f *File) WriteToBuffer() (*bytes.Buffer, error) {
 	buf := new(bytes.Buffer)
+	f.zip64Entries = nil
 	zw := f.ZipWriter(buf)
+	f.configureZipCompression(zw)
 
 	if err := f.writeToZip(zw); err != nil {
 		_ = zw.Close()
@@ -147,7 +224,11 @@ func (f *File) WriteToBuffer() (*bytes.Buffer, error) {
 	if err := zw.Close(); err != nil {
 		return buf, err
 	}
-	err := f.writeZip64LFH(buf)
+	// Only perform ZIP64 fixup if we actually have ZIP64 entries
+	var err error
+	if len(f.zip64Entries) > 0 {
+		err = f.writeZip64LFH(buf)
+	}
 	if f.options != nil && f.options.Password != "" {
 		b, err := Encrypt(buf.Bytes(), f.options)
 		if err != nil {
@@ -180,13 +261,9 @@ func (f *File) writeToZip(zw ZipWriter) error {
 		if err != nil {
 			return err
 		}
-		var from io.Reader
-		if from, err = stream.rawData.Reader(); err != nil {
-			_ = stream.rawData.Close()
-			return err
-		}
-		written, err := io.Copy(fi, from)
+		written, err := stream.rawData.CopyTo(fi)
 		if err != nil {
+			_ = stream.rawData.Close()
 			return err
 		}
 		if written > math.MaxUint32 {
@@ -267,8 +344,82 @@ func (f *File) writeZip64LFH(buf *bytes.Buffer) error {
 		}
 		if inStrSlice(f.zip64Entries, string(data[idx+30:idx+30+filenameLen]), true) != -1 {
 			binary.LittleEndian.PutUint16(data[idx+4:idx+6], 45)
+			// Set compressed and uncompressed sizes to 0xFFFFFFFF to indicate
+			// that the actual sizes are in the ZIP64 extended information field.
+			// Without this, readers see size=0 or a truncated 32-bit value
+			// which causes corruption errors.
+			binary.LittleEndian.PutUint32(data[idx+18:idx+22], 0xFFFFFFFF)
+			binary.LittleEndian.PutUint32(data[idx+22:idx+26], 0xFFFFFFFF)
 		}
 		offset = idx + 1
 	}
 	return nil
 }
+
+// writeZip64LFHFile performs ZIP64 local file header fixup on a file.
+// This is used when encrypting to avoid loading the entire file into memory.
+func (f *File) writeZip64LFHFile(file *os.File) error {
+	if len(f.zip64Entries) == 0 {
+		return nil
+	}
+	if _, err := file.Seek(0, 0); err != nil {
+		return err
+	}
+	return f.fixZip64LFH(file, file)
+}
+
+// fixZip64LFH scans for ZIP local file headers and patches version and size
+// fields for entries in zip64Entries. Reading and writing are done through
+// separate interfaces so that error paths can be exercised in tests.
+func (f *File) fixZip64LFH(r io.ReaderAt, w io.WriterAt) error {
+	const chunkSize = 1024 * 1024 // 1MB chunks
+	buf := make([]byte, chunkSize)
+	var offset int64
+
+	for {
+		n, readErr := r.ReadAt(buf, offset)
+		if n > 0 {
+			searchBuf := buf[:n]
+			searchOffset := 0
+			for searchOffset < n {
+				idx := bytes.Index(searchBuf[searchOffset:], []byte{0x50, 0x4b, 0x03, 0x04})
+				if idx == -1 {
+					break
+				}
+				idx += searchOffset
+				absoluteIdx := offset + int64(idx)
+
+				if idx+30 > n {
+					break
+				}
+
+				filenameLen := int(binary.LittleEndian.Uint16(searchBuf[idx+26 : idx+28]))
+				if idx+30+filenameLen > n {
+					break
+				}
+
+				filename := string(searchBuf[idx+30 : idx+30+filenameLen])
+				if inStrSlice(f.zip64Entries, filename, true) != -1 {
+					// Patch version to 45 and set sizes to 0xFFFFFFFF in-place
+					binary.LittleEndian.PutUint16(searchBuf[idx+4:idx+6], 45)
+					binary.LittleEndian.PutUint32(searchBuf[idx+18:idx+22], 0xFFFFFFFF)
+					binary.LittleEndian.PutUint32(searchBuf[idx+22:idx+26], 0xFFFFFFFF)
+					if _, err := w.WriteAt(searchBuf[idx:idx+26], absoluteIdx); err != nil {
+						return err
+					}
+				}
+				searchOffset = idx + 1
+			}
+		}
+
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return readErr
+		}
+		offset += int64(n) - 30
+	}
+
+	return nil
+}