Merge hotfix/v0.3.1: critical bug fixes for compressed data

Author: kolkov

.github/workflows/test.yml

@@ -9,6 +9,7 @@ name: Tests
 # Branch Strategy (Git Flow):
 # - feature/** branches: Development work
 # - release/** branches: Pre-release testing (test here BEFORE merging to main)
+# - hotfix/** branches: Critical bug fixes
 # - develop branch: Integration branch
 # - main branch: Production-ready code only (protected)
 # - Pull requests: Must pass all tests before merge
@@ -20,6 +21,7 @@ on:
       - develop
       - 'feature/**'
       - 'release/**'
+      - 'hotfix/**'
   pull_request:
     branches:
       - main

CHANGELOG.md

@@ -1,6 +1,36 @@
+## [0.3.1] - 2025-11-25
+
+### Fixed - Critical Bug Fixes (Hotfix)
+
+**Bug Fix #1: Compressed Data Support** (Critical):
+- **Issue**: Files with zlib-compressed data (miCOMPRESSED) returned 0 variables
+- **Impact**: Most real-world MATLAB files use compression - they were unreadable
+- **Fix**: Implemented full zlib decompression in `internal/v5/compressed.go`
+- **Security**: Added compression bomb protection (100MB limit, 1000:1 ratio max)
+
+**Bug Fix #2: Endianness Interpretation** (Critical):
+- **Issue**: "MI"/"IM" endian indicators were interpreted incorrectly
+- **Impact**: Wrong byte order caused garbage data or parse failures
+- **Fix**: Corrected interpretation in `internal/v5/header.go` and `writer.go`
+- **Note**: "IM" = little-endian, "MI" = big-endian (matches MATLAB spec)
+
+**Bug Fix #3: Small Format Tag Data** (Critical):
+- **Issue**: Small format tags (1-4 bytes) lost their embedded data
+- **Impact**: Variable names and small arrays were corrupted
+- **Fix**: Added `SmallData` field to `DataTag` struct in `internal/v5/data_tag.go`
+
+### Technical Details
+
+- Files affected: `header.go`, `writer.go`, `parser.go`, `data_tag.go`, `compressed.go`
+- All 298+ existing tests pass
+- Added test files: `inner_outer_tbl_param.mat` (12 vars), `energy_cascade_signals.mat` (34 vars)
+- Verified with real scientific data files from SciPy project
+
+---
+
 ## [0.3.0] - 2025-11-21
 
-### Added - Production Quality Release 🎉
+### Added - Production Quality Release
 
 **Functional Options Pattern**:
 - `WithEndianness(binary.ByteOrder)` - Set byte order for v5 files (little/big endian)

ROADMAP.md

@@ -2,7 +2,7 @@
 
 > **Strategic Approach**: Leverage existing HDF5 library and MATLAB documentation
 
-**Last Updated**: 2025-11-21 | **Current Version**: v0.3.0 (STABLE ✅) | **Target**: v1.0.0 stable (2026)
+**Last Updated**: 2025-11-25 | **Current Version**: v0.3.1 (HOTFIX ✅) | **Target**: v1.0.0 stable (2026)
 
 ---
 

internal/v5/compressed.go

@@ -2,16 +2,61 @@
 package v5
 
 import (
-	"errors"
+	"bytes"
+	"compress/zlib"
+	"fmt"
 	"io"
 )
 
-// ErrCompressedNotSupported indicates compressed data is not supported.
-var ErrCompressedNotSupported = errors.New("compressed MAT-files not yet supported")
+// maxDecompressedSize is the maximum allowed size after decompression (100MB).
+// This prevents compression bomb attacks (zip bombs).
+const maxDecompressedSize = 100 * 1024 * 1024 // 100MB
 
-// decompress would handle decompression (stub for future implementation).
+// maxCompressionRatio is the maximum allowed compression ratio.
+// Typical zlib compression achieves 2:1 to 10:1 ratios.
+// A ratio above 1000:1 suggests a potential zip bomb.
+const maxCompressionRatio = 1000
+
+// decompress decompresses zlib-compressed data from a MAT-file.
+// It reads compressedSize bytes from r and returns the decompressed content.
 //
-//nolint:unused // Future implementation stub
-func decompress(_ io.Reader) (io.Reader, error) {
-	return nil, ErrCompressedNotSupported
+// Security: Implements protection against compression bombs:
+// - Maximum decompressed size limit (100MB).
+// - Maximum compression ratio check (1000:1).
+func decompress(r io.Reader, compressedSize uint32) ([]byte, error) {
+	// Read compressed data
+	compressed := make([]byte, compressedSize)
+	if _, err := io.ReadFull(r, compressed); err != nil {
+		return nil, fmt.Errorf("failed to read compressed data: %w", err)
+	}
+
+	// Create zlib reader
+	zlibReader, err := zlib.NewReader(bytes.NewReader(compressed))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create zlib reader: %w", err)
+	}
+	defer zlibReader.Close() //nolint:errcheck // Best effort cleanup
+
+	// Read decompressed data with size limit
+	var decompressed bytes.Buffer
+	limited := io.LimitReader(zlibReader, maxDecompressedSize+1)
+	n, err := io.Copy(&decompressed, limited)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decompress data: %w", err)
+	}
+
+	// Check for size limit exceeded
+	if n > maxDecompressedSize {
+		return nil, fmt.Errorf("decompressed size exceeds limit: %d > %d bytes", n, maxDecompressedSize)
+	}
+
+	// Check compression ratio
+	if compressedSize > 0 {
+		ratio := float64(n) / float64(compressedSize)
+		if ratio > maxCompressionRatio {
+			return nil, fmt.Errorf("compression ratio too high: %.1f:1 (max %d:1)", ratio, maxCompressionRatio)
+		}
+	}
+
+	return decompressed.Bytes(), nil
 }

internal/v5/data_tag.go

@@ -12,9 +12,10 @@ const maxReasonableSize = 2 * 1024 * 1024 * 1024 // 2GB
 
 // DataTag represents a data element tag.
 type DataTag struct {
-	DataType uint32 // Data type identifier
-	Size     uint32 // Data size in bytes
-	IsSmall  bool   // True for small data elements
+	DataType  uint32 // Data type identifier
+	Size      uint32 // Data size in bytes
+	IsSmall   bool   // True for small data elements
+	SmallData []byte // For small format: data bytes (up to 4 bytes)
 }
 
 // readTag reads a data tag from the stream.
@@ -36,12 +37,16 @@ func (p *Parser) readTag() (*DataTag, error) {
 	// Lower 16 bits contain data type
 	size := firstWord >> 16
 	if size > 0 && size <= 4 {
-		// Small format
+		// Small format: data is packed in bytes 4-7 of the 8-byte tag
 		dataType := firstWord & 0xFFFF
+		// Copy the small data from bytes 4 to 4+size
+		smallData := make([]byte, size)
+		copy(smallData, buf[4:4+size])
 		return &DataTag{
-			DataType: dataType,
-			Size:     size,
-			IsSmall:  true,
+			DataType:  dataType,
+			Size:      size,
+			IsSmall:   true,
+			SmallData: smallData,
 		}, nil
 	}
 

internal/v5/header.go

@@ -22,10 +22,14 @@ func parseHeader(data []byte) (*Header, error) {
 	}
 
 	// Determine byte order
+	// The endian indicator is the 16-bit value 0x4D49 ("MI") written to bytes 126-127.
+	// On little-endian systems, this is stored as [0x49, 0x4D] which reads as "IM".
+	// On big-endian systems, this is stored as [0x4D, 0x49] which reads as "MI".
+	// So: "IM" → little-endian, "MI" → big-endian
 	switch hdr.EndianIndicator {
-	case "MI":
-		hdr.Order = binary.LittleEndian
 	case "IM":
+		hdr.Order = binary.LittleEndian
+	case "MI":
 		hdr.Order = binary.BigEndian
 	default:
 		return nil, errors.New("invalid endian indicator")

internal/v5/header_test.go

@@ -6,6 +6,11 @@ import (
 )
 
 func TestParseHeader(t *testing.T) {
+	// Note: Endian indicator interpretation:
+	// - "IM" = file created on little-endian system → use LittleEndian
+	// - "MI" = file created on big-endian system → use BigEndian
+	// This is because the 16-bit value 0x4D49 ("MI") is stored as [0x49, 0x4D]
+	// on little-endian systems, which reads as "IM".
 	tests := []struct {
 		name        string
 		header      []byte
@@ -17,46 +22,46 @@ func TestParseHeader(t *testing.T) {
 	}{
 		{
 			name:        "valid little endian v5",
-			header:      makeHeader("MATLAB 5.0 MAT-file", 0x0100, "MI"),
+			header:      makeHeader("MATLAB 5.0 MAT-file", 0x0100, "IM"),
 			wantDesc:    "MATLAB 5.0 MAT-file",
 			wantVersion: 0x0100,
-			wantEndian:  "MI",
+			wantEndian:  "IM",
 			wantOrder:   binary.LittleEndian,
 			wantErr:     false,
 		},
 		{
 			name:        "valid big endian v5",
-			header:      makeHeader("MATLAB 5.0 MAT-file", 0x0100, "IM"),
+			header:      makeHeader("MATLAB 5.0 MAT-file", 0x0100, "MI"),
 			wantDesc:    "MATLAB 5.0 MAT-file",
 			wantVersion: 0x0100,
-			wantEndian:  "IM",
+			wantEndian:  "MI",
 			wantOrder:   binary.BigEndian,
 			wantErr:     false,
 		},
 		{
 			name:        "description with trailing nulls",
-			header:      makeHeader("Test file\x00\x00\x00", 0x0100, "MI"),
+			header:      makeHeader("Test file\x00\x00\x00", 0x0100, "IM"),
 			wantDesc:    "Test file",
 			wantVersion: 0x0100,
-			wantEndian:  "MI",
+			wantEndian:  "IM",
 			wantOrder:   binary.LittleEndian,
 			wantErr:     false,
 		},
 		{
 			name:        "empty description",
-			header:      makeHeader("", 0x0100, "MI"),
+			header:      makeHeader("", 0x0100, "IM"),
 			wantDesc:    "",
 			wantVersion: 0x0100,
-			wantEndian:  "MI",
+			wantEndian:  "IM",
 			wantOrder:   binary.LittleEndian,
 			wantErr:     false,
 		},
 		{
 			name:        "v7.2 format",
-			header:      makeHeader("MATLAB 7.0 MAT-file", 0x0100, "MI"),
+			header:      makeHeader("MATLAB 7.0 MAT-file", 0x0100, "IM"),
 			wantDesc:    "MATLAB 7.0 MAT-file",
 			wantVersion: 0x0100,
-			wantEndian:  "MI",
+			wantEndian:  "IM",
 			wantOrder:   binary.LittleEndian,
 			wantErr:     false,
 		},
@@ -107,6 +112,7 @@ func TestParseHeader(t *testing.T) {
 // TestParseHeaderByteOrderVerification verifies that byte order is correctly detected
 // and used for version number parsing.
 func TestParseHeaderByteOrderVerification(t *testing.T) {
+	// Note: "IM" = little-endian, "MI" = big-endian
 	tests := []struct {
 		name        string
 		endian      string
@@ -115,19 +121,19 @@ func TestParseHeaderByteOrderVerification(t *testing.T) {
 	}{
 		{
 			name:        "little endian version parsing",
-			endian:      "MI",
+			endian:      "IM",
 			version:     0x0100,
 			wantVersion: 0x0100,
 		},
 		{
 			name:        "big endian version parsing",
-			endian:      "IM",
+			endian:      "MI",
 			version:     0x0100,
 			wantVersion: 0x0100,
 		},
 		{
 			name:        "little endian different version",
-			endian:      "MI",
+			endian:      "IM",
 			version:     0x0200,
 			wantVersion: 0x0200,
 		},
@@ -146,11 +152,12 @@ func TestParseHeaderByteOrderVerification(t *testing.T) {
 			}
 
 			// Verify byte order matches endian indicator
-			if tt.endian == "MI" && got.Order != binary.LittleEndian {
-				t.Error("Expected LittleEndian for 'MI' indicator")
+			// "IM" = little-endian, "MI" = big-endian
+			if tt.endian == "IM" && got.Order != binary.LittleEndian {
+				t.Error("Expected LittleEndian for 'IM' indicator")
 			}
-			if tt.endian == "IM" && got.Order != binary.BigEndian {
-				t.Error("Expected BigEndian for 'IM' indicator")
+			if tt.endian == "MI" && got.Order != binary.BigEndian {
+				t.Error("Expected BigEndian for 'MI' indicator")
 			}
 		})
 	}
@@ -164,7 +171,7 @@ func TestParseHeaderLongDescription(t *testing.T) {
 		longDesc = longDesc[:i] + "A" + longDesc[i+1:]
 	}
 
-	header := makeHeader(longDesc, 0x0100, "MI")
+	header := makeHeader(longDesc, 0x0100, "IM") // Use "IM" for little-endian
 	got, err := parseHeader(header)
 	if err != nil {
 		t.Fatalf("parseHeader() unexpected error: %v", err)
@@ -176,18 +183,20 @@ func TestParseHeaderLongDescription(t *testing.T) {
 }
 
 // makeHeader creates a test MAT-file header (128 bytes).
+// Note: "IM" = little-endian, "MI" = big-endian.
 func makeHeader(desc string, version uint16, endian string) []byte {
 	header := make([]byte, 128)
 
 	// Description (bytes 0-115)
 	copy(header, desc)
 
 	// Determine byte order from endian indicator
+	// "IM" = little-endian, "MI" = big-endian
 	var order binary.ByteOrder
 	switch endian {
-	case "MI":
-		order = binary.LittleEndian
 	case "IM":
+		order = binary.LittleEndian
+	case "MI":
 		order = binary.BigEndian
 	default:
 		// For invalid endian, use little endian but write invalid indicator
@@ -205,7 +214,7 @@ func makeHeader(desc string, version uint16, endian string) []byte {
 
 // BenchmarkParseHeader benchmarks header parsing performance.
 func BenchmarkParseHeader(b *testing.B) {
-	header := makeHeader("MATLAB 5.0 MAT-file", 0x0100, "MI")
+	header := makeHeader("MATLAB 5.0 MAT-file", 0x0100, "IM") // Use "IM" for little-endian
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {