fix(restore): improve diagnostics and error handling for GCS restore failures

- Add detailed restore plan logging showing each file's level, TXID range, and size
- Fix compaction error capture (was being swallowed due to variable scoping)
- Improve decode error message to include first file info for debugging EOF errors
- Add warning when restore plan starts from non-TXID-1 (seeded replica detection)
- Add GCS-specific fail-fast for truncated files (< header size)
- Add debug logging when opening LTX files from GCS

These changes help diagnose issue #858 where GCS restores fail with
"decode database: decode header: EOF" by providing better visibility
into the restore plan and catching truncated files early.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: corylanou

gs/replica_client.go

@@ -188,8 +188,27 @@ func (c *ReplicaClient) OpenLTXFile(ctx context.Context, level int, minTXID, max
 		return nil, err
 	}
 
+	// Log the actual size from GCS for debugging
+	actualSize := r.Attrs.Size
+	c.logger.Debug("opened ltx file from GCS",
+		"key", key,
+		"level", level,
+		"minTXID", minTXID,
+		"maxTXID", maxTXID,
+		"actualSize", actualSize,
+		"requestedOffset", offset,
+		"requestedLength", length)
+
+	// Fail fast if file is smaller than LTX header - it can never decode successfully.
+	// This catches truncated uploads or corrupted files early with a clear error.
+	if actualSize < ltx.HeaderSize {
+		_ = r.Close()
+		return nil, fmt.Errorf("gs: ltx file %q is truncated: size %d bytes (minimum %d for header)",
+			key, actualSize, ltx.HeaderSize)
+	}
+
 	internal.OperationTotalCounterVec.WithLabelValues(ReplicaClientType, "GET").Inc()
-	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(r.Attrs.Size))
+	internal.OperationBytesCounterVec.WithLabelValues(ReplicaClientType, "GET").Add(float64(actualSize))
 
 	return r, nil
 }

replica.go

@@ -425,6 +425,16 @@ func (r *Replica) Restore(ctx context.Context, opt RestoreOptions) (err error) {
 
 	r.Logger().Debug("restore plan", "n", len(infos), "txid", infos[len(infos)-1].MaxTXID, "timestamp", infos[len(infos)-1].CreatedAt)
 
+	// Log detailed restore plan for debugging
+	for i, info := range infos {
+		r.Logger().Debug("restore plan file",
+			"index", i,
+			"level", info.Level,
+			"min", info.MinTXID,
+			"max", info.MaxTXID,
+			"size", info.Size)
+	}
+
 	rdrs := make([]io.Reader, 0, len(infos))
 	defer func() {
 		for _, rd := range rdrs {
@@ -441,7 +451,7 @@ func (r *Replica) Restore(ctx context.Context, opt RestoreOptions) (err error) {
 				info.Level, info.MinTXID, info.MaxTXID, info.Size, ltx.HeaderSize)
 		}
 
-		r.Logger().Debug("opening ltx file for restore", "level", info.Level, "min", info.MinTXID, "max", info.MaxTXID)
+		r.Logger().Debug("opening ltx file for restore", "level", info.Level, "min", info.MinTXID, "max", info.MaxTXID, "size", info.Size)
 
 		// Add file to be compacted.
 		f, err := r.Client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0)
@@ -479,16 +489,24 @@ func (r *Replica) Restore(ctx context.Context, opt RestoreOptions) (err error) {
 	go func() {
 		c, err := ltx.NewCompactor(pw, rdrs)
 		if err != nil {
+			r.Logger().Error("compactor init failed", "error", err, "n_readers", len(rdrs))
 			pw.CloseWithError(fmt.Errorf("new ltx compactor: %w", err))
 			return
 		}
 		c.HeaderFlags = ltx.HeaderFlagNoChecksum
-		_ = pw.CloseWithError(c.Compact(ctx))
+		compactErr := c.Compact(ctx)
+		if compactErr != nil {
+			r.Logger().Error("compaction failed", "error", compactErr)
+		}
+		_ = pw.CloseWithError(compactErr)
 	}()
 
 	dec := ltx.NewDecoder(pr)
 	if err := dec.DecodeDatabaseTo(f); err != nil {
-		return fmt.Errorf("decode database: %w", err)
+		// Include first file info for debugging EOF errors
+		firstFile := infos[0]
+		return fmt.Errorf("decode database (first file: level=%d min=%s max=%s size=%d): %w",
+			firstFile.Level, firstFile.MinTXID, firstFile.MaxTXID, firstFile.Size, err)
 	}
 
 	if err := f.Sync(); err != nil {
@@ -574,5 +592,16 @@ func CalcRestorePlan(ctx context.Context, client ReplicaClient, txID ltx.TXID, t
 		return nil, ErrTxNotAvailable
 	}
 
+	// Warn if the first file doesn't start at TXID 1 - this may indicate missing
+	// base snapshot, but could also be a legitimate seeded replica starting from
+	// a later snapshot. Log for diagnostics but allow the restore to proceed.
+	firstFile := infos[0]
+	if firstFile.MinTXID != 1 {
+		logger.Warn("restore plan starts from non-initial TXID (may be seeded replica or missing base snapshot)",
+			"first_file_level", firstFile.Level,
+			"first_file_min", firstFile.MinTXID,
+			"first_file_max", firstFile.MaxTXID)
+	}
+
 	return infos, nil
 }