feat(goldfish): Add support for query results in object storage

pkg/goldfish/migrations/20251007000001_add_result_storage_columns.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+-- Add columns to persist result storage metadata for Goldfish
+
+ALTER TABLE sampled_queries
+  ADD COLUMN cell_a_result_uri TEXT NULL AFTER cell_a_status_code,
+  ADD COLUMN cell_b_result_uri TEXT NULL AFTER cell_a_result_uri,
+  ADD COLUMN cell_a_result_size_bytes BIGINT NULL AFTER cell_b_result_uri,
+  ADD COLUMN cell_b_result_size_bytes BIGINT NULL AFTER cell_a_result_size_bytes,
+  ADD COLUMN cell_a_result_compression VARCHAR(32) NULL AFTER cell_b_result_size_bytes,
+  ADD COLUMN cell_b_result_compression VARCHAR(32) NULL AFTER cell_a_result_compression;
+
+-- +goose Down
+-- Remove result storage metadata columns
+
+ALTER TABLE sampled_queries
+  DROP COLUMN cell_b_result_compression,
+  DROP COLUMN cell_a_result_compression,
+  DROP COLUMN cell_b_result_size_bytes,
+  DROP COLUMN cell_a_result_size_bytes,
+  DROP COLUMN cell_b_result_uri,
+  DROP COLUMN cell_a_result_uri;

pkg/goldfish/storage_mysql.go

@@ -92,6 +92,9 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 			cell_a_response_hash, cell_b_response_hash,
 			cell_a_response_size, cell_b_response_size,
 			cell_a_status_code, cell_b_status_code,
+			cell_a_result_uri, cell_b_result_uri,
+			cell_a_result_size_bytes, cell_b_result_size_bytes,
+			cell_a_result_compression, cell_b_result_compression,
 			cell_a_trace_id, cell_b_trace_id,
 			cell_a_span_id, cell_b_span_id,
 			cell_a_used_new_engine, cell_b_used_new_engine,
@@ -108,6 +111,25 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 		cellBSpanID = sample.CellBSpanID
 	}
 
+	// Prepare nullable result storage metadata
+	var cellAResultURI, cellBResultURI any
+	var cellAResultSize, cellBResultSize any
+	var cellAResultCompression, cellBResultCompression any
+	if sample.CellAResultURI != "" {
+		cellAResultURI = sample.CellAResultURI
+		cellAResultSize = sample.CellAResultSize
+		if sample.CellAResultCompression != "" {
+			cellAResultCompression = sample.CellAResultCompression
+		}
+	}
+	if sample.CellBResultURI != "" {
+		cellBResultURI = sample.CellBResultURI
+		cellBResultSize = sample.CellBResultSize
+		if sample.CellBResultCompression != "" {
+			cellBResultCompression = sample.CellBResultCompression
+		}
+	}
+
 	_, err := s.db.ExecContext(ctx, query,
 		sample.CorrelationID,
 		sample.TenantID,
@@ -142,6 +164,12 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 		sample.CellBResponseSize,
 		sample.CellAStatusCode,
 		sample.CellBStatusCode,
+		cellAResultURI,
+		cellBResultURI,
+		cellAResultSize,
+		cellBResultSize,
+		cellAResultCompression,
+		cellBResultCompression,
 		sample.CellATraceID,
 		sample.CellBTraceID,
 		cellASpanID,
@@ -214,6 +242,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 			cell_a_entries_returned, cell_b_entries_returned, cell_a_splits, cell_b_splits,
 			cell_a_shards, cell_b_shards, cell_a_response_hash, cell_b_response_hash,
 			cell_a_response_size, cell_b_response_size, cell_a_status_code, cell_b_status_code,
+			cell_a_result_uri, cell_b_result_uri,
+			cell_a_result_size_bytes, cell_b_result_size_bytes,
+			cell_a_result_compression, cell_b_result_compression,
 			cell_a_trace_id, cell_b_trace_id,
 			cell_a_span_id, cell_b_span_id,
 			cell_a_used_new_engine, cell_b_used_new_engine,
@@ -245,6 +276,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 		var createdAt time.Time
 		// Use sql.NullString for nullable span ID columns
 		var cellASpanID, cellBSpanID sql.NullString
+		var cellAResultURI, cellBResultURI sql.NullString
+		var cellAResultCompression, cellBResultCompression sql.NullString
+		var cellAResultSize, cellBResultSize sql.NullInt64
 
 		err := rows.Scan(
 			&q.CorrelationID, &q.TenantID, &q.User, &q.Query, &q.QueryType, &q.StartTime, &q.EndTime, &stepDurationMs,
@@ -254,6 +288,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 			&q.CellAStats.TotalEntriesReturned, &q.CellBStats.TotalEntriesReturned, &q.CellAStats.Splits, &q.CellBStats.Splits,
 			&q.CellAStats.Shards, &q.CellBStats.Shards, &q.CellAResponseHash, &q.CellBResponseHash,
 			&q.CellAResponseSize, &q.CellBResponseSize, &q.CellAStatusCode, &q.CellBStatusCode,
+			&cellAResultURI, &cellBResultURI,
+			&cellAResultSize, &cellBResultSize,
+			&cellAResultCompression, &cellBResultCompression,
 			&q.CellATraceID, &q.CellBTraceID,
 			&cellASpanID, &cellBSpanID,
 			&q.CellAUsedNewEngine, &q.CellBUsedNewEngine,
@@ -270,6 +307,24 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 		if cellBSpanID.Valid {
 			q.CellBSpanID = cellBSpanID.String
 		}
+		if cellAResultURI.Valid {
+			q.CellAResultURI = cellAResultURI.String
+		}
+		if cellBResultURI.Valid {
+			q.CellBResultURI = cellBResultURI.String
+		}
+		if cellAResultSize.Valid {
+			q.CellAResultSize = cellAResultSize.Int64
+		}
+		if cellBResultSize.Valid {
+			q.CellBResultSize = cellBResultSize.Int64
+		}
+		if cellAResultCompression.Valid {
+			q.CellAResultCompression = cellAResultCompression.String
+		}
+		if cellBResultCompression.Valid {
+			q.CellBResultCompression = cellBResultCompression.String
+		}
 
 		// Convert step duration from milliseconds to Duration
 		q.Step = time.Duration(stepDurationMs) * time.Millisecond

pkg/goldfish/types.go

@@ -32,6 +32,14 @@ type QuerySample struct {
 	CellASpanID       string `json:"cellASpanID"`
 	CellBSpanID       string `json:"cellBSpanID"`
 
+	// Result storage metadata
+	CellAResultURI         string `json:"cellAResultURI"`
+	CellBResultURI         string `json:"cellBResultURI"`
+	CellAResultSize        int64  `json:"cellAResultSize"`
+	CellBResultSize        int64  `json:"cellBResultSize"`
+	CellAResultCompression string `json:"cellAResultCompression"`
+	CellBResultCompression string `json:"cellBResultCompression"`
+
 	// Query engine version tracking
 	CellAUsedNewEngine bool `json:"cellAUsedNewEngine"`
 	CellBUsedNewEngine bool `json:"cellBUsedNewEngine"`

pkg/ui/goldfish.go

@@ -76,6 +76,14 @@ type SampledQuery struct {
 	CellAStatusCode   *int    `json:"cellAStatusCode" db:"cell_a_status_code"`
 	CellBStatusCode   *int    `json:"cellBStatusCode" db:"cell_b_status_code"`
 
+	// Result storage metadata - nullable when persistence is disabled
+	CellAResultURI         *string `json:"cellAResultURI,omitempty" db:"cell_a_result_uri"`
+	CellBResultURI         *string `json:"cellBResultURI,omitempty" db:"cell_b_result_uri"`
+	CellAResultSizeBytes   *int64  `json:"cellAResultSizeBytes,omitempty" db:"cell_a_result_size_bytes"`
+	CellBResultSizeBytes   *int64  `json:"cellBResultSizeBytes,omitempty" db:"cell_b_result_size_bytes"`
+	CellAResultCompression *string `json:"cellAResultCompression,omitempty" db:"cell_a_result_compression"`
+	CellBResultCompression *string `json:"cellBResultCompression,omitempty" db:"cell_b_result_compression"`
+
 	// Trace IDs - nullable as not all requests have traces
 	CellATraceID *string `json:"cellATraceID" db:"cell_a_trace_id"`
 	CellBTraceID *string `json:"cellBTraceID" db:"cell_b_trace_id"`
@@ -242,6 +250,25 @@ func (s *Service) GetSampledQueriesWithContext(ctx context.Context, page, pageSi
 			CellBUsedNewEngine: q.CellBUsedNewEngine,
 		}
 
+		if q.CellAResultURI != "" {
+			uiQuery.CellAResultURI = strPtr(q.CellAResultURI)
+			size := q.CellAResultSize
+			uiQuery.CellAResultSizeBytes = &size
+			if q.CellAResultCompression != "" {
+				comp := q.CellAResultCompression
+				uiQuery.CellAResultCompression = &comp
+			}
+		}
+		if q.CellBResultURI != "" {
+			uiQuery.CellBResultURI = strPtr(q.CellBResultURI)
+			size := q.CellBResultSize
+			uiQuery.CellBResultSizeBytes = &size
+			if q.CellBResultCompression != "" {
+				comp := q.CellBResultCompression
+				uiQuery.CellBResultCompression = &comp
+			}
+		}
+
 		// Determine comparison status based on response codes and hashes
 		if q.CellAStatusCode < 200 || q.CellAStatusCode >= 300 || q.CellBStatusCode < 200 || q.CellBStatusCode >= 300 {
 			uiQuery.ComparisonStatus = string(goldfish.ComparisonStatusError)

tools/querytee/goldfish/README.md

@@ -16,7 +16,8 @@ Goldfish is a feature within QueryTee that enables sampling and comparison of qu
   - Query complexity metrics (splits, shards)
   - Performance variance detection and reporting
 - **Query Engine Version Tracking**: Tracks which queries used the new experimental query engine vs the old engine
-- **Persistent Storage**: MySQL storage via Google Cloud SQL Proxy or Amazon RDS for storing query samples and comparison results
+- **Persistent Metadata Storage**: MySQL storage via Google Cloud SQL Proxy or Amazon RDS for storing query samples and comparison results
+- **Raw Result Persistence**: Optional upload of exact JSON responses to GCS or S3 for deep diffing
 
 ## Configuration
 
@@ -55,6 +56,22 @@ export GOLDFISH_DB_PASSWORD=your-password
 # Performance comparison settings
 -goldfish.performance-tolerance=0.1                             # 10% tolerance for execution time variance
 
+# Result persistence (optional)
+-goldfish.results.enabled=true                                  # store raw responses
+-goldfish.results.mode=mismatch-only                            # or 'all'
+-goldfish.results.backend=gcs                                   # inferred for CloudSQL
+-goldfish.results.bucket.gcs.bucket-name=<bucket>               # required for GCS
+-goldfish.results.bucket.gcs.service-account=<json>             # optional service account JSON
+-goldfish.results.prefix=goldfish/results                       # optional prefix inside the bucket
+-goldfish.results.compression=gzip                              # gzip or none
+
+# S3 example (RDS deployments)
+-goldfish.results.backend=s3
+-goldfish.results.bucket.s3.bucket-name=<bucket>
+-goldfish.results.bucket.s3.region=<region>
+-goldfish.results.bucket.s3.access-key-id=<key>                 # optional when using IAM roles
+-goldfish.results.bucket.s3.secret-access-key=<secret>          # optional when using IAM roles
+
 # Or run without storage (sampling and comparison only, no persistence)
 # Simply omit the storage configuration
 ```
@@ -79,6 +96,17 @@ export GOLDFISH_DB_PASSWORD=your-password
         └──────────┘ └──────────┘ └──────────┘
 ```
 
+## Result Storage Layout
+
+When result persistence is enabled, Goldfish writes two objects per sampled query under the configured prefix using UTC date partitions and the correlation ID:
+
+```
+<bucket>/<prefix>/<YYYY>/<MM>/<DD>/<correlation-id>/cell-a.json.gz
+<bucket>/<prefix>/<YYYY>/<MM>/<DD>/<correlation-id>/cell-b.json.gz
+```
+
+Metadata includes the tenant, backend name, HTTP status, and original fnv32 hash so you can correlate objects with database entries. Payloads are gzip-compressed by default. Apply lifecycle policies and IAM so raw log data is protected.
+
 ## Database Schema
 
 Goldfish uses two main tables: