Add JSON schema validation for expected_profile.json

Reject invalid/empty JSON files that were silently passing tests.

Validates:
- stacks required (unless note field explains why empty)
- profile-type and stack-content required per stack
- regular_expression required per stack-content entry
- value or percent required (unless parent has value-matching-sum)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: realFlowControl

go.mod

@@ -7,3 +7,9 @@ require (
 	github.com/klauspost/compress v1.17.11
 	github.com/pierrec/lz4/v4 v4.1.21
 )
+
+require (
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+)

go.sum

@@ -1,6 +1,16 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g=
 github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
 github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=

pprof_analysis.go

@@ -17,13 +17,57 @@ import (
 	"github.com/google/pprof/profile"
 	"github.com/klauspost/compress/zstd"
 	"github.com/pierrec/lz4/v4"
+	"github.com/xeipuuv/gojsonschema"
 )
 
 var (
 	_ json.Unmarshaler = (*Optional[int64])(nil)
 	_ json.Marshaler   = (*Optional[int64])(nil)
 )
 
+// JSON Schema for validating expected_profile.json files
+// Basic structure validation - complex business rules validated in Go code
+var expectedProfileSchema = `{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "required": ["stacks"],
+  "properties": {
+    "test_name": { "type": "string" },
+    "note": { "type": "string" },
+    "scale_by_duration": { "type": "boolean" },
+    "pprof-regex": { "type": "string" },
+    "allow_first_profile_failure": { "type": "boolean" },
+    "stacks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["profile-type", "stack-content"],
+        "properties": {
+          "profile-type": { "type": "string", "minLength": 1 },
+          "pprof-regex": { "type": "string" },
+          "stack-content": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "required": ["regular_expression"],
+              "properties": {
+                "regular_expression": { "type": "string", "minLength": 1 },
+                "value": { "type": "integer" },
+                "percent": { "type": "integer" },
+                "error_margin": { "type": "integer" },
+                "labels": { "type": "array" }
+              }
+            }
+          },
+          "error-margin": { "type": "integer" },
+          "value-matching-sum": { "type": "integer" }
+        }
+      }
+    }
+  }
+}`
+
 type Optional[T any] struct {
 	value *T
 }
@@ -94,12 +138,37 @@ type TypedStacks struct {
 
 type StackTestData struct {
 	TestName                 string        `json:"test_name"`
+	Note                     string        `json:"note,omitempty"`
 	ScaleByDuration          bool          `json:"scale_by_duration"`
 	PprofRegex               string        `json:"pprof-regex"`
 	AllowFirstProfileFailure bool          `json:"allow_first_profile_failure,omitempty"`
 	Stacks                   []TypedStacks `json:"stacks"`
 }
 
+// Validate rules that JSON Schema can't express
+func (s *StackTestData) Validate() error {
+	// Stacks must be non-empty unless note is present
+	if len(s.Stacks) == 0 && s.Note == "" {
+		return fmt.Errorf("'stacks' must have at least one entry (or provide a 'note' explaining why it's empty)")
+	}
+
+	// If no value-matching-sum, require value or percent in stack-content
+	for i, stack := range s.Stacks {
+		if _, hasValueMatchingSum := stack.ValueMatchingSum.Value(); hasValueMatchingSum {
+			continue
+		}
+		for j, content := range stack.StackContent {
+			_, hasValue := content.Value.Value()
+			_, hasPercent := content.Percent.Value()
+			if !hasValue && !hasPercent {
+				return fmt.Errorf("stacks[%d].stack-content[%d]: must have 'value' or 'percent' (or parent must have 'value-matching-sum')", i, j)
+			}
+		}
+	}
+
+	return nil
+}
+
 // Custom unmarshaller for Labels to ensure exactly one of Values and ValueRegex is defined
 func (l *Labels) UnmarshalJSON(data []byte) error {
 	type labels Labels
@@ -442,21 +511,41 @@ func writeToJSONFile(data StackTestData, filePath string) error {
 
 func readJSONFile(filePath string) (StackTestData, error) {
 	var data StackTestData
-	jsonFile, err := os.Open(filePath)
+	byteValue, err := os.ReadFile(filePath)
 	if err != nil {
 		return data, err
 	}
-	defer jsonFile.Close()
-	byteValue, err := io.ReadAll(jsonFile)
+
+	// Step 1: Validate JSON syntax
+	if !json.Valid(byteValue) {
+		return data, fmt.Errorf("invalid JSON syntax in %s", filePath)
+	}
+
+	// Step 2: Validate against schema
+	schemaLoader := gojsonschema.NewStringLoader(expectedProfileSchema)
+	documentLoader := gojsonschema.NewBytesLoader(byteValue)
+	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
 	if err != nil {
-		return data, err
+		return data, fmt.Errorf("schema validation error for %s: %v", filePath, err)
 	}
-	if !json.Valid(byteValue) {
-		return data, fmt.Errorf("Invalid json data %s", filePath)
+	if !result.Valid() {
+		var errs []string
+		for _, desc := range result.Errors() {
+			errs = append(errs, desc.String())
+		}
+		return data, fmt.Errorf("JSON schema validation failed for %s:\n  - %s", filePath, strings.Join(errs, "\n  - "))
 	}
+
+	// Step 3: Unmarshal validated JSON
 	if err := json.Unmarshal(byteValue, &data); err != nil {
 		return data, err
 	}
+
+	// Step 4: Validate business rules
+	if err := data.Validate(); err != nil {
+		return data, fmt.Errorf("validation failed for %s: %v", filePath, err)
+	}
+
 	return data, nil
 }
 

scenarios/dotnet_wall/expected_profile.json

@@ -35,39 +35,6 @@
           ]
         }
       ]
-    },
-    {
-      "profile-type": "cpu",
-      "pprof-regex": "",
-      "stack-content": null,
-      "error-margin": 3,
-      "value-matching-sum": null
-    },
-    {
-      "profile-type": "exception",
-      "pprof-regex": "",
-      "stack-content": null,
-      "error-margin": 3,
-      "value-matching-sum": null
-    },
-    {
-      "profile-type": "lock-count",
-      "pprof-regex": "",
-      "stack-content": null,
-      "error-margin": 3,
-      "value-matching-sum": null
-    },
-    {
-      "profile-type": "lock-time",
-      "pprof-regex": "",
-      "stack-content": null
-    },
-    {
-      "profile-type": "timeline",
-      "pprof-regex": "",
-      "stack-content": null,
-      "error-margin": 3,
-      "value-matching-sum": null
     }
   ],
   "scale_by_duration": false

schema_test.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestSchemaValidation(t *testing.T) {
+	tests := []struct {
+		name        string
+		content     string
+		wantErr     bool
+		errContains string
+	}{
+		{
+			name:        "empty JSON",
+			content:     `{}`,
+			wantErr:     true,
+			errContains: "stacks is required",
+		},
+		{
+			name:        "wrong structure",
+			content:     `{"foo": "bar"}`,
+			wantErr:     true,
+			errContains: "stacks is required",
+		},
+		{
+			name:        "empty stacks without note",
+			content:     `{"stacks": []}`,
+			wantErr:     true,
+			errContains: "stacks",
+		},
+		{
+			name:        "empty stacks with note",
+			content:     `{"stacks": [], "note": "This test doesn't check profile content"}`,
+			wantErr:     false,
+		},
+		{
+			name:        "missing profile-type",
+			content:     `{"stacks": [{"stack-content": [{"regular_expression": "test", "value": 100}]}]}`,
+			wantErr:     true,
+			errContains: "profile-type",
+		},
+		{
+			name:        "missing stack-content",
+			content:     `{"stacks": [{"profile-type": "wall-time"}]}`,
+			wantErr:     true,
+			errContains: "stack-content",
+		},
+		{
+			name:        "empty stack-content",
+			content:     `{"stacks": [{"profile-type": "wall-time", "stack-content": []}]}`,
+			wantErr:     true,
+			errContains: "stack-content",
+		},
+		{
+			name:        "missing regular_expression",
+			content:     `{"stacks": [{"profile-type": "wall-time", "stack-content": [{"value": 100}]}]}`,
+			wantErr:     true,
+			errContains: "regular_expression",
+		},
+		{
+			name:        "missing value and percent",
+			content:     `{"stacks": [{"profile-type": "wall-time", "stack-content": [{"regular_expression": "test"}]}]}`,
+			wantErr:     true,
+			errContains: "value",
+		},
+		{
+			name: "no value/percent but has value-matching-sum",
+			content: `{
+				"stacks": [{
+					"profile-type": "cpu-time",
+					"stack-content": [{"regular_expression": ".*test.*"}],
+					"value-matching-sum": 1000000000
+				}]
+			}`,
+			wantErr: false,
+		},
+		{
+			name: "valid JSON with value",
+			content: `{
+				"test_name": "test",
+				"stacks": [{
+					"profile-type": "wall-time",
+					"stack-content": [{"regular_expression": "^test$", "value": 100}]
+				}]
+			}`,
+			wantErr: false,
+		},
+		{
+			name: "valid JSON with percent",
+			content: `{
+				"stacks": [{
+					"profile-type": "wall-time",
+					"stack-content": [{"regular_expression": "^test$", "percent": 50}]
+				}]
+			}`,
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tmpFile, err := os.CreateTemp("", "schema_test_*.json")
+			if err != nil {
+				t.Fatalf("Failed to create temp file: %v", err)
+			}
+			defer os.Remove(tmpFile.Name())
+
+			if _, err := tmpFile.WriteString(tt.content); err != nil {
+				t.Fatalf("Failed to write to temp file: %v", err)
+			}
+			tmpFile.Close()
+
+			_, err = readJSONFile(tmpFile.Name())
+
+			if tt.wantErr {
+				if err == nil {
+					t.Errorf("Expected error containing %q, got nil", tt.errContains)
+				} else if !strings.Contains(err.Error(), tt.errContains) {
+					t.Errorf("Expected error containing %q, got: %v", tt.errContains, err)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("Expected success, got error: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestSchemaValidation_AllExistingProfiles(t *testing.T) {
+	scenariosDir := "scenarios"
+	if _, err := os.Stat(scenariosDir); os.IsNotExist(err) {
+		t.Skip("scenarios directory not found")
+	}
+
+	var jsonFiles []string
+	err := filepath.Walk(scenariosDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.Name() == "expected_profile.json" {
+			jsonFiles = append(jsonFiles, path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Failed to walk scenarios directory: %v", err)
+	}
+
+	if len(jsonFiles) == 0 {
+		t.Skip("No expected_profile.json files found")
+	}
+
+	t.Logf("Found %d expected_profile.json files to validate", len(jsonFiles))
+
+	for _, jsonFile := range jsonFiles {
+		t.Run(jsonFile, func(t *testing.T) {
+			if _, err := readJSONFile(jsonFile); err != nil {
+				t.Errorf("Validation failed: %v", err)
+			}
+		})
+	}
+}