Add JSON schema validation for registry.json (#1438)

Signed-off-by: Dan Barr <[email protected]>
Co-authored-by: Dan Barr <[email protected]>

Author: danbarr

docs/registry/schema.md

@@ -15,13 +15,50 @@ This can also be used to validate a custom registry file to be used with the
 
 ## Schema location
 
-- **File**: [`docs/registry/schema.json`](schema.json)
+- **File**: [`pkg/registry/data/schema.json`](../../pkg/registry/data/schema.json)
 - **Schema ID**:
-  `https://raw.githubusercontent.com/stacklok/toolhive/main/docs/registry/schema.json`
+  `https://raw.githubusercontent.com/stacklok/toolhive/main/pkg/registry/data/schema.json`
 
 ## Usage
 
-### Local validation
+### Automated validation (Go tests)
+
+The registry is automatically validated against the schema during development
+and CI/CD through Go tests. This ensures that any changes to the registry data
+are immediately validated.
+
+The validation is implemented in
+[`pkg/registry/schema_validation.go`](../../pkg/registry/schema_validation.go)
+and tested in
+[`pkg/registry/schema_validation_test.go`](../../pkg/registry/schema_validation_test.go).
+
+**Key tests:**
+
+- `TestEmbeddedRegistrySchemaValidation` - Validates the embedded
+  `registry.json` against the schema
+- `TestRegistrySchemaValidation` - Comprehensive test suite with valid and
+  invalid registry examples
+
+**Running the validation:**
+
+```bash
+# Run all schema validation tests
+go test -v ./pkg/registry -run ".*Schema.*"
+
+# Run just the embedded registry validation
+go test -v ./pkg/registry -run TestEmbeddedRegistrySchemaValidation
+
+# Run all registry tests (includes schema validation)
+go test -v ./pkg/registry
+```
+
+This validation runs automatically as part of:
+
+- Local development (`go test`)
+- CI/CD pipeline (GitHub Actions)
+- Pre-commit hooks (if configured)
+
+### Manual validation
 
 #### Using check-jsonschema
 
@@ -41,7 +78,7 @@ Validate the registry with full format validation:
 
 ```bash
 # Run from the root of the repository
-check-jsonschema --schemafile docs/registry/schema.json pkg/registry/data/registry.json
+check-jsonschema --schemafile pkg/registry/data/schema.json pkg/registry/data/registry.json
 ```
 
 #### Using ajv-cli
@@ -56,7 +93,7 @@ Validate the registry with format validation:
 
 ```bash
 # Run from the root of the repository
-ajv validate -c ajv-formats -s docs/registry/schema.json -d pkg/registry/data/registry.json
+ajv validate -c ajv-formats -s pkg/registry/data/schema.json -d pkg/registry/data/registry.json
 ```
 
 #### Using VS Code
@@ -66,7 +103,7 @@ to the top of any registry JSON file:
 
 ```json
 {
-  "$schema": "https://raw.githubusercontent.com/stacklok/toolhive/main/docs/registry/schema.json",
+  "$schema": "https://raw.githubusercontent.com/stacklok/toolhive/main/pkg/registry/data/schema.json",
   ...
 }
 ```

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/ory/fosite v0.49.0
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
 	github.com/prometheus/client_golang v1.23.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sigstore/protobuf-specs v0.5.0
 	github.com/sigstore/sigstore-go v1.1.1
 	github.com/spf13/viper v1.20.1

go.sum

@@ -1461,6 +1461,8 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
 github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/sassoftware/relic v7.2.1+incompatible h1:Pwyh1F3I0r4clFJXkSI8bOyJINGqpgjJU3DYAZeI05A=
 github.com/sassoftware/relic v7.2.1+incompatible/go.mod h1:CWfAxv73/iLZ17rbyhIEq3K9hs5w6FpNMdUT//qR+zk=
 github.com/sassoftware/relic/v7 v7.6.2 h1:rS44Lbv9G9eXsukknS4mSjIAuuX+lMq/FnStgmZlUv4=

pkg/registry/data/registry.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://raw.githubusercontent.com/stacklok/toolhive/main/docs/registry/schema.json",
+  "$schema": "https://raw.githubusercontent.com/stacklok/toolhive/main/pkg/registry/data/schema.json",
   "last_updated": "2025-08-14T06:59:15Z",
   "servers": {
     "arxiv-mcp-server": {

docs/registry/schema.json renamed to pkg/registry/data/schema.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://raw.githubusercontent.com/stacklok/toolhive/main/docs/registry/schema.json",
+  "$id": "https://raw.githubusercontent.com/stacklok/toolhive/main/pkg/registry/data/schema.json",
   "title": "ToolHive MCP Server Registry Schema",
   "description": "JSON Schema for the ToolHive MCP server registry. This schema validates the structure and content of registry.json entries for MCP servers. See docs/registry/management.md and docs/registry/heuristics.md for inclusion criteria and management processes.",
   "type": "object",

pkg/registry/schema_validation.go

@@ -0,0 +1,113 @@
+package registry
+
+import (
+	"embed"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+//go:embed data/schema.json
+var embeddedSchemaFS embed.FS
+
+// ValidateRegistrySchema validates registry JSON data against the registry schema
+func ValidateRegistrySchema(registryData []byte) error {
+	// Load the schema from the embedded filesystem
+	schemaData, err := embeddedSchemaFS.ReadFile("data/schema.json")
+	if err != nil {
+		return fmt.Errorf("failed to read embedded registry schema: %w", err)
+	}
+
+	// Compile the schema
+	compiler := jsonschema.NewCompiler()
+	schemaID := "file://local/registry-schema.json"
+	if err := compiler.AddResource(schemaID, strings.NewReader(string(schemaData))); err != nil {
+		return fmt.Errorf("failed to add schema resource: %w", err)
+	}
+	schema, err := compiler.Compile(schemaID)
+	if err != nil {
+		return fmt.Errorf("failed to compile registry schema: %w", err)
+	}
+
+	// Parse the registry data
+	var registryDoc interface{}
+	if err := json.Unmarshal(registryData, &registryDoc); err != nil {
+		return fmt.Errorf("failed to parse registry data: %w", err)
+	}
+
+	// Validate the registry data against the schema
+	if err := schema.Validate(registryDoc); err != nil {
+		// Extract all validation errors for better user experience
+		if validationErr, ok := err.(*jsonschema.ValidationError); ok {
+			return formatValidationErrors(validationErr)
+		}
+		return fmt.Errorf("registry schema validation failed: %w", err)
+	}
+
+	return nil
+}
+
+// formatValidationErrors formats all validation errors into a comprehensive error message
+func formatValidationErrors(validationErr *jsonschema.ValidationError) error {
+	var errorMessages []string
+
+	// Collect all validation errors recursively
+	collectErrors(validationErr, &errorMessages)
+
+	if len(errorMessages) == 0 {
+		return fmt.Errorf("registry schema validation failed: %s", validationErr.Error())
+	}
+
+	if len(errorMessages) == 1 {
+		return fmt.Errorf("registry schema validation failed: %s", errorMessages[0])
+	}
+
+	// Format multiple errors
+	result := fmt.Sprintf("registry schema validation failed with %d errors:\n", len(errorMessages))
+	for i, msg := range errorMessages {
+		result += fmt.Sprintf("  %d. %s\n", i+1, msg)
+	}
+
+	return fmt.Errorf("%s", strings.TrimSuffix(result, "\n"))
+}
+
+// collectErrors recursively collects all validation error messages
+func collectErrors(err *jsonschema.ValidationError, messages *[]string) {
+	if err == nil {
+		return
+	}
+
+	// If this error has causes, recurse into them instead of adding this error
+	// This avoids duplicate parent/child error messages
+	if len(err.Causes) > 0 {
+		for _, cause := range err.Causes {
+			collectErrors(cause, messages)
+		}
+		return
+	}
+
+	// This is a leaf error - add it if it has a meaningful message
+	if err.Message != "" {
+		// Create a descriptive error message with path context
+		var pathStr string
+		if err.InstanceLocation != "" {
+			pathStr = fmt.Sprintf(" at '%s'", err.InstanceLocation)
+		}
+
+		errorMsg := fmt.Sprintf("%s%s", err.Message, pathStr)
+		*messages = append(*messages, errorMsg)
+	}
+}
+
+// ValidateEmbeddedRegistry validates the embedded registry.json against the schema
+func ValidateEmbeddedRegistry() error {
+	// Load the embedded registry data
+	registryData, err := embeddedRegistryFS.ReadFile("data/registry.json")
+	if err != nil {
+		return fmt.Errorf("failed to load embedded registry: %w", err)
+	}
+
+	return ValidateRegistrySchema(registryData)
+}