Ensure MCP servers from the registry match their expected provenance information (#1269)

Author: rdimitrov

.github/workflows/update-registry.yml

@@ -45,8 +45,8 @@ jobs:
       - name: Update registry
         id: update
         run: |
-          # Run regup with the specified count
-          ./bin/regup update --count ${{ steps.set-count.outputs.count }}
+          # Run regup with the specified count and provenance verification
+          ./bin/regup update --count ${{ steps.set-count.outputs.count }} --verify-provenance
           
           # Check if there are changes
           if git diff --exit-code pkg/registry/data/registry.json; then
@@ -55,6 +55,9 @@ jobs:
           else
             echo "changes=true" >> $GITHUB_OUTPUT
             echo "Changes detected in the registry"
+            # Log what changed for debugging
+            git diff --name-only --diff-filter=M pkg/registry/data/registry.json || true
+            git diff --stat pkg/registry/data/registry.json || true
           fi
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -64,12 +67,14 @@ jobs:
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "Update registry with latest stars and pulls"
-          title: "Update registry with latest stars and pulls"
+          commit-message: "Refresh registry data - pulls, stars, etc."
+          title: "Refresh registry data - pulls, stars, etc."
           body: |
-            This PR updates the registry with the latest GitHub stars and pulls information.
+            This PR refreshes the registry with the latest GitHub stars and pulls information.
             
-            The update was performed automatically by the `regup` command.
+            The update was performed automatically by the `regup` command with provenance verification enabled.
+
+            ** Note: Any servers that failed provenance verification have been automatically removed from the registry. **
           branch: update-registry
           base: main
-          delete-branch: true
+          delete-branch: true

cmd/regup/app/update.go

@@ -3,6 +3,7 @@ package app
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -13,22 +14,34 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/stacklok/toolhive/pkg/container/verifier"
 	"github.com/stacklok/toolhive/pkg/logger"
 	"github.com/stacklok/toolhive/pkg/registry"
 )
 
 var (
-	count       int
-	dryRun      bool
-	githubToken string
-	serverName  string
+	count            int
+	dryRun           bool
+	githubToken      string
+	serverName       string
+	verifyProvenance bool
 )
 
 type serverWithName struct {
 	name   string
 	server *registry.ImageMetadata
 }
 
+// ProvenanceVerificationError represents an error during provenance verification
+type ProvenanceVerificationError struct {
+	ServerName string
+	Reason     string
+}
+
+func (e *ProvenanceVerificationError) Error() string {
+	return fmt.Sprintf("provenance verification failed for server %s: %s", e.ServerName, e.Reason)
+}
+
 var updateCmd = &cobra.Command{
 	Use:   "update",
 	Short: "Update registry entries with latest information",
@@ -45,6 +58,8 @@ func init() {
 		"GitHub token for API authentication (can also be set via GITHUB_TOKEN env var)")
 	updateCmd.Flags().StringVarP(&serverName, "server", "s", "",
 		"Specific server name to update")
+	updateCmd.Flags().BoolVar(&verifyProvenance, "verify-provenance", false,
+		"Verify provenance information and remove servers that fail verification")
 
 	// Mark count and server flags as mutually exclusive
 	updateCmd.MarkFlagsMutuallyExclusive("count", "server")
@@ -69,10 +84,16 @@ func updateCmdFunc(_ *cobra.Command, _ []string) error {
 	}
 
 	// Update servers
-	updatedServers := updateServers(servers)
+	updatedServers, failedServers := updateServers(servers, reg)
+
+	// Log summary
+	if len(failedServers) > 0 {
+		logger.Warnf("Removed %d servers due to provenance verification failures: %v",
+			len(failedServers), failedServers)
+	}
 
 	// Save results
-	return saveResults(reg, updatedServers)
+	return saveResults(reg, updatedServers, failedServers)
 }
 
 func loadRegistry() (*registry.Registry, error) {
@@ -159,37 +180,49 @@ func isOlder(serverI, serverJ *registry.ImageMetadata) bool {
 	return timeI.Before(timeJ)
 }
 
-func updateServers(servers []serverWithName) []string {
+func updateServers(servers []serverWithName, reg *registry.Registry) ([]string, []string) {
 	updatedServers := make([]string, 0, len(servers))
+	failedServers := make([]string, 0)
 
 	for _, s := range servers {
 		logger.Infof("Updating server: %s", s.name)
 
 		if err := updateServerInfo(s.name, s.server); err != nil {
+			var provenanceErr *ProvenanceVerificationError
+			if errors.As(err, &provenanceErr) {
+				logger.Errorf("Provenance verification failed for server %s: %v", s.name, err)
+				failedServers = append(failedServers, s.name)
+				continue
+			}
 			logger.Errorf("Failed to update server %s: %v", s.name, err)
 			continue
 		}
 
 		updatedServers = append(updatedServers, s.name)
 	}
 
-	return updatedServers
+	// Remove failed servers from registry if provenance verification is enabled
+	if verifyProvenance && len(failedServers) > 0 {
+		removeFailedServers(reg, failedServers)
+	}
+
+	return updatedServers, failedServers
 }
 
-func saveResults(reg *registry.Registry, updatedServers []string) error {
+func saveResults(reg *registry.Registry, updatedServers []string, failedServers []string) error {
 	// If we're in dry run mode, don't save changes
 	if dryRun {
 		logger.Info("Dry run completed, no changes made")
 		return nil
 	}
 
-	// If we updated any servers, save the registry
-	if len(updatedServers) > 0 {
+	// If we updated any servers or removed any servers, save the registry
+	if len(updatedServers) > 0 || len(failedServers) > 0 {
 		// Update the last_updated timestamp
 		reg.LastUpdated = time.Now().UTC().Format(time.RFC3339)
 
 		// Save the updated registry
-		if err := saveRegistry(reg, updatedServers); err != nil {
+		if err := saveRegistry(reg, updatedServers, failedServers); err != nil {
 			return fmt.Errorf("failed to save registry: %w", err)
 		}
 
@@ -203,6 +236,16 @@ func saveResults(reg *registry.Registry, updatedServers []string) error {
 
 // updateServerInfo updates the GitHub stars and pulls for a server
 func updateServerInfo(name string, server *registry.ImageMetadata) error {
+	// Verify provenance if requested
+	if verifyProvenance {
+		if err := verifyServerProvenance(name, server); err != nil {
+			return &ProvenanceVerificationError{
+				ServerName: name,
+				Reason:     err.Error(),
+			}
+		}
+	}
+
 	// Skip if no repository URL
 	if server.RepositoryURL == "" {
 		logger.Warnf("ImageMetadata %s has no repository URL, skipping", name)
@@ -245,6 +288,50 @@ func updateServerInfo(name string, server *registry.ImageMetadata) error {
 	return nil
 }
 
+// verifyServerProvenance verifies the provenance information for a server
+func verifyServerProvenance(name string, server *registry.ImageMetadata) error {
+	// Skip if no provenance information
+	if server.Provenance == nil {
+		logger.Warnf("Server %s has no provenance information, skipping verification", name)
+		return nil
+	}
+
+	// Skip if no image reference
+	if server.Image == "" {
+		return fmt.Errorf("no image reference provided")
+	}
+
+	logger.Infof("Verifying provenance for server %s with image %s", name, server.Image)
+
+	// Create verifier
+	v, err := verifier.New(server)
+	if err != nil {
+		return fmt.Errorf("failed to create verifier: %w", err)
+	}
+
+	// Get verification results
+	isVerified, err := v.VerifyServer(server.Image, server)
+	if err != nil {
+		return fmt.Errorf("verification failed: %w", err)
+	}
+
+	// Check if we have valid verification results
+	if isVerified {
+		logger.Infof("Server %s verified successfully", name)
+		return nil
+	}
+
+	return fmt.Errorf("no verified signatures found")
+}
+
+// removeFailedServers removes servers that failed provenance verification from the registry
+func removeFailedServers(reg *registry.Registry, failedServers []string) {
+	for _, serverName := range failedServers {
+		logger.Warnf("Removing server %s from registry due to provenance verification failure", serverName)
+		delete(reg.Servers, serverName)
+	}
+}
+
 // extractOwnerRepo extracts the owner and repo from a GitHub repository URL
 func extractOwnerRepo(url string) (string, string, error) {
 	// Remove trailing .git if present
@@ -333,7 +420,7 @@ func getGitHubRepoInfo(owner, repo, serverName string, currentPulls int) (stars
 }
 
 // saveRegistry saves the registry to the filesystem while preserving the order of entries
-func saveRegistry(reg *registry.Registry, updatedServers []string) error {
+func saveRegistry(reg *registry.Registry, updatedServers []string, failedServers []string) error {
 	// Find the registry file path
 	registryPath := filepath.Join("pkg", "registry", "data", "registry.json")
 
@@ -359,6 +446,12 @@ func saveRegistry(reg *registry.Registry, updatedServers []string) error {
 		return fmt.Errorf("invalid servers map in registry")
 	}
 
+	// Remove failed servers from the JSON
+	for _, name := range failedServers {
+		logger.Infof("Removing server %s from registry JSON", name)
+		delete(serversMap, name)
+	}
+
 	// Update only the servers that were modified
 	for _, name := range updatedServers {
 		server, ok := reg.Servers[name]
@@ -369,7 +462,7 @@ func saveRegistry(reg *registry.Registry, updatedServers []string) error {
 		// Get the server from the original JSON
 		serverJSON, ok := serversMap[name].(map[string]interface{})
 		if !ok {
-			logger.Warnf("ImageMetadata %s not found in original registry, skipping", name)
+			logger.Warnf("Server %s not found in original registry, skipping", name)
 			continue
 		}