fix: restore accidentally deleted mirror data scripts

Author: tadasant

scripts/mirror_data/fetch_production_data.go

@@ -0,0 +1,87 @@
+// This tool was created by Claude Code as a simple way to kick the tires on data migrations
+// by fetching production data from the public registry API.
+// It is not intended for production use.
+//
+//go:build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+)
+
+type ServerResponse struct {
+	Servers  []json.RawMessage `json:"servers"`
+	Metadata struct {
+		NextCursor string `json:"next_cursor,omitempty"`
+		Count      int    `json:"count"`
+	} `json:"metadata"`
+}
+
+func main() {
+	baseURL := "https://registry.modelcontextprotocol.io/v0/servers"
+	var allServers []json.RawMessage
+	cursor := ""
+	pageCount := 0
+
+	for {
+		pageCount++
+		url := baseURL
+		if cursor != "" {
+			url = fmt.Sprintf("%s?cursor=%s", baseURL, cursor)
+		}
+
+		fmt.Printf("Fetching page %d: %s\n", pageCount, url)
+
+		resp, err := http.Get(url)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to fetch: %v", err))
+		}
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to read body: %v", err))
+		}
+
+		var serverResp ServerResponse
+		if err := json.Unmarshal(body, &serverResp); err != nil {
+			panic(fmt.Sprintf("Failed to parse JSON: %v", err))
+		}
+
+		allServers = append(allServers, serverResp.Servers...)
+		fmt.Printf("  Got %d servers (total: %d)\n", len(serverResp.Servers), len(allServers))
+
+		if serverResp.Metadata.NextCursor == "" {
+			break
+		}
+		cursor = serverResp.Metadata.NextCursor
+
+		// Be nice to the API
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Save all servers to a file
+	output := map[string]interface{}{
+		"servers": allServers,
+		"count":   len(allServers),
+		"fetched": time.Now().Format(time.RFC3339),
+	}
+
+	data, err := json.MarshalIndent(output, "", "  ")
+	if err != nil {
+		panic(fmt.Sprintf("Failed to marshal output: %v", err))
+	}
+
+	outputFile := "production_servers.json"
+	if err := os.WriteFile(outputFile, data, 0644); err != nil {
+		panic(fmt.Sprintf("Failed to write file: %v", err))
+	}
+
+	fmt.Printf("\nDone! Saved %d servers to %s\n", len(allServers), outputFile)
+}

scripts/mirror_data/load_production_data.go

@@ -0,0 +1,144 @@
+// This tool was created by Claude Code as a simple way to kick the tires on data migrations
+// by loading production data into a test database for migration testing.
+// It is not intended for production use.
+//
+//go:build ignore
+
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+
+	"github.com/google/uuid"
+	_ "github.com/lib/pq"
+)
+
+func main() {
+	// Database connection
+	db, err := sql.Open("postgres", "postgres://postgres:testpass@localhost:5433/registry_test?sslmode=disable")
+	if err != nil {
+		log.Fatal("Failed to connect to database:", err)
+	}
+	defer db.Close()
+
+	// Run migrations up to a specific point (configure as needed)
+	maxMigration := 7 // Change this to test different migration states
+	fmt.Printf("Running migrations 001-%03d...\n", maxMigration)
+	migrationsDir := "internal/database/migrations"
+	for i := 1; i <= maxMigration; i++ {
+		migrationFile := filepath.Join(migrationsDir, fmt.Sprintf("%03d_*.sql", i))
+		files, err := filepath.Glob(migrationFile)
+		if err != nil || len(files) == 0 {
+			log.Fatalf("Migration file %d not found", i)
+		}
+
+		content, err := ioutil.ReadFile(files[0])
+		if err != nil {
+			log.Fatalf("Failed to read migration %d: %v", i, err)
+		}
+
+		fmt.Printf("  Applying migration %d: %s\n", i, filepath.Base(files[0]))
+		if _, err := db.Exec(string(content)); err != nil {
+			log.Fatalf("Failed to apply migration %d: %v", i, err)
+		}
+	}
+
+	// Load production data
+	fmt.Println("\nLoading production data...")
+	data, err := ioutil.ReadFile("scripts/mirror_data/production_servers.json")
+	if err != nil {
+		log.Fatal("Failed to read production data:", err)
+	}
+
+	var prodData struct {
+		Servers []json.RawMessage `json:"servers"`
+	}
+	if err := json.Unmarshal(data, &prodData); err != nil {
+		log.Fatal("Failed to parse production data:", err)
+	}
+
+	fmt.Printf("Loading %d servers...\n", len(prodData.Servers))
+
+	// Prepare insert statement
+	stmt, err := db.Prepare("INSERT INTO servers (version_id, value) VALUES ($1, $2)")
+	if err != nil {
+		log.Fatal("Failed to prepare statement:", err)
+	}
+	defer stmt.Close()
+
+	// Insert each server
+	for i, server := range prodData.Servers {
+		// Generate a unique version_id
+		versionID := uuid.New().String()
+
+		// The server data is already JSON, just insert it
+		if _, err := stmt.Exec(versionID, server); err != nil {
+			log.Printf("Failed to insert server %d: %v", i, err)
+			continue
+		}
+	}
+
+	fmt.Println("Data loaded successfully!")
+
+	// Verify the data
+	var count int
+	db.QueryRow("SELECT COUNT(*) FROM servers").Scan(&count)
+	fmt.Printf("\nTotal servers in database: %d\n", count)
+
+	// Check for NULL status values in the JSON
+	fmt.Println("\nAnalyzing status field in JSON data...")
+
+	rows, err := db.Query(`
+		SELECT
+			COUNT(*) as total,
+			COUNT(CASE WHEN value->>'status' IS NULL THEN 1 END) as null_status,
+			COUNT(CASE WHEN value->>'status' = '' THEN 1 END) as empty_status,
+			COUNT(CASE WHEN value->>'status' = 'null' THEN 1 END) as string_null_status,
+			COUNT(CASE WHEN value->>'status' = 'active' THEN 1 END) as active_status,
+			COUNT(CASE WHEN value->>'status' = 'deprecated' THEN 1 END) as deprecated_status,
+			COUNT(CASE WHEN value->>'status' = 'deleted' THEN 1 END) as deleted_status
+		FROM servers
+	`)
+	if err != nil {
+		log.Fatal("Failed to analyze data:", err)
+	}
+	defer rows.Close()
+
+	if rows.Next() {
+		var total, nullStatus, emptyStatus, stringNullStatus, activeStatus, deprecatedStatus, deletedStatus int
+		rows.Scan(&total, &nullStatus, &emptyStatus, &stringNullStatus, &activeStatus, &deprecatedStatus, &deletedStatus)
+
+		fmt.Printf("  Total servers: %d\n", total)
+		fmt.Printf("  NULL status: %d\n", nullStatus)
+		fmt.Printf("  Empty status: %d\n", emptyStatus)
+		fmt.Printf("  'null' string status: %d\n", stringNullStatus)
+		fmt.Printf("  'active' status: %d\n", activeStatus)
+		fmt.Printf("  'deprecated' status: %d\n", deprecatedStatus)
+		fmt.Printf("  'deleted' status: %d\n", deletedStatus)
+		fmt.Printf("  Other/Invalid: %d\n", total-nullStatus-emptyStatus-stringNullStatus-activeStatus-deprecatedStatus-deletedStatus)
+	}
+
+	// Print sample servers with no status
+	fmt.Println("\nSample servers with NULL status:")
+	rows, err = db.Query(`
+		SELECT value->>'name', value->>'version'
+		FROM servers
+		WHERE value->>'status' IS NULL
+		LIMIT 5
+	`)
+	if err == nil {
+		defer rows.Close()
+		for rows.Next() {
+			var name, version string
+			rows.Scan(&name, &version)
+			fmt.Printf("  - %s@%s\n", name, version)
+		}
+	}
+
+	fmt.Printf("\nDatabase is ready for testing migration %03d!\n", maxMigration+1)
+}