feat: implement safe YAML parsing options and enhance comparative testing documentation for Rust HTTP API

Author: tikazyq

packages/ui/package.json

@@ -7,9 +7,9 @@
     "leanspec-ui": "./bin/ui.js"
   },
   "scripts": {
-    "dev": "next dev",
+    "dev": "next dev --port 3001",
     "build": "next build && node ./scripts/post-build.mjs",
-    "start": "next start",
+    "start": "next start --port 3001",
     "lint": "eslint",
     "typecheck": "tsc --noEmit",
     "test": "vitest run",

packages/ui/src/lib/db/seed.ts

@@ -8,6 +8,7 @@ import { readFileSync, readdirSync, statSync } from 'fs';
 import { join } from 'path';
 import { randomUUID } from 'crypto';
 import { eq } from 'drizzle-orm';
+import { safeMatterOptions } from '../spec-utils/frontmatter';
 
 // Path to specs directory (relative to monorepo root)
 const SPECS_DIR = join(process.cwd(), '../../specs');
@@ -38,7 +39,7 @@ function parseSpecDirectory(dirPath: string): ParsedSpec | null {
   try {
     const readmePath = join(dirPath, 'README.md');
     const content = readFileSync(readmePath, 'utf-8');
-    const { data: frontmatter, content: markdownContent } = matter(content);
+    const { data: frontmatter, content: markdownContent } = matter(content, safeMatterOptions);
 
     // Extract spec number and name from directory name
     const dirName = dirPath.split('/').pop() || '';

packages/ui/src/lib/db/service-queries.ts

@@ -11,6 +11,7 @@ import { detectSubSpecs } from '../sub-specs';
 import { join, resolve, dirname } from 'path';
 import { readFileSync, existsSync, statSync, readdirSync } from 'node:fs';
 import matter from 'gray-matter';
+import { safeMatterOptions } from '../spec-utils/frontmatter';
 
 /**
  * Spec with parsed tags (for client consumption)
@@ -66,7 +67,7 @@ function parseSpecTags(spec: Spec): ParsedSpec {
   let contentMd = spec.contentMd;
   if (contentMd.startsWith('---')) {
     try {
-      const { content } = matter(contentMd);
+      const { content } = matter(contentMd, safeMatterOptions);
       contentMd = content;
     } catch {
       // If parsing fails, use original content

packages/ui/src/lib/spec-utils/frontmatter.ts

@@ -9,6 +9,13 @@
 import matter from 'gray-matter';
 import { load, FAILSAFE_SCHEMA } from 'js-yaml';
 
+// Shared YAML engine for gray-matter to avoid js-yaml safeLoad removal
+export const safeMatterOptions = {
+  engines: {
+    yaml: (str: string) => load(str, { schema: FAILSAFE_SCHEMA }) as Record<string, unknown>,
+  },
+};
+
 export type SpecStatus = 'planned' | 'in-progress' | 'complete' | 'archived';
 export type SpecPriority = 'low' | 'medium' | 'high' | 'critical';
 
@@ -168,13 +175,7 @@ export function createUpdatedFrontmatter(
   existingContent: string,
   updates: Partial<SpecFrontmatter>
 ): { content: string; frontmatter: SpecFrontmatter } {
-  const parsed = matter(existingContent, {
-    engines: {
-      // FAILSAFE_SCHEMA is the safest option - only allows strings, arrays, and objects
-      // This prevents any code execution vulnerabilities from YAML parsing
-      yaml: (str) => load(str, { schema: FAILSAFE_SCHEMA }) as Record<string, unknown>
-    }
-  });
+  const parsed = matter(existingContent, safeMatterOptions);
 
   // Store previous data for timestamp enrichment
   const previousData = { ...parsed.data };

packages/ui/src/lib/specs/relationships.ts

@@ -8,6 +8,7 @@
 import matter from 'gray-matter';
 import type { SpecRelationships } from './types';
 import type { Spec } from '../db/schema';
+import { safeMatterOptions } from '../spec-utils/frontmatter';
 
 /**
  * Normalize a depends_on value to a string array
@@ -31,7 +32,7 @@ export function normalizeRelationshipList(value: unknown): string[] {
 export function extractDependsOn(contentOrFrontmatter: string | Record<string, unknown>): string[] {
   if (typeof contentOrFrontmatter === 'string') {
     try {
-      const { data } = matter(contentOrFrontmatter);
+      const { data } = matter(contentOrFrontmatter, safeMatterOptions);
       return normalizeRelationshipList(data?.depends_on ?? data?.dependsOn);
     } catch {
       return [];

packages/ui/src/lib/specs/sources/filesystem-source.ts

@@ -8,6 +8,7 @@ import * as path from 'node:path';
 import matter from 'gray-matter';
 import type { SpecSource, CachedSpec } from '../types';
 import type { Spec } from '../../db/schema';
+import { safeMatterOptions } from '../../spec-utils/frontmatter';
 
 // Cache TTL from environment
 // Default: 0ms in development, 60s in production
@@ -218,7 +219,7 @@ export class FilesystemSource implements SpecSource {
 
     try {
       const rawContent = await fs.readFile(readmePath, 'utf-8');
-      const { data: frontmatter, content: markdownContent } = matter(rawContent);
+      const { data: frontmatter, content: markdownContent } = matter(rawContent, safeMatterOptions);
 
       if (!frontmatter || !frontmatter.status) {
         return null;

packages/ui/src/lib/specs/sources/multi-project-source.ts

@@ -9,6 +9,7 @@ import matter from 'gray-matter';
 import type { SpecSource, CachedSpec } from '../types';
 import type { Spec } from '../../db/schema';
 import { projectRegistry } from '../../projects';
+import { safeMatterOptions } from '../../spec-utils/frontmatter';
 
 // Cache TTL from environment
 const isDev = process.env.NODE_ENV === 'development';
@@ -225,7 +226,7 @@ export class MultiProjectFilesystemSource implements SpecSource {
     projectId: string
   ): Spec | null {
     try {
-      const { data: frontmatter, content: markdown } = matter(content);
+      const { data: frontmatter, content: markdown } = matter(content, safeMatterOptions);
 
       // Validate frontmatter has required status field
       if (!frontmatter || !frontmatter.status) {

packages/ui/src/lib/sub-specs.ts

@@ -5,6 +5,7 @@
 import { readdirSync, readFileSync, existsSync, statSync } from 'fs';
 import { join } from 'path';
 import matter from 'gray-matter';
+import { safeMatterOptions } from './spec-utils/frontmatter';
 
 export interface SubSpec {
   name: string;
@@ -129,7 +130,7 @@ export function detectSubSpecs(specDirPath: string): SubSpec[] {
       try {
         const content = readFileSync(filePath, 'utf-8');
         // Remove frontmatter if present
-        const { content: markdownContent } = matter(content);
+        const { content: markdownContent } = matter(content, safeMatterOptions);
         const iconConfig = getIconForSubSpec(entry);
 
         subSpecs.push({
@@ -162,7 +163,7 @@ export function getSubSpec(specDirPath: string, fileName: string): SubSpec | nul
 
   try {
     const content = readFileSync(filePath, 'utf-8');
-    const { content: markdownContent } = matter(content);
+    const { content: markdownContent } = matter(content, safeMatterOptions);
     const iconConfig = getIconForSubSpec(fileName);
 
     return {

specs/191-rust-http-api-test-suite/README.md

@@ -85,6 +85,13 @@ rust/leanspec-http/tests/
 - Verify response structure compatibility
 - Test serialization consistency
 
+**Comparative Testing** (RECOMMENDED):
+- Run both Next.js API (`pnpm -F @leanspec/ui dev`) and Rust HTTP server side-by-side
+- Make identical requests to both servers with same test data
+- Compare JSON responses field-by-field
+- Validate identical behavior and structure
+- Catch subtle differences that static schema checks might miss
+
 **Test Fixtures**:
 - Reusable test projects with known spec structure
 - Sample specs with various statuses, priorities, tags
@@ -93,10 +100,11 @@ rust/leanspec-http/tests/
 **Tools**:
 - `axum-test` or raw Axum router testing
 - `reqwest` for HTTP client
-- `serde_json` for response validation
+- `serde_json` for response validation and comparison
 - `schemars` for JSON Schema generation and validation
 - `tempfile` for temporary test projects
 - `tokio::test` for async tests
+- `assert_json_diff` for comparing JSON responses between servers
 
 ## Plan
 
@@ -107,6 +115,8 @@ rust/leanspec-http/tests/
 - [ ] Add test utilities (assertions, matchers)
 - [ ] Set up schema validation utilities
 - [ ] Document reference Next.js API schemas
+- [ ] Create dual-server test helper (optional: spawn both Next.js + Rust)
+- [ ] Add JSON response comparison utilities
 
 ### Phase 2: Project Management Tests (Day 2)
 - [ ] Test GET `/api/projects` (list all, empty state, multi-project)
@@ -183,9 +193,82 @@ rust/leanspec-http/tests/
 - [ ] Large dataset testing (100+ specs)
 - [ ] Test documentation/examples
 - [ ] JSON Schema exports for documentation
+- [ ] **Comparative tests with live Next.js API** (side-by-side validation)
 
 ## Test Examples
 
+### Comparative Testing (Next.js vs Rust)
+
+```rust
+#[tokio::test]
+#[ignore] // Only run when Next.js server is running
+async fn test_compare_specs_response_with_nextjs() {
+    // Set up test project with known specs
+    let test_project = setup_test_project_with_fixtures().await;
+    
+    // Start Rust HTTP server
+    let rust_client = reqwest::Client::new();
+    let rust_base = "http://localhost:3001"; // Rust server
+    
+    // Assume Next.js is running on default port
+    let nextjs_client = reqwest::Client::new();
+    let nextjs_base = "http://localhost:3000"; // Next.js server
+    
+    // Compare GET /api/specs responses
+    let rust_res = rust_client
+        .get(format!("{}/api/specs", rust_base))
+        .send()
+        .await
+        .unwrap();
+    let rust_json: serde_json::Value = rust_res.json().await.unwrap();
+    
+    let nextjs_res = nextjs_client
+        .get(format!("{}/api/projects/default/specs", nextjs_base))
+        .send()
+        .await
+        .unwrap();
+    let nextjs_json: serde_json::Value = nextjs_res.json().await.unwrap();
+    
+    // Compare response structure
+    assert_json_diff::assert_json_eq!(
+        rust_json["specs"][0]["specNumber"],
+        nextjs_json["specs"][0]["specNumber"]
+    );
+    assert_json_diff::assert_json_eq!(
+        rust_json["specs"][0]["specName"],
+        nextjs_json["specs"][0]["specName"]
+    );
+    
+    // Verify both use camelCase
+    assert!(rust_json["specs"][0].get("specNumber").is_some());
+    assert!(rust_json["specs"][0].get("spec_number").is_none());
+}
+
+#[tokio::test]
+async fn test_compare_stats_response_structure() {
+    let rust_app = test_server_with_fixtures().await;
+    
+    // Get stats from Rust API
+    let rust_res = rust_app.get("/api/stats").send().await;
+    let rust_json: serde_json::Value = rust_res.json().await;
+    
+    // Validate structure matches Next.js format
+    // Next.js returns: { stats: { total, byStatus, byPriority, byTag, ... } }
+    assert!(rust_json.get("total").is_some());
+    assert!(rust_json.get("byStatus").is_some());
+    assert!(rust_json.get("byPriority").is_some());
+    assert!(rust_json.get("byTag").is_some());
+    assert!(rust_json.get("completionPercentage").is_some());
+    
+    // Validate nested structure (camelCase)
+    let by_status = &rust_json["byStatus"];
+    assert!(by_status.get("planned").is_some());
+    assert!(by_status.get("inProgress").is_some()); // camelCase
+    assert!(by_status.get("in_progress").is_none()); // NOT snake_case
+    assert!(by_status.get("complete").is_some());
+}
+```
+
 ### Schema Validation Test
 
 ```rust
@@ -347,6 +430,35 @@ async fn test_search_relevance_ranking() {
 4. Document any intentional differences
 5. Create compatibility tests that would fail on schema drift
 
+**Comparative Testing (Recommended)**:
+1. Run Next.js dev server: `pnpm -F @leanspec/ui dev` (port 3000)
+2. Run Rust HTTP server: `cargo run --bin leanspec-http` (port 3001)
+3. Point both at same test project directory
+4. Make identical requests to both APIs
+5. Compare JSON responses field-by-field using `assert_json_diff`
+6. Validates not just schema but actual behavior
+
+**Benefits of Live Comparison**:
+- Catches subtle differences that static checks miss
+- Validates actual serialization behavior
+- Tests with real Next.js API implementation
+- No need to manually extract/maintain reference schemas
+- Confirms identical responses for same input
+
+**Setup for Comparative Tests**:
+```bash
+# Terminal 1: Start Next.js API
+cd /path/to/lean-spec
+pnpm -F @leanspec/ui dev
+
+# Terminal 2: Start Rust HTTP server
+cd /path/to/lean-spec/rust/leanspec-http
+cargo run
+
+# Terminal 3: Run comparative tests
+cargo test --test comparative -- --ignored
+```
+
 **Key Fields to Validate**:
 - `specNumber` (not `spec_number`)
 - `specName` (not `spec_name`)
@@ -405,3 +517,10 @@ async fn test_search_relevance_ranking() {
 - Defined test architecture and strategy
 - 5-day implementation plan
 - Priority: HIGH - prerequisite for Spec 190
+
+### 2025-12-19: Schema Alignment Added
+- Added explicit schema compatibility testing with Next.js APIs
+- Documented key fields to validate (camelCase serialization)
+- Added comparative testing strategy (run both servers side-by-side)
+- Confirmed Rust types already use camelCase via `#[serde(rename_all = "camelCase")]`
+- Added example tests for live API comparison