diff --git a/.claude/settings.local.json b/.claude/settings.local.json index ef871e5..a01f553 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -33,7 +33,19 @@ "Bash(npm run build:*)", "Bash(curl:*)", "mcp__sequential-thinking__sequentialthinking", - "Bash(gh repo set-default:*)" + "Bash(gh repo set-default:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(pkill:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(cd:*)", + "Bash(git fetch:*)", + "Bash(git merge:*)" ] }, "enableAllProjectMcpServers": true, diff --git a/SPEC_PRP/PRPs/semantic-link-analysis-redesign.md b/SPEC_PRP/PRPs/semantic-link-analysis-redesign.md new file mode 100644 index 0000000..23e12f2 --- /dev/null +++ b/SPEC_PRP/PRPs/semantic-link-analysis-redesign.md @@ -0,0 +1,365 @@ +# PRP: Semantic Link Analysis System Redesign + +## Executive Summary +Transform the current analysis system to enable bi-directional semantic link discovery between blog posts using Claude AI. The system will analyze individual posts on-demand, finding both incoming link opportunities (from other posts) and outgoing link opportunities (to other posts), with queue-based batch processing, preview capabilities, and CSV export functionality. + +## Current State Assessment + +### State Documentation + +```yaml +current_state: + files: + - src/app/api/analysis/start/route.ts # Batch analysis for all posts + - src/app/api/analysis/status/route.ts # Global analysis status + - src/lib/job-processor.ts # Parallel job processing + - src/lib/semantic-analyzer.ts # Claude API integration + - src/app/components/LinkReviewPanel.tsx # Link review UI + + behavior: + - Analyzes all posts in database at once + - No single-post analysis capability + - Basic link review interface + - Limited control over analysis process + - No preview of link context + + issues: + - Cannot analyze individual posts on demand + - No bi-directional link discovery + - Missing context preview for suggested links + - No queue visibility during analysis + - Limited to analyzing all posts at once + +desired_state: + files: + - src/app/api/analysis/analyze-post/route.ts # Single post analysis + - src/app/api/analysis/queue/route.ts # Queue management + - src/lib/semantic-link-analyzer.ts # Enhanced Claude integration + - src/app/components/AnalysisTab.tsx # New analysis tab + - src/app/components/LinkPreview.tsx # Link context preview + + behavior: + - Analyze individual posts on-demand + - Bi-directional link discovery (incoming & outgoing) + - Queue-based batch processing with visibility + - Preview link placement in context + - Confidence scores and reasoning display + - Export approved links as HTML in CSV + + benefits: + - Targeted analysis of specific posts + - Better link quality with context preview + - Transparent queue progress + - Efficient batch processing + - Production-ready export format +``` + +## Hierarchical Objectives + +### 1. High-Level: Enable On-Demand Bi-Directional Link Analysis +Transform the analysis system to support individual post analysis with both incoming and outgoing link discovery, queue management, and enhanced review capabilities. + +### 2. Mid-Level Milestones + +#### 2.1 Create Single-Post Analysis Infrastructure +- Add analyze button to individual posts +- Implement bi-directional link discovery +- Queue management with batch processing + +#### 2.2 Enhance Link Review Experience +- Add Analysis tab to blog viewer +- Show link previews with context +- Display confidence scores and reasoning + +#### 2.3 Implement Export Functionality +- Generate HTML links with proper attributes +- Export in original CSV format +- Include only approved links + +### 3. Low-Level Tasks + +## Task Specifications + +### Task 1: Create Single Post Analysis Endpoint +```yaml +task_name: create_single_post_analysis_endpoint +action: CREATE +file: src/app/api/analysis/analyze-post/route.ts +changes: | + - Create POST endpoint accepting postId + - Queue analysis jobs for bi-directional links + - Find places in OTHER posts to link TO this post + - Find places in THIS post to link to OTHER posts + - Return job IDs for tracking +validation: + - command: "curl -X POST http://localhost:3000/api/analysis/analyze-post -d '{"postId": 1}'" + - expect: "Returns job queue information" +``` + +### Task 2: Implement Enhanced Semantic Analyzer +```yaml +task_name: create_enhanced_semantic_analyzer +action: CREATE +file: src/lib/semantic-link-analyzer.ts +changes: | + - Use Claude SDK (@anthropic-ai/sdk) + - Implement bi-directional analysis logic + - Extract link text (max 4 words) + - Calculate confidence scores + - Generate reasoning for each suggestion + - Handle rate limiting (max 20 concurrent) +validation: + - command: "npm run type-check" + - expect: "No type errors" +``` + +### Task 3: Create Analysis Queue Management +```yaml +task_name: create_queue_management_api +action: CREATE +file: src/app/api/analysis/queue/route.ts +changes: | + - GET endpoint for queue status + - Show queued, processing, completed jobs + - Real-time progress updates + - Support batch processing +validation: + - command: "curl http://localhost:3000/api/analysis/queue" + - expect: "Returns queue status with job details" +``` + +### Task 4: Add Analysis Tab Component +```yaml +task_name: create_analysis_tab_component +action: CREATE +file: src/app/components/AnalysisTab.tsx +changes: | + - Display incoming link suggestions + - Display outgoing link suggestions + - Show confidence scores and reasoning + - Approve/reject functionality + - Group by source/target post +validation: + - command: "npm run dev" + - expect: "Analysis tab displays link suggestions" +``` + +### Task 5: Implement Link Preview Component +```yaml +task_name: create_link_preview_component +action: CREATE +file: src/app/components/LinkPreview.tsx +changes: | + - Show surrounding context (before/after) + - Highlight proposed link text + - Display as it would appear with HTML + - Show target post title on hover +validation: + - command: "npm run dev" + - expect: "Link previews show context" +``` + +### Task 6: Update EnhancedBlogViewer +```yaml +task_name: update_blog_viewer_for_analysis +action: MODIFY +file: src/app/components/EnhancedBlogViewer.tsx +changes: | + - Add "Analyze" button to header + - Add "Analysis" tab to tab list + - Integrate AnalysisTab component + - Handle analysis state and progress +validation: + - command: "npm run type-check" + - expect: "No type errors" +``` + +### Task 7: Enhance Export with HTML Links +```yaml +task_name: enhance_csv_export +action: MODIFY +file: src/app/api/export/csv/route.ts +changes: | + - Apply approved links as HTML tags + - Include proper rel attributes + - Maintain original CSV format + - Only include approved links +validation: + - command: "npm run test-export" + - expect: "CSV contains HTML links" +``` + +### Task 8: Update Claude Prompt Strategy +```yaml +task_name: update_analysis_prompt +action: CREATE +file: src/lib/prompts/semantic-link-prompt.ts +changes: | + - Adapt prompt from example project + - Focus on 2-4 word link text + - Emphasize high-quality connections + - Return structured JSON response + - 70% confidence threshold +validation: + - command: "npm run type-check" + - expect: "Valid prompt structure" +``` + +### Task 9: Implement Queue Progress UI +```yaml +task_name: create_queue_progress_component +action: CREATE +file: src/app/components/QueueProgress.tsx +changes: | + - Real-time queue status display + - Show processing progress + - List active analysis jobs + - Cancel functionality +validation: + - command: "npm run dev" + - expect: "Queue progress displays correctly" +``` + +### Task 10: Add Database Schema Updates +```yaml +task_name: update_database_schema +action: MODIFY +file: prisma/schema.prisma +changes: | + - Add analysisType to AnalysisJob + - Add queuePosition field + - Add batchId for grouping + - Run migration +validation: + - command: "npx prisma migrate dev" + - expect: "Migration successful" +``` + +## Implementation Strategy + +### Dependencies Order +1. Database schema updates first (Task 10) +2. Core analysis infrastructure (Tasks 1, 2, 3) +3. UI components (Tasks 4, 5, 6, 9) +4. Prompt optimization (Task 8) +5. Export enhancement (Task 7) + +### Claude API Integration +```typescript +// Example structure for semantic analysis +interface AnalysisResult { + sourcePostId: number; + targetPostId: number; + linkText: string; // 2-4 words + linkPosition: number; + confidence: number; // 0-100 + reasoning: string; + contextBefore: string; + contextAfter: string; +} +``` + +### Rate Limiting Strategy +- Maximum 20 concurrent Claude API calls +- Queue-based processing with backpressure +- Retry failed analyses up to 3 times +- Exponential backoff for rate limits + +## Risk Assessment + +### Identified Risks +1. **API Costs**: Multiple analyses per post + - Mitigation: Efficient prompts, caching results + +2. **Performance**: Large-scale analysis load + - Mitigation: Queue management, batch processing + +3. **Link Quality**: False positives + - Mitigation: 70% confidence threshold, preview + +4. **User Experience**: Complex approval process + - Mitigation: Intuitive UI with bulk actions + +## User Interaction Points + +### Analysis Workflow +1. User clicks "Analyze" on a blog post +2. System queues bi-directional analysis +3. Progress shown in real-time +4. Results appear in Analysis tab +5. User reviews with preview capability +6. Approve/reject individual links +7. Export includes approved links + +### Bulk Operations +- Approve all high-confidence links (85%+) +- Filter by confidence level +- Sort by relevance score +- Group by source/target post + +## Success Criteria +- [ ] Individual posts can be analyzed on-demand +- [ ] Bi-directional links discovered accurately +- [ ] Queue progress visible in real-time +- [ ] Link previews show context clearly +- [ ] Confidence scores guide decisions +- [ ] Export produces valid HTML links +- [ ] Rate limiting prevents API overuse + +## Technical Specifications + +### Prompt Structure (Adapted) +```typescript +const ANALYSIS_PROMPT = ` +You are a conservative semantic link analyst... + +CRITICAL: Only suggest a link if ALL criteria are met: +1. STRONG SEMANTIC RELEVANCE +2. USER VALUE at specific point +3. NATURAL CONTEXT flow +4. SPECIFIC CONNECTION +5. CLEAR USER INTENT +6. NO EXISTING LINK +7. LINK TEXT LENGTH: 2-4 words only + +Respond with JSON: +{ + "shouldLink": boolean, + "linkText": "2-4 word phrase", + "confidence": 0-100, + "reasoning": "explanation" +} +`; +``` + +### Export Format +```csv +Content,"

Learn about service management best practices...

" +``` + +## Notes +- Leverage existing Prisma models and infrastructure +- Maintain compatibility with current link review system +- Focus on quality over quantity in suggestions +- Ensure single link per target post rule is enforced + +## Implementation Updates (Not in Original PRP) + +### Additional Changes Made During Implementation + +1. **Prisma Client Regeneration**: After database schema changes, must run `npx prisma generate` and restart the development server for changes to take effect. + +2. **Claude API Integration**: + - Attempted to use `@anthropic-ai/claude-code` package for authentication-free API access + - The package's `query` function returns an async generator of `SDKMessage` objects + - SDKMessage structure differs from standard Anthropic SDK response format + - Message content extraction requires checking multiple properties (`text`, `content`) with type guards + +3. **Actual Implementation Differences**: + - Created `semantic-link-analyzer.ts` as a new file rather than modifying existing `semantic-analyzer.ts` + - Implemented phrase extraction to analyze multiple 2-4 word segments per post + - Added proper type handling for different SDKMessage types from claude-code package + +4. **Authentication Consideration**: + - The `@anthropic-ai/claude-code` package is designed for CLI environments + - For web applications, may need alternative authentication approach or environment setup \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 97ae646..1d4bc2b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "blog-viewer", "version": "0.1.0", "dependencies": { + "@anthropic-ai/claude-code": "^1.0.61", "@anthropic-ai/sdk": "^0.57.0", "@prisma/client": "^6.12.0", "class-variance-authority": "^0.7.1", @@ -68,6 +69,235 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/claude-code": { + "version": "1.0.61", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code/-/claude-code-1.0.61.tgz", + "integrity": "sha512-+gjKzY1hsWfHoH52SgKR6E0ujCDPyyRsjyRShtZfS0urKd8VQq3D/DF3hvT3P4JJeW0YuWp5Dep0aSRON+/FFA==", + "license": "SEE LICENSE IN README.md", + "bin": { + "claude": "cli.js" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "^0.33.5", + "@img/sharp-darwin-x64": "^0.33.5", + "@img/sharp-linux-arm": "^0.33.5", + "@img/sharp-linux-arm64": "^0.33.5", + "@img/sharp-linux-x64": "^0.33.5", + "@img/sharp-win32-x64": "^0.33.5" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@anthropic-ai/claude-code/node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@anthropic-ai/sdk": { "version": "0.57.0", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.57.0.tgz", diff --git a/package.json b/package.json index 721d3b6..9653705 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "import-csv": "tsx scripts/import-csv.ts" }, "dependencies": { + "@anthropic-ai/claude-code": "^1.0.61", "@anthropic-ai/sdk": "^0.57.0", "@prisma/client": "^6.12.0", "class-variance-authority": "^0.7.1", diff --git a/prisma/database/blog.db b/prisma/database/blog.db index 140c52b..1257a5d 100644 Binary files a/prisma/database/blog.db and b/prisma/database/blog.db differ diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e74346e..a66f8fa 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -62,6 +62,9 @@ model SemanticLink { confidence Float status String @default("pending") // pending, approved, rejected reasoning String + contextBefore String? + contextAfter String? + analysisOutput String? // Raw analysis output from Claude createdAt DateTime @default(now()) reviewedAt DateTime? reviewedBy String? @@ -79,11 +82,17 @@ model SemanticLink { model AnalysisJob { id String @id @default(cuid()) postId Int + analysisType String @default("full") // full, incoming, outgoing status String @default("queued") // queued, processing, completed, failed progress Float @default(0) + queuePosition Int? + batchId String? startedAt DateTime? completedAt DateTime? error String? + analysisSummary String? // Summary of what was analyzed + linksFound Int @default(0) // Count of links found + linksAboveThreshold Int @default(0) // Count of links above 70% threshold createdAt DateTime @default(now()) updatedAt DateTime @updatedAt @@ -92,4 +101,5 @@ model AnalysisJob { @@index([status]) @@index([postId]) + @@index([batchId]) } \ No newline at end of file diff --git a/src/app/api/analysis/analyze-post/route.ts b/src/app/api/analysis/analyze-post/route.ts new file mode 100644 index 0000000..4fd732d --- /dev/null +++ b/src/app/api/analysis/analyze-post/route.ts @@ -0,0 +1,100 @@ +import { NextResponse } from "next/server"; +import { PrismaClient } from "@prisma/client"; +import { initializeJobProcessor } from "@/lib/job-processor"; + +const prisma = new PrismaClient(); + +export async function POST(request: Request) { + try { + const { postId } = await request.json(); + + if (!postId || typeof postId !== "number") { + return NextResponse.json({ success: false, error: "Invalid postId" }, { status: 400 }); + } + + // Verify post exists + const post = await prisma.blogPost.findUnique({ + where: { id: postId }, + }); + + if (!post) { + return NextResponse.json({ success: false, error: "Post not found" }, { status: 404 }); + } + + // Generate batch ID for this analysis + const batchId = `batch_${postId}_${Date.now()}`; + + // Get current queue size for position assignment + const queuedJobsCount = await prisma.analysisJob.count({ + where: { status: "queued" }, + }); + + // Get all other posts for bi-directional analysis + const otherPosts = await prisma.blogPost.findMany({ + where: { + id: { not: postId }, + content: { not: null }, + }, + select: { id: true }, + }); + + const jobs = []; + let queuePosition = queuedJobsCount; + + // Create job for finding links FROM this post TO other posts (outgoing) + const outgoingJob = await prisma.analysisJob.create({ + data: { + postId, + analysisType: "outgoing", + status: "queued", + progress: 0, + batchId, + queuePosition: queuePosition++, + }, + }); + jobs.push(outgoingJob); + + // Create jobs for finding links FROM other posts TO this post (incoming) + for (const otherPost of otherPosts) { + const incomingJob = await prisma.analysisJob.create({ + data: { + postId: otherPost.id, + analysisType: "incoming", + status: "queued", + progress: 0, + batchId, + queuePosition: queuePosition++, + }, + }); + jobs.push(incomingJob); + } + + // Start job processing in background + const queue = initializeJobProcessor(); + if (!queue.isRunning()) { + queue.start().catch((error) => { + console.error("Queue processing error:", error); + }); + } + + return NextResponse.json({ + success: true, + message: `Created ${jobs.length} analysis jobs for post ${postId}`, + batchId, + jobs: jobs.map((job) => ({ + id: job.id, + type: job.analysisType, + postId: job.postId, + queuePosition: job.queuePosition, + })), + }); + } catch (error) { + console.error("Error creating analysis jobs:", error); + return NextResponse.json( + { success: false, error: "Failed to create analysis jobs" }, + { status: 500 } + ); + } finally { + await prisma.$disconnect(); + } +} diff --git a/src/app/api/analysis/queue/route.ts b/src/app/api/analysis/queue/route.ts new file mode 100644 index 0000000..546bf7b --- /dev/null +++ b/src/app/api/analysis/queue/route.ts @@ -0,0 +1,145 @@ +import { NextResponse } from "next/server"; +import { PrismaClient } from "@prisma/client"; + +const prisma = new PrismaClient(); + +export async function GET(request: Request) { + try { + const { searchParams } = new URL(request.url); + const batchId = searchParams.get("batchId"); + const postId = searchParams.get("postId"); + + // Build query filters + interface WhereFilter { + batchId?: string; + postId?: number; + } + const where: WhereFilter = {}; + if (batchId) where.batchId = batchId; + if (postId) where.postId = parseInt(postId); + + // Get all jobs with their related posts and semantic links + const jobs = await prisma.analysisJob.findMany({ + where, + include: { + post: { + include: { + linksFrom: { + select: { + id: true, + linkText: true, + confidence: true, + reasoning: true, + analysisOutput: true, + targetPost: { + select: { + name: true, + }, + }, + }, + orderBy: { + createdAt: "desc", + }, + take: 10, // Limit to most recent 10 links per job + }, + }, + }, + }, + orderBy: [ + { status: "asc" }, // Show queued first, then processing, then completed + { queuePosition: "asc" }, + { createdAt: "asc" }, + ], + }); + + // Get queue statistics + const stats = await prisma.analysisJob.groupBy({ + by: ["status"], + where: batchId ? { batchId } : undefined, + _count: true, + }); + + const statsMap = stats.reduce( + (acc, stat) => { + acc[stat.status] = stat._count; + return acc; + }, + {} as Record + ); + + // Get active batches if no specific batch requested + let activeBatches: { batchId: string; count: number; minPosition: number | null }[] = []; + if (!batchId) { + const batchGroups = await prisma.analysisJob.groupBy({ + by: ["batchId"], + where: { + status: { in: ["queued", "processing"] }, + batchId: { not: null }, + }, + _count: true, + _min: { + queuePosition: true, + }, + }); + + activeBatches = batchGroups + .filter((group) => group.batchId !== null) + .map((group) => ({ + batchId: group.batchId!, + count: group._count, + minPosition: group._min.queuePosition, + })) + .sort((a, b) => (a.minPosition || 0) - (b.minPosition || 0)); + } + + // Format response + const response = { + stats: { + queued: statsMap.queued || 0, + processing: statsMap.processing || 0, + completed: statsMap.completed || 0, + failed: statsMap.failed || 0, + total: jobs.length, + }, + activeBatches, + jobs: jobs.map((job) => ({ + id: job.id, + postId: job.postId, + postName: job.post.name, + postSlug: job.post.slug, + analysisType: job.analysisType, + status: job.status, + progress: job.progress, + queuePosition: job.queuePosition, + batchId: job.batchId, + error: job.error, + analysisSummary: job.analysisSummary, + linksFound: job.linksFound, + linksAboveThreshold: job.linksAboveThreshold, + createdAt: job.createdAt, + startedAt: job.startedAt, + completedAt: job.completedAt, + semanticLinks: job.post.linksFrom + ? job.post.linksFrom.map((link) => ({ + id: link.id, + linkText: link.linkText, + confidence: link.confidence, + reasoning: link.reasoning, + analysisOutput: link.analysisOutput, + targetPostName: link.targetPost.name, + })) + : [], + })), + }; + + return NextResponse.json(response); + } catch (error) { + console.error("Error fetching queue status:", error); + return NextResponse.json( + { success: false, error: "Failed to fetch queue status" }, + { status: 500 } + ); + } finally { + await prisma.$disconnect(); + } +} diff --git a/src/app/api/analysis/retry/route.ts b/src/app/api/analysis/retry/route.ts new file mode 100644 index 0000000..9d4fa56 --- /dev/null +++ b/src/app/api/analysis/retry/route.ts @@ -0,0 +1,62 @@ +import { NextRequest, NextResponse } from "next/server"; +import { PrismaClient } from "@prisma/client"; +import { getJobQueue } from "@/lib/job-queue"; + +const prisma = new PrismaClient(); + +export async function POST(request: NextRequest) { + try { + const { jobIds } = await request.json(); + + if (!jobIds || !Array.isArray(jobIds)) { + return NextResponse.json({ error: "jobIds array is required" }, { status: 400 }); + } + + // Get failed jobs + const failedJobs = await prisma.analysisJob.findMany({ + where: { + id: { in: jobIds }, + status: "failed", + }, + include: { + post: true, + }, + }); + + if (failedJobs.length === 0) { + return NextResponse.json({ message: "No failed jobs found to retry" }); + } + + // Reset jobs to queued status + await prisma.analysisJob.updateMany({ + where: { + id: { in: failedJobs.map((job) => job.id) }, + }, + data: { + status: "queued", + error: null, + progress: 0, + startedAt: null, + completedAt: null, + analysisSummary: null, + linksFound: 0, + linksAboveThreshold: 0, + }, + }); + + // Start the job queue to process retried jobs + const queue = getJobQueue(); + queue.start(); + + return NextResponse.json({ + success: true, + retriedCount: failedJobs.length, + jobIds: failedJobs.map((job) => job.id), + }); + } catch (error) { + console.error("Error retrying jobs:", error); + return NextResponse.json({ error: "Failed to retry jobs" }, { status: 500 }); + } finally { + await prisma.$disconnect(); + } +} diff --git a/src/app/components/AnalysisTab.tsx b/src/app/components/AnalysisTab.tsx new file mode 100644 index 0000000..06e5b22 --- /dev/null +++ b/src/app/components/AnalysisTab.tsx @@ -0,0 +1,465 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Check, X, ArrowRight, ArrowLeft, XCircle } from "lucide-react"; +import LinkPreview from "./LinkPreview"; + +interface SemanticLink { + id: string; + sourcePostId: number; + targetPostId: number; + linkText: string; + linkPosition: number; + altText: string; + confidence: number; + status: string; + reasoning: string; + contextBefore?: string; + contextAfter?: string; + analysisOutput?: string; + sourcePost: { + id: number; + name: string; + slug: string; + }; + targetPost: { + id: number; + name: string; + slug: string; + }; +} + +interface AnalysisTabProps { + postId: number; +} + +interface FailedJob { + id: string; + postName: string; + error: string; +} + +export default function AnalysisTab({ postId }: AnalysisTabProps) { + const [incomingLinks, setIncomingLinks] = useState([]); + const [outgoingLinks, setOutgoingLinks] = useState([]); + const [loading, setLoading] = useState(true); + const [activeTab, setActiveTab] = useState<"incoming" | "outgoing">("outgoing"); + const [expandedAnalysis, setExpandedAnalysis] = useState(null); + const [failedJobs, setFailedJobs] = useState([]); + + useEffect(() => { + fetchLinks(); + }, [postId]); + + const fetchLinks = async () => { + setLoading(true); + try { + // Fetch semantic links + const linksResponse = await fetch(`/api/links?postId=${postId}`); + const linksData = await linksResponse.json(); + + // Separate incoming and outgoing links + const incoming = linksData.links.filter((link: SemanticLink) => link.targetPostId === postId); + const outgoing = linksData.links.filter((link: SemanticLink) => link.sourcePostId === postId); + + setIncomingLinks(incoming); + setOutgoingLinks(outgoing); + + // Fetch failed jobs for this post + const queueResponse = await fetch(`/api/analysis/queue?postId=${postId}`); + const queueData = await queueResponse.json(); + + const failed = + queueData.jobs + ?.filter((job: any) => job.status === "failed") + .map((job: any) => ({ + id: job.id, + postName: job.postName, + error: job.error || "Analysis failed", + })) || []; + + setFailedJobs(failed); + } catch (error) { + console.error("Error fetching links:", error); + } finally { + setLoading(false); + } + }; + + const updateLinkStatus = async (linkId: string, status: "approved" | "rejected" | "pending") => { + try { + const response = await fetch(`/api/links/${linkId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ status }), + }); + + if (response.ok) { + // Update local state + const updateLinks = (links: SemanticLink[]) => + links.map((link) => (link.id === linkId ? { ...link, status } : link)); + + setIncomingLinks(updateLinks); + setOutgoingLinks(updateLinks); + } + } catch (error) { + console.error("Error updating link:", error); + } + }; + + const bulkApproveHighConfidence = async () => { + const links = activeTab === "incoming" ? incomingLinks : outgoingLinks; + const highConfidenceLinks = links.filter( + (link) => link.confidence >= 85 && link.status === "pending" + ); + + try { + const response = await fetch("/api/links/bulk-update", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + ids: highConfidenceLinks.map((link) => link.id), + status: "approved", + }), + }); + + if (response.ok) { + await fetchLinks(); + } + } catch (error) { + console.error("Error bulk approving:", error); + } + }; + + const getConfidenceColor = (confidence: number) => { + if (confidence >= 85) return "text-green-600 bg-green-50"; + if (confidence >= 70) return "text-yellow-600 bg-yellow-50"; + return "text-gray-600 bg-gray-50"; + }; + + const renderLinkGroup = (links: SemanticLink[], direction: "incoming" | "outgoing") => { + const pendingLinks = links.filter((link) => link.status === "pending"); + const approvedLinks = links.filter((link) => link.status === "approved"); + const rejectedLinks = links.filter((link) => link.status === "rejected"); + + return ( +
+ {pendingLinks.length > 0 && ( +
+
+

+ Pending Review + + ({pendingLinks.length}) + +

+ {pendingLinks.some((link) => link.confidence >= 85) && ( + + )} +
+
+ {pendingLinks.map((link) => ( +
+
+
+ {/* Link Direction Header */} +
+
+ From: + {direction === "incoming" ? ( + <> + + {link.sourcePost.name} + + + This post + + ) : ( + <> + This post + + + {link.targetPost.name} + + + )} +
+
+ + {/* Link Text and Confidence */} +
+
+ Link text: + + {link.linkText} + + + {link.confidence}% confidence + +
+ + {/* Reasoning */} +
+

{link.reasoning}

+
+
+ + {/* Preview Section */} + {link.contextBefore && link.contextAfter && ( +
+

+ Preview: +

+ +
+ )} + + {/* Claude's Analysis */} + {link.analysisOutput && ( +
+ + {expandedAnalysis === link.id && ( +
+ 
+                                {link.analysisOutput}
+
+
+ )} +
+ )} +
+ + {/* Action Buttons */} +
+ + +
+
+
+ ))} +
+
+ )} + + {approvedLinks.length > 0 && ( +
+

+ Approved + + ({approvedLinks.length}) + +

+
+ {approvedLinks.map((link) => ( +
+
+
+ + {link.linkText} + + + {direction === "incoming" ? link.sourcePost.name : link.targetPost.name} + +
+ +
+
+ ))} +
+
+ )} + + {rejectedLinks.length > 0 && ( +
+

+ Rejected + + ({rejectedLinks.length}) + +

+
+ {rejectedLinks.map((link) => ( +
+
+
+ + + {link.linkText} + + + + {direction === "incoming" ? link.sourcePost.name : link.targetPost.name} + +
+ +
+
+ ))} +
+
+ )} +
+ ); + }; + + if (loading) { + return ( +
+
+
+ ); + } + + return ( +
+ {/* Tab Navigation */} +
+
+ + +
+
+ + {/* Tab Content */} +
+ {activeTab === "incoming" && ( +
+

+ Links from other posts that could point to this post +

+ {incomingLinks.length === 0 ? ( +
+

No incoming links found

+
+ ) : ( + renderLinkGroup(incomingLinks, "incoming") + )} +
+ )} + + {activeTab === "outgoing" && ( +
+

+ Links from this post to other posts +

+ {outgoingLinks.length === 0 ? ( +
+

No outgoing links found

+
+ ) : ( + renderLinkGroup(outgoingLinks, "outgoing") + )} +
+ )} +
+ + {failedJobs.length > 0 && ( +
+
+ +

+ Failed Analyses ({failedJobs.length}) +

+
+
+ {failedJobs.map((job) => ( +
+

{job.postName}

+

{job.error}

+

+ Retry this analysis from the Queue Progress sidebar +

+
+ ))} +
+
+ )} +
+ ); +} diff --git a/src/app/components/EnhancedBlogViewer.tsx b/src/app/components/EnhancedBlogViewer.tsx index 49e9403..74b7cb8 100644 --- a/src/app/components/EnhancedBlogViewer.tsx +++ b/src/app/components/EnhancedBlogViewer.tsx @@ -4,8 +4,10 @@ import { useEffect, useState } from "react"; import { useRouter } from "next/navigation"; import { EnhancedBlogSidebar } from "./EnhancedBlogSidebar"; import { BlogPost } from "@prisma/client"; -import { Check, X, ExternalLink } from "lucide-react"; +import { Check, X, ExternalLink, Sparkles, FileText } from "lucide-react"; import { DatabaseInfoPanel } from "./DatabaseInfoPanel"; +import AnalysisTab from "./AnalysisTab"; +import QueueProgress from "./QueueProgress"; interface SemanticLink { id: string; @@ -17,6 +19,7 @@ interface SemanticLink { confidence: number; status: string; reasoning: string; + analysisOutput?: string; sourcePost: { id: number; name: string; @@ -71,10 +74,12 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl const [links, setLinks] = useState([]); const [analysisStatus, setAnalysisStatus] = useState(null); const [isAnalyzing, setIsAnalyzing] = useState(false); - const [activeTab, setActiveTab] = useState<"content" | "updated" | "from" | "to" | "database">( - "content" - ); + const [activeTab, setActiveTab] = useState< + "content" | "updated" | "from" | "to" | "database" | "analysis" + >("content"); const [updatedContent, setUpdatedContent] = useState(""); + const [postAnalysisBatchId, setPostAnalysisBatchId] = useState(null); + const [showQueueProgress, setShowQueueProgress] = useState(false); // Check if any analysis has been performed const hasAnalysis = analysisStatus && analysisStatus.total > 0; @@ -91,6 +96,8 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl useEffect(() => { if (post && hasAnalysis) { fetchLinksForPost(post.id); + // Check if there's an active analysis for this post + checkActiveAnalysisForPost(post.id); } }, [post, hasAnalysis]); @@ -109,6 +116,16 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl } }, [isAnalyzing]); + useEffect(() => { + // Poll for new links during analysis + if (isAnalyzing && post) { + const interval = setInterval(() => { + fetchLinksForPost(post.id); + }, 3000); // Poll every 3 seconds + return () => clearInterval(interval); + } + }, [isAnalyzing, post]); + useEffect(() => { const handleKeyPress = (e: KeyboardEvent) => { if (e.key === "ArrowLeft" && navigation?.hasPrevious) { @@ -200,6 +217,29 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl } }; + const checkActiveAnalysisForPost = async (postId: number) => { + try { + const response = await fetch(`/api/analysis/queue?postId=${postId}`); + const data = await response.json(); + + if (data.jobs && data.jobs.length > 0) { + // Check if any jobs are in progress for this post + const activeJob = data.jobs.find( + (job: { status: string; batchId: string }) => + job.status === "queued" || job.status === "processing" + ); + + if (activeJob) { + setIsAnalyzing(true); + setPostAnalysisBatchId(activeJob.batchId); + setShowQueueProgress(true); + } + } + } catch (error) { + console.error("Error checking active analysis for post:", error); + } + }; + const fetchLinksForPost = async (postId: number) => { try { const response = await fetch(`/api/links?postId=${postId}`); @@ -271,6 +311,33 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl } }; + const analyzeCurrentPost = async () => { + if (!post) return; + + try { + setIsAnalyzing(true); + const response = await fetch("/api/analysis/analyze-post", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ postId: post.id }), + }); + + const data = await response.json(); + if (data.success) { + setPostAnalysisBatchId(data.batchId); + setShowQueueProgress(true); + // Don't set isAnalyzing to false here - let checkAnalysisStatus handle it + } else { + alert("Failed to start analysis"); + setIsAnalyzing(false); + } + } catch (error) { + console.error("Error analyzing post:", error); + alert("Error analyzing post"); + setIsAnalyzing(false); + } + }; + const handleLinkUpdate = async (linkId: string, status: string) => { try { const response = await fetch(`/api/links/${linkId}`, { @@ -420,6 +487,7 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl const LinkItem = ({ link, type }: { link: SemanticLink; type: "from" | "to" }) => { const isFrom = type === "from"; const relatedPost = isFrom ? link.sourcePost : link.targetPost; + const [showAnalysis, setShowAnalysis] = useState(false); return (
@@ -483,7 +551,25 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl View + {link.analysisOutput && ( + + )}
+ + {showAnalysis && link.analysisOutput && ( +
+
Claude's Analysis:
+ 
+              {link.analysisOutput}
+
+
+ )} ); }; @@ -560,6 +646,17 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl > Database Info + )} @@ -638,6 +735,48 @@ export function EnhancedBlogViewer({ initialIndex = 0, initialSlug }: EnhancedBl )} {activeTab === "database" && } + + {activeTab === "analysis" && ( +
+
+
+

+ Semantic Link Analysis +

+

+ Analyze this post to find potential links to and from other posts +

+
+ +
+ + {showQueueProgress && postAnalysisBatchId && ( + { + setShowQueueProgress(false); + // Refresh links when closing queue progress + if (post) { + fetchLinksForPost(post.id); + } + }} + /> + )} + + +
+ )} )} diff --git a/src/app/components/LinkPreview.tsx b/src/app/components/LinkPreview.tsx new file mode 100644 index 0000000..8e76cee --- /dev/null +++ b/src/app/components/LinkPreview.tsx @@ -0,0 +1,37 @@ +"use client"; + +interface LinkPreviewProps { + contextBefore: string; + contextAfter: string; + linkText: string; + targetTitle: string; +} + +export default function LinkPreview({ + contextBefore, + contextAfter, + linkText, + targetTitle, +}: LinkPreviewProps) { + return ( +
+
+ {contextBefore} + + e.preventDefault()} + > + {linkText} + + + {targetTitle} + + + + {contextAfter} +
+
+ ); +} diff --git a/src/app/components/QueueProgress.tsx b/src/app/components/QueueProgress.tsx new file mode 100644 index 0000000..c20178a --- /dev/null +++ b/src/app/components/QueueProgress.tsx @@ -0,0 +1,348 @@ +"use client"; + +import { useState, useEffect } from "react"; +import { Clock, CheckCircle, XCircle, Loader2, RefreshCw } from "lucide-react"; + +interface SemanticLink { + id: string; + linkText: string; + confidence: number; + reasoning: string; + analysisOutput: string | null; + targetPostName: string; +} + +interface AnalysisJob { + id: string; + postId: number; + postName: string; + postSlug: string; + analysisType: string; + status: string; + progress: number; + queuePosition: number | null; + batchId: string | null; + error: string | null; + analysisSummary: string | null; + linksFound: number; + linksAboveThreshold: number; + createdAt: string; + startedAt: string | null; + completedAt: string | null; + semanticLinks: SemanticLink[]; +} + +interface QueueStats { + queued: number; + processing: number; + completed: number; + failed: number; + total: number; +} + +interface QueueProgressProps { + batchId?: string; + postId?: number; + onClose?: () => void; +} + +export default function QueueProgress({ batchId, postId, onClose }: QueueProgressProps) { + const [jobs, setJobs] = useState([]); + const [stats, setStats] = useState({ + queued: 0, + processing: 0, + completed: 0, + failed: 0, + total: 0, + }); + const [loading, setLoading] = useState(true); + const [expandedSummaries, setExpandedSummaries] = useState>(new Set()); + const [expandedClaudeOutputs, setExpandedClaudeOutputs] = useState>(new Set()); + const [retryingJobs, setRetryingJobs] = useState>(new Set()); + + useEffect(() => { + fetchQueueStatus(); + const interval = setInterval(fetchQueueStatus, 2000); // Poll every 2 seconds + return () => clearInterval(interval); + }, [batchId, postId]); + + const fetchQueueStatus = async () => { + try { + const params = new URLSearchParams(); + if (batchId) params.append("batchId", batchId); + if (postId) params.append("postId", postId.toString()); + + const response = await fetch(`/api/analysis/queue?${params}`); + const data = await response.json(); + + setJobs(data.jobs || []); + setStats(data.stats || stats); + } catch (error) { + console.error("Error fetching queue status:", error); + } finally { + setLoading(false); + } + }; + + const retryFailedJobs = async (jobIds: string[]) => { + setRetryingJobs(new Set(jobIds)); + try { + const response = await fetch("/api/analysis/retry", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jobIds }), + }); + + if (response.ok) { + await fetchQueueStatus(); + } + } catch (error) { + console.error("Error retrying jobs:", error); + } finally { + setRetryingJobs(new Set()); + } + }; + + const getStatusIcon = (status: string) => { + switch (status) { + case "queued": + return ; + case "processing": + return ; + case "completed": + return ; + case "failed": + return ; + default: + return null; + } + }; + + const getStatusColor = (status: string) => { + switch (status) { + case "queued": + return "text-gray-600 bg-gray-50"; + case "processing": + return "text-blue-600 bg-blue-50"; + case "completed": + return "text-green-600 bg-green-50"; + case "failed": + return "text-red-600 bg-red-50"; + default: + return "text-gray-600 bg-gray-50"; + } + }; + + const formatDuration = (start: string | null, end: string | null) => { + if (!start) return "-"; + const startTime = new Date(start).getTime(); + const endTime = end ? new Date(end).getTime() : Date.now(); + const duration = Math.floor((endTime - startTime) / 1000); + + if (duration < 60) return `${duration}s`; + const minutes = Math.floor(duration / 60); + const seconds = duration % 60; + return `${minutes}m ${seconds}s`; + }; + + if (loading) { + return ( +
+ +
+ ); + } + + const progressPercentage = + stats.total > 0 ? Math.round(((stats.completed + stats.failed) / stats.total) * 100) : 0; + + return ( +
+ {/* Overall Progress */} +
+
+

Analysis Progress

+ {onClose && ( + + )} +
+ +
+
+ Overall Progress + {progressPercentage}% +
+
+
+
+
+ +
+
+

{stats.queued}

+

Queued

+
+
+

{stats.processing}

+

Processing

+
+
+

{stats.completed}

+

Completed

+
+
+

{stats.failed}

+

Failed

+
+
+
+ + {/* Job List */} +
+
+

Analysis Jobs

+
+
+ {jobs.length === 0 ? ( +
+ No analysis jobs found +
+ ) : ( + jobs.map((job) => ( +
+
+
+ {getStatusIcon(job.status)} +
+

{job.postName}

+

+ {job.analysisType === "incoming" ? "Incoming links" : "Outgoing links"} + {job.queuePosition !== null && + job.status === "queued" && + ` • Position ${job.queuePosition}`} +

+
+
+
+ {job.status === "processing" && job.progress > 0 && ( +
+
+ {Math.round(job.progress)}% +
+
+
+
+
+ )} +
+ + {job.status} + +

+ {formatDuration(job.startedAt, job.completedAt)} +

+
+
+
+ {job.error && ( +
+

{job.error}

+ {job.status === "failed" && ( + + )} +
+ )} + {job.analysisSummary && ( +
+ + {expandedSummaries.has(job.id) && ( +
+ 
+                          {job.analysisSummary}
+
+
+ )} +
+ )} + {job.semanticLinks && job.semanticLinks.length > 0 && ( +
+ + {expandedClaudeOutputs.has(job.id) && ( +
+ {job.semanticLinks.map((link) => ( +
+
+ + Link to "{link.targetPostName}": + + + "{link.linkText}" ({link.confidence}% confidence) + +
+ {link.analysisOutput && ( +
+
Claude's Analysis:
+ 
+                                  {link.analysisOutput}
+
+
+ )} +
+ ))} +
+ )} +
+ )} +
+ )) + )} +
+
+
+ ); +} diff --git a/src/lib/job-processor.ts b/src/lib/job-processor.ts index 20ebea9..fcd9387 100644 --- a/src/lib/job-processor.ts +++ b/src/lib/job-processor.ts @@ -1,98 +1,65 @@ import { PrismaClient, AnalysisJob } from "@prisma/client"; -import { SemanticAnalyzer } from "./semantic-analyzer"; +import { SemanticLinkAnalyzer } from "./semantic-link-analyzer"; import { getJobQueue } from "./job-queue"; const prisma = new PrismaClient(); export async function processAnalysisJob(job: AnalysisJob): Promise { - const analyzer = new SemanticAnalyzer(); + const analyzer = new SemanticLinkAnalyzer(); try { - // Get the source post - const sourcePost = await prisma.blogPost.findUnique({ - where: { id: job.postId }, - }); - - if (!sourcePost || !sourcePost.content) { - throw new Error("Source post not found or has no content"); - } - - // Get all other posts to analyze against - const targetPosts = await prisma.blogPost.findMany({ - where: { - id: { not: job.postId }, - content: { not: null }, + // Update job status to processing + await prisma.analysisJob.update({ + where: { id: job.id }, + data: { + status: "processing", + startedAt: new Date(), }, }); - const totalTargets = targetPosts.length; - let processedTargets = 0; + // Get the post for this job + const post = await prisma.blogPost.findUnique({ + where: { id: job.postId }, + }); - // Analyze against each target post - for (const targetPost of targetPosts) { - if (!targetPost.content) continue; + if (!post || !post.content) { + throw new Error("Post not found or has no content"); + } - try { - // Check if link already exists - const existingLink = await prisma.semanticLink.findFirst({ - where: { - sourcePostId: sourcePost.id, - targetPostId: targetPost.id, - }, + // Handle different analysis types + if (job.analysisType === "outgoing") { + // Find links FROM this post TO other posts + await processOutgoingLinks( + analyzer, + { id: post.id, name: post.name, content: post.content }, + job + ); + } else if (job.analysisType === "incoming") { + // Find links FROM other posts TO this specific post + // Get the target post based on batchId + const batchMatch = job.batchId?.match(/batch_(\d+)_/); + if (batchMatch) { + const targetPostId = parseInt(batchMatch[1]); + const targetPost = await prisma.blogPost.findUnique({ + where: { id: targetPostId }, }); - if (!existingLink) { - // Analyze semantic relationship - const result = await analyzer.analyzeSemanticLink( - sourcePost.content, - targetPost.content, - sourcePost.name, - targetPost.name + if (targetPost && targetPost.content) { + await processIncomingLink( + analyzer, + { id: post.id, name: post.name, content: post.content }, + { id: targetPost.id, name: targetPost.name, content: targetPost.content }, + job ); - - if (result.shouldLink && result.confidence >= 70) { - // Find exact position of link text - const position = analyzer.findLinkPosition(sourcePost.content, result.linkText); - - if (position !== -1) { - // Check if text is already linked - const alreadyLinked = analyzer.isAlreadyLinked( - sourcePost.content, - position, - result.linkText.length - ); - - if (!alreadyLinked) { - // Create semantic link record - await prisma.semanticLink.create({ - data: { - sourcePostId: sourcePost.id, - targetPostId: targetPost.id, - linkText: result.linkText, - linkPosition: position, - altText: result.altText, - confidence: result.confidence, - reasoning: result.reasoning, - status: "pending", - }, - }); - } - } - } } - - processedTargets++; - - // Update job progress - const progress = (processedTargets / totalTargets) * 100; - await prisma.analysisJob.update({ - where: { id: job.id }, - data: { progress }, - }); - } catch (error) { - console.error(`Error analyzing ${sourcePost.name} -> ${targetPost.name}:`, error); - // Continue with next target } + } else { + // Full analysis (legacy mode) + await processFullAnalysis( + analyzer, + { id: post.id, name: post.name, content: post.content }, + job + ); } // Mark job as completed @@ -121,6 +88,242 @@ export async function processAnalysisJob(job: AnalysisJob): Promise { } } +async function processOutgoingLinks( + analyzer: SemanticLinkAnalyzer, + sourcePost: { id: number; name: string; content: string }, + job: AnalysisJob +): Promise { + // Get all other posts as potential targets + const targetPosts = await prisma.blogPost.findMany({ + where: { + id: { not: sourcePost.id }, + content: { not: null }, + }, + }); + + const totalTargets = targetPosts.length; + let processedTargets = 0; + + for (const targetPost of targetPosts) { + if (!targetPost.content) continue; + + try { + // Analyze for outgoing links + const results = await analyzer.analyzeSemanticLink( + sourcePost.content, + targetPost.content, + sourcePost.name, + targetPost.name, + { analysisType: "outgoing" } + ); + + // Save all suggested links + for (const result of results) { + if (result.shouldLink && result.confidence >= 70 && result.linkPosition !== undefined) { + // Check if link already exists + const existingLink = await prisma.semanticLink.findFirst({ + where: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + }, + }); + + if (!existingLink) { + await prisma.semanticLink.create({ + data: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + altText: result.altText, + confidence: result.confidence, + reasoning: result.reasoning, + contextBefore: result.contextBefore, + contextAfter: result.contextAfter, + status: "pending", + }, + }); + } + } + } + + processedTargets++; + const progress = (processedTargets / totalTargets) * 100; + await prisma.analysisJob.update({ + where: { id: job.id }, + data: { progress }, + }); + } catch (error) { + console.error(`Error analyzing ${sourcePost.name} -> ${targetPost.name}:`, error); + } + } +} + +async function processIncomingLink( + analyzer: SemanticLinkAnalyzer, + sourcePost: { id: number; name: string; content: string }, + targetPost: { id: number; name: string; content: string }, + job: AnalysisJob +): Promise { + try { + // Analyze for incoming links (from sourcePost TO targetPost) + const results = await analyzer.analyzeSemanticLink( + sourcePost.content, + targetPost.content, + sourcePost.name, + targetPost.name, + { analysisType: "incoming", targetPostId: targetPost.id } + ); + + const foundLink = results.length > 0 && results[0].shouldLink && results[0].confidence >= 70; + + console.log(`Analysis results for ${sourcePost.name} -> ${targetPost.name}:`, { + foundLink, + linkText: foundLink ? results[0].linkText : "none", + confidence: foundLink ? results[0].confidence : 0, + }); + + // Update job with analysis summary + const analysisSummary = + `Analyzed "${sourcePost.name}" → "${targetPost.name}"\n` + + `Claude analyzed the entire posts for the best semantic link.\n` + + (foundLink + ? `\nFound link: "${results[0].linkText}" (${results[0].confidence}% confidence)\n` + + `Reasoning: ${results[0].reasoning}` + : "\nNo semantic link found. Claude determined there was no strong enough connection between these posts."); + + await prisma.analysisJob.update({ + where: { id: job.id }, + data: { + analysisSummary, + linksFound: results.length, + linksAboveThreshold: foundLink ? 1 : 0, + }, + }); + + // Save all suggested links + for (const result of results) { + if (result.shouldLink && result.confidence >= 70 && result.linkPosition !== undefined) { + // Check if link already exists + const existingLink = await prisma.semanticLink.findFirst({ + where: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + }, + }); + + if (!existingLink) { + console.log( + `Creating semantic link: "${result.linkText}" at position ${result.linkPosition} with confidence ${result.confidence}` + ); + await prisma.semanticLink.create({ + data: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + altText: result.altText, + confidence: result.confidence, + reasoning: result.reasoning, + contextBefore: result.contextBefore, + contextAfter: result.contextAfter, + analysisOutput: result.analysisOutput, + status: "pending", + }, + }); + } else { + console.log( + `Skipping duplicate link: "${result.linkText}" at position ${result.linkPosition}` + ); + } + } + } + + // Update progress to 100% when done + await prisma.analysisJob.update({ + where: { id: job.id }, + data: { progress: 100 }, + }); + } catch (error) { + console.error(`Error analyzing incoming link ${sourcePost.name} -> ${targetPost.name}:`, error); + throw error; + } +} + +async function processFullAnalysis( + analyzer: SemanticLinkAnalyzer, + sourcePost: { id: number; name: string; content: string }, + job: AnalysisJob +): Promise { + // Legacy full analysis mode + const targetPosts = await prisma.blogPost.findMany({ + where: { + id: { not: sourcePost.id }, + content: { not: null }, + }, + }); + + const totalTargets = targetPosts.length; + let processedTargets = 0; + + for (const targetPost of targetPosts) { + if (!targetPost.content) continue; + + try { + const results = await analyzer.analyzeSemanticLink( + sourcePost.content, + targetPost.content, + sourcePost.name, + targetPost.name, + { analysisType: "full" } + ); + + for (const result of results) { + if (result.shouldLink && result.confidence >= 70 && result.linkPosition !== undefined) { + const existingLink = await prisma.semanticLink.findFirst({ + where: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + }, + }); + + if (!existingLink) { + await prisma.semanticLink.create({ + data: { + sourcePostId: sourcePost.id, + targetPostId: targetPost.id, + linkText: result.linkText, + linkPosition: result.linkPosition, + altText: result.altText, + confidence: result.confidence, + reasoning: result.reasoning, + contextBefore: result.contextBefore, + contextAfter: result.contextAfter, + status: "pending", + }, + }); + } + } + } + + processedTargets++; + const progress = (processedTargets / totalTargets) * 100; + await prisma.analysisJob.update({ + where: { id: job.id }, + data: { progress }, + }); + } catch (error) { + console.error(`Error analyzing ${sourcePost.name} -> ${targetPost.name}:`, error); + } + } +} + // Initialize the job queue with the processor export function initializeJobProcessor() { const queue = getJobQueue(); diff --git a/src/lib/job-queue.ts b/src/lib/job-queue.ts index 5f3dc5f..3e19076 100644 --- a/src/lib/job-queue.ts +++ b/src/lib/job-queue.ts @@ -28,15 +28,16 @@ export class JobQueue extends EventEmitter { throw new Error("No processor set for job queue"); } + console.log(`Starting job queue with ${this.workerCount} workers`); this.isProcessing = true; this.emit("started"); - // Start worker loops - const workers = Array(this.workerCount) - .fill(null) - .map((_, index) => this.workerLoop(index)); - - await Promise.all(workers); + // Start worker loops in background - don't await them + for (let i = 0; i < this.workerCount; i++) { + this.workerLoop(i).catch((error) => { + console.error(`Worker ${i} crashed:`, error); + }); + } } stop() { @@ -45,6 +46,7 @@ export class JobQueue extends EventEmitter { } private async workerLoop(workerId: number) { + console.log(`Worker ${workerId} started`); while (this.isProcessing) { try { // Get next available job @@ -56,6 +58,7 @@ export class JobQueue extends EventEmitter { continue; } + console.log(`Worker ${workerId} processing job ${job.id} for post ${job.postId}`); this.activeWorkers++; this.emit("job:started", { workerId, job }); @@ -63,6 +66,7 @@ export class JobQueue extends EventEmitter { await this.processor!(job); this.activeWorkers--; + console.log(`Worker ${workerId} completed job ${job.id}`); this.emit("job:completed", { workerId, job }); } catch (error) { this.activeWorkers--; @@ -70,6 +74,7 @@ export class JobQueue extends EventEmitter { this.emit("job:failed", { workerId, error }); } } + console.log(`Worker ${workerId} stopped`); } private async getNextJob(): Promise { diff --git a/src/lib/prompts/semantic-link-prompt.ts b/src/lib/prompts/semantic-link-prompt.ts new file mode 100644 index 0000000..aaa0c2c --- /dev/null +++ b/src/lib/prompts/semantic-link-prompt.ts @@ -0,0 +1,75 @@ +export const SEMANTIC_LINK_ANALYSIS_PROMPT = ` +You are an intelligent semantic link analyst reviewing blog content for potential cross-references. + +Your task is to analyze two blog posts and identify the SINGLE BEST phrase in the source post that should link to the target post. + +CRITICAL: Only suggest a link if ALL of the following criteria are met: +1. STRONG SEMANTIC RELEVANCE - The content is directly and specifically related +2. USER VALUE - The link provides clear value to the reader at that specific point +3. NATURAL CONTEXT FLOW - The link fits naturally within the sentence flow +4. SPECIFIC CONNECTION - The relationship is concrete, not generic or tangential +5. CLEAR USER INTENT - A reader would likely want to explore this connection +6. NO EXISTING LINK - The text is not already part of an HTML link (look for tags) +7. LINK TEXT LENGTH - The selected text must be 2-4 words only + +Additional considerations: +- Links should enhance understanding, not distract +- Avoid linking common terms or generic concepts +- Focus on unique insights, methodologies, or specific topics +- Consider the reader's journey and information needs +- Look for phrases that reference concepts explained in detail in the target post +- Prioritize actionable phrases (e.g., "implement authentication", "optimize performance") +- Choose the MOST VALUABLE link location - where users would benefit most + +You will analyze the ENTIRE source post and select the SINGLE BEST link to the target post. + +Respond with a JSON object containing either one suggested link or no link: +{ + "shouldLink": true/false, + "link": { + "linkText": "exact 2-4 word phrase from source post", + "confidence": 85, + "reasoning": "This phrase directly relates to the authentication system explained in the target post", + "contextBefore": "50 chars before the phrase", + "contextAfter": "50 chars after the phrase" + } +} + +IMPORTANT: +- Return only ONE link per source-target pair +- Choose the most valuable and relevant link location +- If no good link exists, set shouldLink to false and link to null +- The link must be highly relevant and valuable to justify inclusion`; + +export interface SemanticLinkSuggestion { + linkText: string; + confidence: number; + reasoning: string; + contextBefore: string; + contextAfter: string; +} + +export interface SemanticLinkPromptResponse { + shouldLink: boolean; + link: SemanticLinkSuggestion | null; +} + +export function buildSemanticLinkPrompt( + sourceContent: string, + targetContent: string, + sourceTitle: string, + targetTitle: string +): string { + return `${SEMANTIC_LINK_ANALYSIS_PROMPT} + +SOURCE POST: "${sourceTitle}" +TARGET POST: "${targetTitle}" + +SOURCE POST CONTENT: +${sourceContent} + +TARGET POST CONTENT: +${targetContent} + +Analyze the source post and identify ALL phrases that should link to the target post.`; +} diff --git a/src/lib/semantic-analyzer.ts b/src/lib/semantic-analyzer.ts deleted file mode 100644 index 213eae1..0000000 --- a/src/lib/semantic-analyzer.ts +++ /dev/null @@ -1,151 +0,0 @@ -import Anthropic from "@anthropic-ai/sdk"; - -interface AnalysisResult { - shouldLink: boolean; - linkText: string; - altText: string; - confidence: number; - reasoning: string; - linkPosition?: number; -} - -export class SemanticAnalyzer { - private anthropic: Anthropic; - - constructor() { - this.anthropic = new Anthropic({ - apiKey: process.env.ANTHROPIC_API_KEY || "", - }); - } - - async analyzeSemanticLink( - sourceContent: string, - targetContent: string, - sourceName: string, - targetName: string - ): Promise { - const prompt = `You are a conservative semantic link analyst. Most pages should NOT have links added unless there's a compelling reason. - -Source Post: "${sourceName}" -Target Post: "${targetName}" - -SOURCE CONTENT: - -${sourceContent} - - -TARGET CONTENT: - -${targetContent} - - -CRITICAL: Be highly selective. Only suggest a link if ALL these criteria are met: -1. STRONG SEMANTIC RELEVANCE - The source directly discusses concepts that the target page explains in detail -2. USER VALUE - A reader would genuinely benefit from accessing the target content at that specific point -3. NATURAL CONTEXT - The surrounding text naturally leads to needing the information in the target -4. SPECIFIC CONNECTION - Not just generic mentions of broad topics -5. CLEAR USER INTENT - The link helps users accomplish their specific goal on the source page -6. NO EXISTING LINK - The text you want to link MUST NOT already be part of a link to any URL -7. LINK TEXT LENGTH - The text to be linked MUST be 2-4 words only (e.g., "SBA loan", "buy a business", "loan application") - -IMPORTANT: The source content may contain HTML. Look for existing links in the format text. - -Default to "No suitable location for semantic link" unless there's exceptional relevance. - -Respond with ONLY a JSON object: -{ - "shouldLink": true/false, - "linkText": "exact 2-4 word text where link should be placed", - "altText": "best in class SEO alt text for the link", - "confidence": 0-100 (only suggest if 70%+ confident), - "reasoning": "Brief explanation", - "linkPosition": approximate character position in source content where link should go (optional) -}`; - - try { - const response = await this.anthropic.messages.create({ - model: "claude-3-5-sonnet-20241022", - max_tokens: 500, - temperature: 0.3, - messages: [ - { - role: "user", - content: prompt, - }, - ], - }); - - // Extract JSON from response - const content = response.content[0]; - if (content.type === "text") { - const jsonMatch = content.text.match(/\{[\s\S]*\}/); - if (jsonMatch) { - const result = JSON.parse(jsonMatch[0]); - - // Only return links with high confidence - if (result.shouldLink && result.confidence >= 70) { - return { - shouldLink: true, - linkText: result.linkText || "", - altText: result.altText || "", - confidence: result.confidence, - reasoning: result.reasoning || "", - linkPosition: result.linkPosition, - }; - } - } - } - - return { - shouldLink: false, - linkText: "", - altText: "", - confidence: 0, - reasoning: "No suitable location for semantic link", - }; - } catch (error) { - console.error("Error analyzing with Claude:", error); - throw new Error("Failed to analyze semantic link"); - } - } - - /** - * Find the exact position of link text in content - */ - findLinkPosition(content: string, linkText: string): number { - // First try exact match - let position = content.indexOf(linkText); - if (position !== -1) return position; - - // Try case-insensitive match - const lowerContent = content.toLowerCase(); - const lowerLinkText = linkText.toLowerCase(); - position = lowerContent.indexOf(lowerLinkText); - - return position; - } - - /** - * Check if text at position is already part of a link - */ - isAlreadyLinked(content: string, position: number, length: number): boolean { - // Look for tags before and after the position - const beforeText = content.substring(0, position); - const afterText = content.substring(position + length); - - // Check if we're inside an tag - const lastOpenTag = beforeText.lastIndexOf(""); - - if (lastOpenTag > lastCloseTag) { - // We have an unclosed tag before our position - // Check if it closes after our position - const nextCloseTag = afterText.indexOf(""); - if (nextCloseTag !== -1) { - return true; // We're inside a link - } - } - - return false; - } -} diff --git a/src/lib/semantic-link-analyzer.ts b/src/lib/semantic-link-analyzer.ts new file mode 100644 index 0000000..c3d25aa --- /dev/null +++ b/src/lib/semantic-link-analyzer.ts @@ -0,0 +1,130 @@ +import { query, type SDKMessage } from "@anthropic-ai/claude-code"; +import { + buildSemanticLinkPrompt, + type SemanticLinkPromptResponse, +} from "./prompts/semantic-link-prompt"; + +interface AnalysisResult { + shouldLink: boolean; + linkText: string; + altText: string; + confidence: number; + reasoning: string; + linkPosition?: number; + contextBefore?: string; + contextAfter?: string; + analysisOutput?: string; // Raw output from Claude +} + +interface AnalysisOptions { + analysisType: "incoming" | "outgoing" | "full"; + targetPostId?: number; +} + +export class SemanticLinkAnalyzer { + constructor() { + // No initialization needed for claude-code + } + + /** + * Analyze potential semantic links between posts + */ + async analyzeSemanticLink( + sourceContent: string, + targetContent: string, + sourceName: string, + targetName: string, + _options?: AnalysisOptions + ): Promise { + const results: AnalysisResult[] = []; + + // Analyze for a single best link from source to target + const result = await this.analyzePostPair(sourceContent, targetContent, sourceName, targetName); + + if (result) { + results.push(result); + } + + return results; + } + + /** + * Analyze entire post pair for the single best link + */ + private async analyzePostPair( + sourceContent: string, + targetContent: string, + sourceName: string, + targetName: string + ): Promise { + console.log(`Analyzing for best link from "${sourceName}" to "${targetName}"`); + + const prompt = buildSemanticLinkPrompt(sourceContent, targetContent, sourceName, targetName); + + try { + // Use claude-code query API + const messages: SDKMessage[] = []; + const controller = new AbortController(); + + for await (const message of query({ + prompt: prompt, + abortController: controller, + options: { + maxTurns: 20, + }, + })) { + messages.push(message); + } + + console.log({ messages }); + + // Extract JSON from the response + if (messages.length > 0) { + // Look for the result message which contains the actual response + const resultMessage = messages.find((msg) => msg.type === "result" && "result" in msg); + let responseText = ""; + + if (resultMessage && typeof resultMessage.result === "string") { + responseText = resultMessage.result; + } else { + // Fallback to checking other message types + const lastMessage = messages[messages.length - 1]; + if ("text" in lastMessage && typeof lastMessage.text === "string") { + responseText = lastMessage.text; + } else if ("content" in lastMessage && typeof lastMessage.content === "string") { + responseText = lastMessage.content; + } + } + + const jsonMatch = responseText.match(/\{[\s\S]*\}/); + if (jsonMatch) { + const result: SemanticLinkPromptResponse = JSON.parse(jsonMatch[0]); + + if (result.shouldLink && result.link) { + // Find the position of the link text in the source content + const linkPosition = sourceContent.indexOf(result.link.linkText); + + if (linkPosition !== -1) { + return { + shouldLink: true, + linkText: result.link.linkText, + altText: `Read more about ${targetName}`, + confidence: result.link.confidence, + reasoning: result.link.reasoning, + linkPosition: linkPosition, + contextBefore: result.link.contextBefore, + contextAfter: result.link.contextAfter, + analysisOutput: responseText, + }; + } + } + } + } + + return null; + } catch (error) { + console.error("Error analyzing with Claude:", error); + throw new Error("Failed to analyze semantic link"); + } + } +}