Migrate scraper tests to real PostgreSQL database with Drizzle ORM

Co-authored-by: git <[email protected]>

Author: cursoragent

sitio/src/routes/api/internal/scraper/scrap/server.test.ts

@@ -1,28 +1,22 @@
-import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from 'vitest';
 import { POST } from './+server.js';
 import { GET as lastIdsGET } from '../last-ids/+server.js';
 import type { RequestEvent } from '@sveltejs/kit';
 import { nanoid } from 'nanoid';
+import { drizzle } from 'drizzle-orm/postgres-js';
+import postgres from 'postgres';
+import * as schema from '../../../../../schema.js';
+import { eq, sql } from 'drizzle-orm';
 
-// Mock the database module
-vi.mock('$lib/db/index.js', () => ({
-    db: {
-        query: {
-            scraperTokens: {
-                findFirst: vi.fn()
-            },
-            tweets: {
-                findMany: vi.fn()
-            }
-        },
-        transaction: vi.fn(),
-        insert: vi.fn(),
-        update: vi.fn(),
-    }
-}));
+// Set up test database URL
+const TEST_DATABASE_URL = 'postgresql://testuser:testpass@localhost:5432/milei_test';
 
-// Import the mocked db
-import { db } from '$lib/db/index.js';
+// Override the DATABASE_URL environment variable for testing
+process.env.DATABASE_URL = TEST_DATABASE_URL;
+
+// Create database connection for testing
+const client = postgres(TEST_DATABASE_URL, { max: 1 });
+const testDb = drizzle(client, { schema });
 
 // Mock request/response for testing
 function mockRequest(body: any, headers: Record<string, string> = {}): RequestEvent {
@@ -85,9 +79,80 @@ const validScrapData = {
 
 const validToken = 'test-token-123';
 
-describe('Scraper API Endpoints', () => {
-    beforeEach(() => {
-        vi.clearAllMocks();
+describe('Scraper API Real Database Tests', () => {
+    beforeAll(async () => {
+        // Create the database schema
+        await testDb.execute(sql`
+            CREATE TABLE IF NOT EXISTS db_scraper_tokens (
+                id SERIAL PRIMARY KEY,
+                token TEXT NOT NULL
+            );
+        `);
+
+        await testDb.execute(sql`
+            CREATE TABLE IF NOT EXISTS db_scraps (
+                id SERIAL PRIMARY KEY,
+                uid TEXT UNIQUE,
+                at TIMESTAMP WITH TIME ZONE NOT NULL,
+                cuenta_id TEXT,
+                total_tweets_seen INTEGER
+            );
+        `);
+
+        await testDb.execute(sql`
+            CREATE TABLE IF NOT EXISTS db_liked_tweets (
+                url TEXT PRIMARY KEY,
+                first_seen_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                last_seen_at TIMESTAMP WITH TIME ZONE,
+                text TEXT,
+                scrap_id INTEGER
+            );
+        `);
+
+        await testDb.execute(sql`
+            CREATE TABLE IF NOT EXISTS db_retweets (
+                poster_id TEXT NOT NULL,
+                poster_handle TEXT,
+                post_id TEXT NOT NULL,
+                first_seen_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                retweet_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                posted_at TIMESTAMP WITH TIME ZONE NOT NULL,
+                text TEXT,
+                scrap_id INTEGER,
+                PRIMARY KEY (poster_id, post_id)
+            );
+        `);
+
+        await testDb.execute(sql`
+            CREATE TABLE IF NOT EXISTS db_tweets (
+                id TEXT PRIMARY KEY,
+                twitter_scraper_json JSONB NOT NULL,
+                captured_at TIMESTAMP WITH TIME ZONE NOT NULL
+            );
+        `);
+
+        // Insert test token
+        await testDb.insert(schema.scraperTokens).values({
+            token: validToken
+        });
+    });
+
+    beforeEach(async () => {
+        // Clean up test data before each test
+        await testDb.delete(schema.tweets);
+        await testDb.delete(schema.retweets);
+        await testDb.delete(schema.likedTweets);
+        await testDb.delete(schema.scraps);
+    });
+
+    afterAll(async () => {
+        // Clean up after all tests
+        await testDb.execute(sql`DROP TABLE IF EXISTS db_tweets CASCADE;`);
+        await testDb.execute(sql`DROP TABLE IF EXISTS db_retweets CASCADE;`);
+        await testDb.execute(sql`DROP TABLE IF EXISTS db_liked_tweets CASCADE;`);
+        await testDb.execute(sql`DROP TABLE IF EXISTS db_scraps CASCADE;`);
+        await testDb.execute(sql`DROP TABLE IF EXISTS db_scraper_tokens CASCADE;`);
+        await client.end();
     });
 
     describe('POST /api/internal/scraper/scrap', () => {
@@ -104,9 +169,6 @@ describe('Scraper API Endpoints', () => {
         });
 
         it('should reject requests with invalid token', async () => {
-            // Mock database to return no token
-            vi.mocked(db.query.scraperTokens.findFirst).mockResolvedValue(undefined);
-
             const request = mockRequest(validScrapData, {
                 Authorization: 'Bearer invalid-token',
             });
@@ -121,12 +183,6 @@ describe('Scraper API Endpoints', () => {
         });
 
         it('should reject requests with invalid scrap data', async () => {
-            // Mock database to return a valid token
-            vi.mocked(db.query.scraperTokens.findFirst).mockResolvedValue({
-                id: 1,
-                token: validToken
-            });
-
             const request = mockRequest(
                 { invalidField: 'invalid' },
                 { Authorization: `Bearer ${validToken}` }
@@ -140,77 +196,87 @@ describe('Scraper API Endpoints', () => {
             }
         });
 
-        it('should accept valid scrap data with valid token', async () => {
-            // Mock database interactions
-            vi.mocked(db.query.scraperTokens.findFirst).mockResolvedValue({
-                id: 1,
-                token: validToken
-            });
-
-            // Mock transaction
-            const mockTransaction = vi.fn(async (callback) => {
-                const mockTx = {
-                    insert: vi.fn().mockReturnValue({
-                        values: vi.fn().mockReturnValue({
-                            returning: vi.fn().mockReturnValue({
-                                onConflictDoNothing: vi.fn().mockResolvedValue([{ id: 123 }])
-                            }),
-                            onConflictDoUpdate: vi.fn().mockResolvedValue([{ id: 123 }])
-                        })
-                    }),
-                    update: vi.fn().mockReturnValue({
-                        set: vi.fn().mockReturnValue({
-                            where: vi.fn().mockResolvedValue({})
-                        })
-                    }),
-                    query: {
-                        scraps: {
-                            findFirst: vi.fn()
-                        }
-                    }
-                };
-                return await callback(mockTx);
-            });
-
-            vi.mocked(db.transaction).mockImplementation(mockTransaction);
-
+        it('should successfully process valid scrap data with PostgreSQL', async () => {
             const request = mockRequest(validScrapData, {
                 Authorization: `Bearer ${validToken}`,
             });
 
             const response = await POST(request);
-
             expect(response.status).toBe(200);
+
             const data = await response.json();
             expect(data).toHaveProperty('scrapId');
             expect(typeof data.scrapId).toBe('number');
+
+            // Verify data was inserted into PostgreSQL database
+            const scraps = await testDb.select().from(schema.scraps);
+            expect(scraps.length).toBe(1);
+            expect(scraps[0].uid).toBe(validScrapData.uid);
+
+            const likedTweets = await testDb.select().from(schema.likedTweets);
+            expect(likedTweets.length).toBe(1);
+            expect(likedTweets[0].url).toBe(validScrapData.likedTweets[0].url);
+
+            const retweets = await testDb.select().from(schema.retweets);
+            expect(retweets.length).toBe(1);
+            expect(retweets[0].posterId).toBe(validScrapData.retweets[0].posterId);
+
+            const tweets = await testDb.select().from(schema.tweets);
+            expect(tweets.length).toBe(1);
+            expect(tweets[0].id).toBe(validScrapData.tweets[0].id);
+        });
+
+        it('should handle PostgreSQL conflict resolution correctly', async () => {
+            // First request
+            const request1 = mockRequest(validScrapData, {
+                Authorization: `Bearer ${validToken}`,
+            });
+
+            const response1 = await POST(request1);
+            expect(response1.status).toBe(200);
+
+            // Second request with same UID should use PostgreSQL onConflictDoNothing
+            const request2 = mockRequest(validScrapData, {
+                Authorization: `Bearer ${validToken}`,
+            });
+
+            const response2 = await POST(request2);
+            expect(response2.status).toBe(200);
+
+            // Should still only have one scrap record due to PostgreSQL unique constraint
+            const scraps = await testDb.select().from(schema.scraps);
+            expect(scraps.length).toBe(1);
         });
     });
 
     describe('GET /api/internal/scraper/last-ids', () => {
-        it('should return last tweet IDs', async () => {
-            // Mock database to return some tweet IDs
-            vi.mocked(db.query.tweets.findMany).mockResolvedValue([
-                { id: 'tweet123', twitterScraperJson: {}, capturedAt: new Date() },
-                { id: 'tweet456', twitterScraperJson: {}, capturedAt: new Date() },
-                { id: 'tweet789', twitterScraperJson: {}, capturedAt: new Date() }
+        it('should return last tweet IDs from PostgreSQL', async () => {
+            // Insert test tweets into PostgreSQL
+            await testDb.insert(schema.tweets).values([
+                {
+                    id: 'tweet1',
+                    twitterScraperJson: { test: 'data1' },
+                    capturedAt: new Date('2023-01-01')
+                },
+                {
+                    id: 'tweet2',
+                    twitterScraperJson: { test: 'data2' },
+                    capturedAt: new Date('2023-01-02')
+                }
             ]);
 
             const response = await lastIdsGET();
-
             expect(response.status).toBe(200);
 
             const data = await response.json();
             expect(Array.isArray(data)).toBe(true);
-            expect(data).toEqual(['tweet123', 'tweet456', 'tweet789']);
+            expect(data.length).toBe(2);
+            expect(data).toContain('tweet1');
+            expect(data).toContain('tweet2');
         });
 
-        it('should handle empty tweet results', async () => {
-            // Mock database to return empty results
-            vi.mocked(db.query.tweets.findMany).mockResolvedValue([]);
-
+        it('should handle empty PostgreSQL results', async () => {
             const response = await lastIdsGET();
-
             expect(response.status).toBe(200);
 
             const data = await response.json();