test: fix flaky tests

those were failing almost always locally on faster machines like mb m4 pro

Author: B4nan

test/core/crawlers/basic_crawler.test.ts

@@ -965,11 +965,11 @@ describe('BasicCrawler', () => {
         requestQueue.fetchNextRequest = async () => queue.pop()!;
         requestQueue.isEmpty = async () => Promise.resolve(!queue.length);
 
-        setTimeout(() => queue.push(request0), 10);
-        setTimeout(() => queue.push(request1), 100);
+        setTimeout(() => queue.push(request0), 50);
+        setTimeout(() => queue.push(request1), 150);
         setTimeout(() => {
             isFinished = true;
-        }, 150);
+        }, 500);
 
         await basicCrawler.run();
 
@@ -1020,7 +1020,7 @@ describe('BasicCrawler', () => {
         setTimeout(() => queue.push(request1), 100);
         setTimeout(() => {
             void basicCrawler.teardown();
-        }, 300);
+        }, 500);
 
         await basicCrawler.run();
 

test/core/sitemap_request_list.test.ts

@@ -371,7 +371,7 @@ describe('SitemapRequestList', () => {
             signal: controller.signal,
         });
 
-        await sleep(50); // Loads the first sub-sitemap, but not the second
+        await sleep(100); // Loads the first sub-sitemap, but not the second
         controller.abort();
 
         for await (const request of list) {

test/utils/fixtures/child.js

@@ -1,4 +1,4 @@
 // dummy script for testing ps-tree.ts
 setTimeout(() => {
     /* Does nothing, but prevents exit */
-}, 1000);
+}, 5000);

test/utils/psTree.test.ts

@@ -8,19 +8,42 @@ const scripts = {
     child: path.join(__dirname, 'fixtures', 'child.js'),
 };
 
+// Helper to poll for child processes
+async function waitForChildren(pid: number, maxWaitMs = 3000): Promise<any[]> {
+    const startTime = Date.now();
+    while (Date.now() - startTime < maxWaitMs) {
+        const children = await psTree(pid);
+        if (children.length > 0) {
+            return children;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    return psTree(pid); // Final attempt
+}
+
+// Helper to poll for processes to terminate
+async function waitForTermination(pid: number, maxWaitMs = 10000): Promise<void> {
+    const startTime = Date.now();
+    while (Date.now() - startTime < maxWaitMs) {
+        const children = await psTree(pid);
+        if (children.length === 0) {
+            return;
+        }
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+}
+
 describe('psTree()', () => {
     test('Spawn a Parent process which has Ten Child Processes', async () => {
         const parent = exec(`node ${scripts.parent}`);
 
-        // Wait for the child process(es) to spawn.
-        await new Promise((resolve) => setTimeout(resolve, 500));
-
-        const children = await psTree(parent.pid!);
+        // Poll until child processes are detected
+        const children = await waitForChildren(parent.pid!);
 
         expect(children.length).toBeGreaterThan(0);
 
-        // Allow time for the processes to be terminated.
-        await new Promise((resolve) => setTimeout(resolve, 2000));
+        // Poll until processes terminate
+        await waitForTermination(parent.pid!);
 
         const postKillChildren = await psTree(parent.pid!);
 
@@ -30,17 +53,27 @@ describe('psTree()', () => {
     test('Includes itself if includeRoot is true', async () => {
         const parent = exec(`node ${scripts.parent}`);
 
-        // Wait for the child process(es) to spawn.
-        await new Promise((resolve) => setTimeout(resolve, 500));
-
-        const processes = await psTree(parent.pid!, true);
+        // Poll until processes are detected (parent + children, so > 1)
+        let processes: any[] = [];
+        const startTime = Date.now();
+        while (Date.now() - startTime < 3000) {
+            processes = await psTree(parent.pid!, true);
+            // Wait for parent + at least one child (length > 1)
+            if (processes.length > 1) {
+                break;
+            }
+            await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+        if (processes.length <= 1) {
+            processes = await psTree(parent.pid!, true); // Final attempt
+        }
 
         const parentProcess = processes.find((process) => Number.parseInt(process.PID, 10) === parent.pid!);
 
         expect(parentProcess).toBeDefined();
 
-        // Allow time for the processes to be terminated.
-        await new Promise((resolve) => setTimeout(resolve, 2000));
+        // Poll until processes terminate
+        await waitForTermination(parent.pid!);
 
         const postKillChildren = await psTree(parent.pid!);