feat: Update testing (#538)

Updates test to consolidate all functionality, and only test changed projects.

Author: willum070

e2e/samples.spec.ts

@@ -17,20 +17,78 @@
 import { test, expect } from '@playwright/test';
 import fs from 'fs';
 import path from 'path';
-import childProcess from 'child_process';
+import childProcess, { execSync } from 'child_process';
 
 const samplesDir = path.join(__dirname, '..', 'samples');
 
-const sampleFolders = fs.readdirSync(samplesDir).filter((file) => {
-  return fs.statSync(path.join(samplesDir, file)).isDirectory();
-});
+// Function to return all sample folders.
+const getAllSampleFolders = () => {
+  return fs.readdirSync(samplesDir).filter((file) => {
+    // Ensure we are only looking at directories within samplesDir, excluding find-changes.sh itself if it's a file
+    const filePath = path.join(samplesDir, file);
+    return fs.statSync(filePath).isDirectory();
+  });
+};
+
+// Function to return only changed sample folders.
+const getChangedSampleFolders = (): string[] => {
+  try {
+    const scriptPath = path.join(__dirname, '..', 'samples', 'find-changes.sh'); 
+    
+    if (!fs.existsSync(scriptPath)) {
+      console.warn(`Warning: find-changes.sh not found at ${scriptPath}. Running tests for all samples.`);
+      return getAllSampleFolders();
+    }
+
+    // Execute the script from the project root.
+    const projectRoot = path.join(__dirname, '..'); 
+    const output = execSync(`sh ${scriptPath}`, { cwd: projectRoot, encoding: 'utf-8' });
+    
+    // Get all folder names outputted by the script
+    const rawChangedFolders = output.trim().split('\n');
+    // Filter out empty strings that might result from multiple newlines or a trailing newline
+    const changedFolders = rawChangedFolders.filter(folder => folder.trim().length > 0);
+
+    if (changedFolders.length === 0) {
+      // This means find-changes.sh ran successfully and reported no changes.
+      console.log("find-changes.sh reported no changed folders. Skipping tests.");
+      return []; 
+    }
+
+    // Validate that changed folders actually exist in samplesDir
+    const validChangedFolders = changedFolders.filter(folderName => {
+      const folderPath = path.join(samplesDir, folderName);
+      return fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory();
+    });
+
+    if (validChangedFolders.length === 0 && changedFolders.length > 0 && changedFolders[0] !== "") {
+      console.warn("find-changes.sh outputted folder names, but none are valid sample directories. Running for all samples.");
+      return [];
+    }
+
+    console.log("Running tests only for changed samples: ", validChangedFolders);
+    return validChangedFolders;
+
+  } catch (error) {
+    console.error("Error running find-changes.sh, falling back to all samples:", error);
+    return getAllSampleFolders();
+  }
+};
+
+const foldersToTest = getChangedSampleFolders();
+
+if (foldersToTest.length === 0) {
+  console.log("No sample folders identified to test. This might indicate no changes or an issue with find-changes.sh.");
+} else {
+  console.log(`Will run tests for the following folders: ${foldersToTest.join(', ')}`);
+}
 
 // Iterate through samples and run the same test for each one.
-sampleFolders.forEach((sampleFolder) => {
+foldersToTest.forEach((sampleFolder) => {
   test(`test ${sampleFolder}`, async ({ page }) => {
 
     // START Build the sample
-    const buildProcess = childProcess.spawn('npm', ['run', 'test'], {
+    const buildProcess = childProcess.spawn('npm', ['run', 'build'], {
       cwd: path.join(samplesDir, sampleFolder),
       stdio: 'inherit',
     });
@@ -40,24 +98,28 @@ sampleFolders.forEach((sampleFolder) => {
           if (code === 0) {
             resolve(true);
           } else {
-            reject(`Build process exited with code ${code}`);
+            reject(new Error(`Build process for ${sampleFolder} exited with code ${code}`));
           }
         });
+        buildProcess.on('error', (err) => {
+            reject(new Error(`Failed to start build process for ${sampleFolder}: ${err.message}`));
+        });
       });
     // END Build the sample
 
     // START run the preview
     // Get an available port
     const port = 8080;
-
     const url = `http://localhost:${port}/`;
 
     const viteProcess = childProcess.spawn('vite', ['preview', `--port=${port}`], {
       cwd: path.join(samplesDir, sampleFolder),
-      stdio: 'inherit', 
+      stdio: 'inherit',
+      detached: true, // Allows parent to exit independently, though we kill it in finally
     });
 
-    await new Promise((resolve) => setTimeout(resolve, 500)); // Set a timeout to let the web server start.
+    //await new Promise((resolve) => setTimeout(resolve, 500)); // Set a timeout to let the web server start.
+    await page.waitForTimeout(500);
     // END run the preview
 
     /**
@@ -66,32 +128,24 @@ sampleFolders.forEach((sampleFolder) => {
      * Run `npx playwright test --ui` to launch Playwright in UI mode to iteratively debug this file.
      */
     try {
-      // Check for console errors. Define a promise, then call after page load.
       const consoleErrors: string[] = [];
-      const errorPromise = new Promise<void>((resolve) => {
-        page.on('console', (msg => {
-          if (msg.type() === 'error') {
-            consoleErrors.push(msg.text());
-            resolve();
-          }
-        }));
-
-        page.on('pageerror', (exception) => {
-          consoleErrors.push(exception.message);
-          resolve();
-        });
-
-        // Set a timeout to resolve the promise even if no error occurs.
-        setTimeout(() => {
-          resolve();
-        }, 500);
+      // Capture console errors and page errors
+      page.on('console', (msg) => {
+        if (msg.type() === 'error') {
+          consoleErrors.push(msg.text());
+        }
+      });
+      page.on('pageerror', (exception) => {
+        consoleErrors.push(exception.message);
       });
 
       // Navigate to the page.
+      //await page.goto(url, { waitUntil: 'networkidle', timeout: 500 });
       await page.goto(url);
 
-      // There must be no critical console errors.
-      await errorPromise;
+      // Allow some time for async operations and errors to be caught
+      await page.waitForTimeout(500);
+
       // Filter out error messages we can safely avoid.
       const filteredErrorMessages = [
         'Falling back to Raster',
@@ -100,23 +154,26 @@ sampleFolders.forEach((sampleFolder) => {
       const criticalErrors = consoleErrors.filter(error =>
         !filteredErrorMessages.some(message => error.includes(message))
       );
+
+      if (criticalErrors.length > 0) {
+        console.error(`Critical console errors found in ${sampleFolder}:`, criticalErrors);
+      }
       expect(criticalErrors).toHaveLength(0);
 
       // Wait for the page DOM to load; this does NOT include the Google Maps APIs.
-      await page.waitForLoadState('domcontentloaded');
+      await page.waitForLoadState('domcontentloaded', { timeout: 500 });
 
       // Wait for Google Maps to load.
-      await page.waitForFunction(() => window.google && window.google.maps);
+      await page.waitForFunction(() => window.google && window.google.maps, { timeout: 500 });
 
       // Insert a delay in ms to let the map load.
-      await new Promise((resolve) => setTimeout(resolve, 1000));
+      await page.waitForTimeout(500);
 
       // Assertions. These must be met or the test will fail.
       // The sample must load the Google Maps API.
       const hasGoogleMaps = await page.evaluate(() => {
         return typeof window.google !== 'undefined' && typeof window.google.maps !== 'undefined';
       });
-
       await expect(hasGoogleMaps).toBeTruthy();
 
       /**const mapElement = await page.locator('#map');
@@ -127,7 +184,17 @@ sampleFolders.forEach((sampleFolder) => {
         throw new Error('Assertion failed: Map is not visible.');
       }*/
     } finally {
-      viteProcess.kill();
+      //viteProcess.kill(); // We used to just kill the process. Curious to see about how the other stuff works.
+      if (viteProcess.pid) {
+        try {
+          // Use process.kill for cross-platform compatibility, sending SIGINT
+          process.kill(viteProcess.pid, 'SIGINT');
+        } catch (e) {
+          console.warn(`Failed to kill Vite process for ${sampleFolder} (PID: ${viteProcess.pid}):`, e.message);
+        }
+      }
+      // Add a small delay to allow the process to terminate
+      await page.waitForTimeout(500);
     }
   });
 });