feat: Add Playwright test for Run Suite

page-objects/service-spec-config-page.ts

@@ -30,6 +30,13 @@ export class ServiceSpecConfigPage extends BasePage {
   private readonly alertDismissButton: Locator;
   private readonly bccErrorToggle: Locator;
   private readonly bccErrorContent: Locator;
+  private readonly runSuitebtn: Locator;
+  readonly executionProgressDropdown: Locator;
+  private readonly saveSpecBtn: Locator;
+  readonly executionLog: Locator;
+  readonly alertContainer: Locator;
+  readonly alertTitle: Locator;
+  readonly alertDescription: Locator;
 
   constructor(page: Page, testInfo: TestInfo, eyes: any, specName: string) {
     super(page, testInfo, eyes, specName);
@@ -59,6 +66,18 @@ export class ServiceSpecConfigPage extends BasePage {
     this.alertDismissButton = this.alertMessage.locator("button");
     this.bccErrorToggle = this.specSection.locator(".bcc-errors-btn");
     this.bccErrorContent = this.specSection.locator(".bcc-errors-content");
+    this.runSuitebtn = this.specSection.locator(".executeBtn");
+    this.executionProgressDropdown = this.specSection.locator(
+      ".execution-progress-panel",
+    );
+    this.saveSpecBtn = this.specSection.locator("button.savebtn.save");
+
+    this.executionLog = this.executionProgressDropdown.locator(
+      ".execution-progress-log",
+    );
+    this.alertContainer = page.locator(".alert-msg.error");
+    this.alertTitle = this.alertContainer.locator("p");
+    this.alertDescription = this.alertContainer.locator("pre");
   }
   async openSpecTab() {
     return this.openApiTabPage.openSpecTab();
@@ -123,6 +142,40 @@ export class ServiceSpecConfigPage extends BasePage {
     });
   }
 
+  async deleteSpecLinesInEditor(searchText: string, lineCount: number = 1) {
+    await test.step(`Delete ${lineCount} spec line(s) starting with '${searchText}'`, async () => {
+      const lines = this.specSection.locator(".cm-content .cm-line");
+      await expect(lines.first()).toBeVisible({ timeout: 10000 });
+
+      const editorContent = this.specSection.locator(".cm-content");
+      await editorContent.click();
+      await this.page.keyboard.press("Control+End");
+      await this.page.waitForTimeout(300);
+      await this.page.keyboard.press("Control+Home");
+      await this.page.waitForTimeout(200);
+
+      const targetLine = lines.filter({ hasText: searchText }).first();
+      await expect(targetLine).toBeVisible({ timeout: 10000 });
+      await targetLine.scrollIntoViewIfNeeded();
+      await targetLine.click();
+
+      await this.page.keyboard.press("Home");
+      for (let i = 1; i < lineCount; i++) {
+        await this.page.keyboard.press("Shift+ArrowDown");
+      }
+      await this.page.keyboard.press("Shift+End");
+
+      await this.page.keyboard.press("Shift+Delete");
+
+      const safeFileName = searchText.replace(/[^a-zA-Z0-9]/g, "-");
+      await takeAndAttachScreenshot(
+        this.page,
+        `delete-spec-block-${safeFileName}`,
+        this.eyes,
+      );
+    });
+  }
+
   private async verifyTextHighlightedInEditor(text: string) {
     await test.step(`Visual evidence: highlight '${text}' in editor`, async () => {
       const editorContent = this.specSection.locator(".cm-content");
@@ -323,4 +376,62 @@ export class ServiceSpecConfigPage extends BasePage {
       await this.openSpecTab();
     });
   }
+
+  async clickRunSuite() {
+    console.log("Clicked Run suite");
+    await this.runSuitebtn.click();
+    await takeAndAttachScreenshot(this.page, "clicked-run-suite");
+  }
+
+  async waitForExecutionToComplete(
+    pollIntervalMs: number = 3000,
+    timeout: number = 60000,
+  ) {
+    await test.step(`Wait for execution to complete (poll every ${pollIntervalMs}ms, timeout: ${timeout}ms)`, async () => {
+      const dropdown = this.executionProgressDropdown;
+      await expect(dropdown).toHaveAttribute("data-state", "running", {
+        timeout: 15000,
+      });
+      console.log("\tExecution is running — polling until it completes...");
+
+      await expect
+        .poll(
+          async () => {
+            const state = await dropdown.getAttribute("data-state");
+            console.log(`\t[poll] data-state = '${state}'`);
+            return state;
+          },
+          {
+            intervals: [pollIntervalMs],
+            timeout,
+            message: `Execution did not leave 'running' state within ${timeout}ms`,
+          },
+        )
+        .not.toBe("running");
+
+      const finalState = await dropdown.getAttribute("data-state");
+      console.log(`\tExecution completed with state: '${finalState}'`);
+      await takeAndAttachScreenshot(
+        this.page,
+        `execution-completed-state-${finalState}`,
+      );
+    });
+  }
+
+  async expandExecutionProgressDropdown() {
+    await this.executionProgressDropdown.click();
+    await takeAndAttachScreenshot(
+      this.page,
+      "expanded-execution-progress-dropdown",
+    );
+  }
+
+  async clickSaveAfterEdit() {
+    console.log("Saved new spec");
+    await test.step("Click Save button after editing spec", async () => {
+      await this.saveSpecBtn.waitFor({ state: "visible", timeout: 5000 });
+      await this.saveSpecBtn.click();
+      await takeAndAttachScreenshot(this.page, "save-button-clicked");
+    });
+  }
 }

specs/config/service-spec-config/edit-config.spec.ts

@@ -2,6 +2,7 @@ import { test, expect } from "../../../utils/eyesFixture";
 import { takeAndAttachScreenshot } from "../../../utils/screenshotUtils";
 import { SPECMATIC_CONFIG } from "../../specNames";
 import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page";
+import { Page } from "playwright/test";
 const CONFIG_NAME = SPECMATIC_CONFIG;
 
 test.describe("Specmatic Config", () => {
@@ -21,6 +22,82 @@ test.describe("Specmatic Config", () => {
         await configPage.sideBar.selectSpec(CONFIG_NAME);
         await configPage.openSpecTab();
       });
+
+      await test.step("Click Run Suite and Assert the Status", async () => {
+        await configPage.clickRunSuite();
+        await assertErrorDialog(configPage, "Failed to execute");
+        await configPage.dismissAlert();
+        await configPage.closeRightSidebarByClickingOutside();
+        await assertExecutionDropDown(configPage, page, "error", "Failed");
+        await expect(configPage.executionLog).toContainText(
+          "Invalid mock configuration for proxy_generated.yaml",
+        );
+        await takeAndAttachScreenshot(
+          page,
+          "asserted-execution-progress-dropdown",
+          eyes,
+        );
+      });
+
+      await test.step("Edit Spec, Save and Verify Progress", async () => {
+        await configPage.deleteSpecLinesInEditor("- port: 9090", 3);
+        await configPage.clickSaveAfterEdit();
+        await configPage.clickRunSuite();
+        await configPage.closeRightSidebarByClickingOutside();
+        await assertExecutionDropDown(configPage, page, "running", "Running");
+        await takeAndAttachScreenshot(page, "suite-status-running", eyes);
+        await configPage.waitForExecutionToComplete();
+        await assertErrorDialog(
+          configPage,
+          "Mock Server Stopped",
+          "StandaloneCoroutine was cancelled",
+        );
+        await configPage.dismissAlert();
+        await assertExecutionDropDown(configPage, page, "error", "Failed");
+        await expect(configPage.executionLog).toContainText(
+          "Starting mock: kafka.yaml (port=0)",
+        );
+        await expect(configPage.executionLog).toContainText(
+          "Failed to start mock server for kafka.yaml",
+        );
+        await takeAndAttachScreenshot(
+          page,
+          "execution-progress-assertion-after-spec-change",
+          eyes,
+        );
+      });
     },
   );
 });
+
+async function assertExecutionDropDown(
+  configPage: ServiceSpecConfigPage,
+  page: Page,
+  state: "error" | "success" | "running",
+  expectedStatus: string,
+) {
+  const dropdown = configPage.executionProgressDropdown;
+
+  await expect(dropdown).toBeVisible({ timeout: 10000 });
+  await expect(dropdown).toHaveAttribute("data-state", state, {
+    timeout: 5000,
+  });
+  await expect(dropdown).toHaveAttribute("open", "");
+
+  const statusText = dropdown.locator(".execution-progress-status");
+  await expect(statusText).toHaveText(expectedStatus, { timeout: 5000 });
+
+  await takeAndAttachScreenshot(page, `execution-progress-asserted-${state}`);
+}
+
+async function assertErrorDialog(
+  configPage: ServiceSpecConfigPage,
+  expectedTitle: string,
+  expectedDesc?: string,
+) {
+  await expect(configPage.alertContainer).toBeVisible();
+  await expect(configPage.alertTitle).toHaveText(expectedTitle);
+  if (expectedDesc) {
+    await expect(configPage.alertDescription).toContainText(expectedDesc);
+  }
+}

utils/global-setup.ts

@@ -3,6 +3,8 @@ import {
   EyesRunner,
   VisualGridRunner,
 } from "@applitools/eyes-playwright";
+import { readdir, rm } from "fs/promises";
+import path from "path";
 
 export const ENABLE_VISUAL = process.env.ENABLE_VISUAL === "true";
 
@@ -79,5 +81,35 @@ if (process.env.GROUP_NAME) {
   Batch.addProperty("group_name", process.env.GROUP_NAME);
 }
 export default async function globalSetup() {
-  // No-op: Applitools batch/runner setup is handled in worker context for each test process.
+  const specsDirectory = path.resolve(
+    process.cwd(),
+    "specmatic-studio-demo",
+    "specs",
+  );
+
+  let entries;
+  try {
+    entries = await readdir(specsDirectory, { withFileTypes: true });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      return;
+    }
+    throw error;
+  }
+
+  await Promise.all(
+    entries
+      .filter(
+        (entry) =>
+          entry.isDirectory() &&
+          (entry.name.startsWith("product_search_bff_v5_") ||
+            entry.name === "proxy_recordings_examples"),
+      )
+      .map((entry) =>
+        rm(path.resolve(specsDirectory, entry.name), {
+          recursive: true,
+          force: true,
+        }),
+      ),
+  );
 }