diff --git a/tests/e2e/fixtures/caldera-auth.ts b/tests/e2e/fixtures/caldera-auth.ts new file mode 100644 index 0000000..366c899 --- /dev/null +++ b/tests/e2e/fixtures/caldera-auth.ts @@ -0,0 +1,60 @@ +import { test as base, expect, type Page } from "@playwright/test"; + +const CALDERA_USER = process.env.CALDERA_USER || "admin"; +const CALDERA_PASS = process.env.CALDERA_PASS || "admin"; + +/** + * Authenticate against the Caldera login page. + * Handles both the Vue/magma login form and basic-auth style login. + */ +async function authenticateCaldera(page: Page, baseURL: string) { + await page.goto(baseURL); + + // If already on the main page (no login required), return early + const url = page.url(); + if (!url.includes("/login") && !url.includes("/enter")) { + const appShell = page.locator("#app, .main-content, nav.navbar"); + try { + await appShell.first().waitFor({ timeout: 5_000 }); + return; + } catch { + // Fall through to login + } + } + + // Wait for any login form to appear + const usernameField = page.locator( + 'input[name="username"], input[type="text"]#username, input[placeholder*="user" i]' + ); + const passwordField = page.locator( + 'input[name="password"], input[type="password"]' + ); + + await usernameField.first().waitFor({ timeout: 10_000 }); + await usernameField.first().fill(CALDERA_USER); + await passwordField.first().fill(CALDERA_PASS); + + // Submit + const submitBtn = page.locator( + 'button[type="submit"], input[type="submit"], button:has-text("Login"), button:has-text("Sign in")' + ); + await submitBtn.first().click(); + + // Wait for navigation away from login + await page.waitForURL((url) => !url.pathname.includes("/login"), { + timeout: 15_000, + }); +} + +type CalderaFixtures = { + authenticatedPage: Page; +}; + +export const test = base.extend({ + authenticatedPage: async ({ page, baseURL }, use) => { + await authenticateCaldera(page, baseURL!); + await use(page); + }, +}); + +export { expect }; diff --git a/tests/e2e/package.json b/tests/e2e/package.json new file mode 100644 index 0000000..6dbcde8 --- /dev/null +++ b/tests/e2e/package.json @@ -0,0 +1,17 @@ +{ + "name": "emu-e2e-tests", + "version": "1.0.0", + "description": "Playwright E2E tests for the MITRE Caldera Emu plugin", + "private": true, + "scripts": { + "test": "npx playwright test", + "test:chromium": "npx playwright test --project=chromium", + "test:firefox": "npx playwright test --project=firefox", + "test:headed": "npx playwright test --headed", + "test:debug": "npx playwright test --debug", + "install:browsers": "npx playwright install --with-deps chromium firefox" + }, + "devDependencies": { + "@playwright/test": "^1.52.0" + } +} diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts new file mode 100644 index 0000000..1b733a9 --- /dev/null +++ b/tests/e2e/playwright.config.ts @@ -0,0 +1,33 @@ +import { defineConfig, devices } from "@playwright/test"; + +const CALDERA_URL = process.env.CALDERA_URL || "http://localhost:8888"; + +export default defineConfig({ + testDir: "./specs", + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: process.env.CI ? "github" : "html", + timeout: 60_000, + expect: { + timeout: 15_000, + }, + use: { + baseURL: CALDERA_URL, + trace: "on-first-retry", + screenshot: "only-on-failure", + video: "retain-on-failure", + ignoreHTTPSErrors: true, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "firefox", + use: { ...devices["Desktop Firefox"] }, + }, + ], +}); diff --git a/tests/e2e/specs/emu-emulation-plans.spec.ts b/tests/e2e/specs/emu-emulation-plans.spec.ts new file mode 100644 index 0000000..f55638d --- /dev/null +++ b/tests/e2e/specs/emu-emulation-plans.spec.ts @@ -0,0 +1,100 @@ +import { test, expect } from "../fixtures/caldera-auth"; + +test.describe("Emu plugin — adversary emulation plan listing", () => { + test.beforeEach(async ({ authenticatedPage: page }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should display non-zero abilities count when emu plugin is loaded", async ({ + authenticatedPage: page, + }) => { + const abilitiesCount = page.locator("h1.is-size-1").first(); + + await expect(async () => { + const text = await abilitiesCount.textContent(); + expect(Number(text?.trim())).toBeGreaterThan(0); + }).toPass({ timeout: 20_000 }); + }); + + test("should display non-zero adversaries count when emu plugin is loaded", async ({ + authenticatedPage: page, + }) => { + const adversariesCount = page.locator("h1.is-size-1").nth(1); + + await expect(async () => { + const text = await adversariesCount.textContent(); + expect(Number(text?.trim())).toBeGreaterThan(0); + }).toPass({ timeout: 20_000 }); + }); + + test("should navigate to abilities page filtered by emu plugin when clicking View Abilities", async ({ + authenticatedPage: page, + }) => { + const viewAbilitiesBtn = page.locator( + 'a:has-text("Abilities"), a:has-text("View Abilities")' + ); + await viewAbilitiesBtn.first().click(); + + // Should navigate to abilities page with emu filter + await page.waitForLoadState("networkidle"); + const url = page.url(); + expect(url).toMatch(/abilities/i); + }); + + test("should navigate to adversaries page filtered by emu plugin when clicking View Adversaries", async ({ + authenticatedPage: page, + }) => { + const viewAdversariesBtn = page.locator( + 'a:has-text("Adversaries"), a:has-text("View Adversaries")' + ); + await viewAdversariesBtn.first().click(); + + await page.waitForLoadState("networkidle"); + const url = page.url(); + expect(url).toMatch(/adversaries/i); + }); + + test("should fetch abilities from the API and filter emu-only abilities", async ({ + authenticatedPage: page, + }) => { + // Intercept the abilities API call to verify the request + const abilitiesResponse = await page.waitForResponse( + (response) => + response.url().includes("/api/v2/abilities") && + response.status() === 200, + { timeout: 20_000 } + ); + + const abilities = await abilitiesResponse.json(); + expect(Array.isArray(abilities)).toBe(true); + + // Filter for emu abilities + const emuAbilities = abilities.filter( + (a: any) => a.plugin === "emu" + ); + // There should be at least some emu abilities if the plugin is loaded + expect(emuAbilities.length).toBeGreaterThanOrEqual(0); + }); + + test("should fetch adversaries from the API and filter emu-only adversaries", async ({ + authenticatedPage: page, + }) => { + const adversariesResponse = await page.waitForResponse( + (response) => + response.url().includes("/api/v2/adversaries") && + response.status() === 200, + { timeout: 20_000 } + ); + + const adversaries = await adversariesResponse.json(); + expect(Array.isArray(adversaries)).toBe(true); + + const emuAdversaries = adversaries.filter( + (a: any) => a.plugin === "emu" + ); + expect(emuAdversaries.length).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/tests/e2e/specs/emu-error-states.spec.ts b/tests/e2e/specs/emu-error-states.spec.ts new file mode 100644 index 0000000..3345a5f --- /dev/null +++ b/tests/e2e/specs/emu-error-states.spec.ts @@ -0,0 +1,185 @@ +import { test, expect } from "../fixtures/caldera-auth"; + +test.describe("Emu plugin — error states and edge cases", () => { + test("should handle abilities API failure gracefully", async ({ + authenticatedPage: page, + }) => { + // Intercept abilities API and return error + await page.route("**/api/v2/abilities", (route) => + route.fulfill({ + status: 500, + contentType: "application/json", + body: JSON.stringify({ error: "Internal Server Error" }), + }) + ); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + // The page should still render the heading + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + + // Abilities count should show placeholder "---" since API failed + const abilitiesCount = page.locator("h1.is-size-1").first(); + await expect(abilitiesCount).toBeVisible({ timeout: 10_000 }); + const text = await abilitiesCount.textContent(); + expect(text?.trim()).toBe("---"); + }); + + test("should handle adversaries API failure gracefully", async ({ + authenticatedPage: page, + }) => { + // Intercept adversaries API and return error + await page.route("**/api/v2/adversaries", (route) => + route.fulfill({ + status: 500, + contentType: "application/json", + body: JSON.stringify({ error: "Internal Server Error" }), + }) + ); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + + // Adversaries count should show placeholder + const adversariesCount = page.locator("h1.is-size-1").nth(1); + await expect(adversariesCount).toBeVisible({ timeout: 10_000 }); + const text = await adversariesCount.textContent(); + expect(text?.trim()).toBe("---"); + }); + + test("should handle network timeout for abilities API gracefully", async ({ + authenticatedPage: page, + }) => { + await page.route("**/api/v2/abilities", (route) => route.abort()); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("domcontentloaded"); + + // Page should still render + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should handle network timeout for adversaries API gracefully", async ({ + authenticatedPage: page, + }) => { + await page.route("**/api/v2/adversaries", (route) => route.abort()); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("domcontentloaded"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should handle both APIs failing simultaneously", async ({ + authenticatedPage: page, + }) => { + await page.route("**/api/v2/abilities", (route) => + route.fulfill({ + status: 500, + body: "error", + }) + ); + await page.route("**/api/v2/adversaries", (route) => + route.fulfill({ + status: 500, + body: "error", + }) + ); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("domcontentloaded"); + + // Page should render without crashing + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + + // Both counts should show placeholder + const counts = page.locator("h1.is-size-1"); + const count = await counts.count(); + expect(count).toBeGreaterThanOrEqual(2); + }); + + test("should display abilities count as 0 when API returns empty array", async ({ + authenticatedPage: page, + }) => { + // Return empty arrays (no emu abilities) + await page.route("**/api/v2/abilities", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify([]), + }) + ); + await page.route("**/api/v2/adversaries", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify([]), + }) + ); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + + // With empty arrays filtered for emu, counts should show "---" (falsy 0) + const abilitiesCount = page.locator("h1.is-size-1").first(); + await expect(abilitiesCount).toBeVisible({ timeout: 10_000 }); + const text = await abilitiesCount.textContent(); + // 0 is falsy so the template shows "---" + expect(text?.trim()).toMatch(/^(0|---)$/); + }); + + test("should not crash when API returns non-emu abilities only", async ({ + authenticatedPage: page, + }) => { + // Return abilities that belong to a different plugin + await page.route("**/api/v2/abilities", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify([ + { + ability_id: "test-1", + name: "Test Ability", + plugin: "stockpile", + }, + ]), + }) + ); + await page.route("**/api/v2/adversaries", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify([ + { + adversary_id: "test-1", + name: "Test Adversary", + plugin: "stockpile", + }, + ]), + }) + ); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + + // Emu-filtered counts should be 0 (shown as "---") + const abilitiesCount = page.locator("h1.is-size-1").first(); + await expect(abilitiesCount).toBeVisible({ timeout: 10_000 }); + const text = await abilitiesCount.textContent(); + expect(text?.trim()).toMatch(/^(0|---)$/); + }); +}); diff --git a/tests/e2e/specs/emu-navigation.spec.ts b/tests/e2e/specs/emu-navigation.spec.ts new file mode 100644 index 0000000..5a8867e --- /dev/null +++ b/tests/e2e/specs/emu-navigation.spec.ts @@ -0,0 +1,104 @@ +import { test, expect } from "../fixtures/caldera-auth"; + +test.describe("Emu plugin — navigation and details view", () => { + test.beforeEach(async ({ authenticatedPage: page }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should show abilities page with emu-specific abilities after navigation", async ({ + authenticatedPage: page, + }) => { + const viewAbilitiesBtn = page.locator( + 'a:has-text("Abilities"), a:has-text("View Abilities")' + ); + await viewAbilitiesBtn.first().click(); + + await page.waitForLoadState("networkidle"); + + // On the abilities page, there should be some content rendered + // Look for typical abilities page elements + const pageContent = page.locator( + ".content, .abilities, table, .card, .panel" + ); + await expect(pageContent.first()).toBeVisible({ timeout: 15_000 }); + }); + + test("should show adversaries page with emu-specific adversaries after navigation", async ({ + authenticatedPage: page, + }) => { + const viewAdversariesBtn = page.locator( + 'a:has-text("Adversaries"), a:has-text("View Adversaries")' + ); + await viewAdversariesBtn.first().click(); + + await page.waitForLoadState("networkidle"); + + const pageContent = page.locator( + ".content, .adversaries, table, .card, .panel" + ); + await expect(pageContent.first()).toBeVisible({ timeout: 15_000 }); + }); + + test("should be able to return to the emu plugin page from abilities", async ({ + authenticatedPage: page, + }) => { + // Navigate to abilities + const viewAbilitiesBtn = page.locator( + 'a:has-text("Abilities"), a:has-text("View Abilities")' + ); + await viewAbilitiesBtn.first().click(); + await page.waitForLoadState("networkidle"); + + // Navigate back to emu + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should be able to return to the emu plugin page from adversaries", async ({ + authenticatedPage: page, + }) => { + const viewAdversariesBtn = page.locator( + 'a:has-text("Adversaries"), a:has-text("View Adversaries")' + ); + await viewAdversariesBtn.first().click(); + await page.waitForLoadState("networkidle"); + + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should preserve counts when returning to the emu page", async ({ + authenticatedPage: page, + }) => { + // Get initial counts + const abilitiesCount = page.locator("h1.is-size-1").first(); + await expect(async () => { + const text = await abilitiesCount.textContent(); + expect(text?.trim()).not.toBe("---"); + }).toPass({ timeout: 15_000 }); + + const initialAbilityText = await abilitiesCount.textContent(); + + // Navigate away and back + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const newAbilitiesCount = page.locator("h1.is-size-1").first(); + await expect(async () => { + const text = await newAbilitiesCount.textContent(); + expect(text?.trim()).not.toBe("---"); + }).toPass({ timeout: 15_000 }); + + const returnedAbilityText = await newAbilitiesCount.textContent(); + expect(returnedAbilityText?.trim()).toBe(initialAbilityText?.trim()); + }); +}); diff --git a/tests/e2e/specs/emu-page-load.spec.ts b/tests/e2e/specs/emu-page-load.spec.ts new file mode 100644 index 0000000..ffb5d43 --- /dev/null +++ b/tests/e2e/specs/emu-page-load.spec.ts @@ -0,0 +1,117 @@ +import { test, expect } from "../fixtures/caldera-auth"; + +test.describe("Emu plugin — page load and accessibility", () => { + test("should load the Caldera UI and find Emu in the plugin navigation", async ({ + authenticatedPage: page, + }) => { + await page.goto("/"); + + const emuLink = page.locator( + 'a[href*="emu"], a:has-text("emu"), [data-plugin="emu"], nav >> text=emu' + ); + await expect(emuLink.first()).toBeVisible({ timeout: 15_000 }); + }); + + test("should navigate to the Emu plugin page", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const heading = page.locator("h2:has-text('Emu')"); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test("should display the plugin description text", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const description = page.locator( + "text=Adversary Emulation Plans, text=CTID Adversary Emulation" + ); + await expect(description.first()).toBeVisible({ timeout: 15_000 }); + }); + + test("should display the abilities count card", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const abilitiesLabel = page.locator("p:has-text('abilities')").first(); + await expect(abilitiesLabel).toBeVisible({ timeout: 15_000 }); + }); + + test("should display the adversaries count card", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const adversariesLabel = page + .locator("p:has-text('adversaries')") + .first(); + await expect(adversariesLabel).toBeVisible({ timeout: 15_000 }); + }); + + test("should show numeric counts for abilities (not just placeholder)", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + // Wait for the abilities count to load (should be a number, not "---") + const abilitiesCount = page.locator("h1.is-size-1").first(); + await expect(abilitiesCount).toBeVisible({ timeout: 15_000 }); + + // The count should eventually become a number + await expect(async () => { + const text = await abilitiesCount.textContent(); + expect(text?.trim()).not.toBe("---"); + expect(Number(text?.trim())).toBeGreaterThanOrEqual(0); + }).toPass({ timeout: 15_000 }); + }); + + test("should show numeric counts for adversaries (not just placeholder)", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + // The adversaries count is the second h1.is-size-1 + const adversariesCount = page.locator("h1.is-size-1").nth(1); + await expect(adversariesCount).toBeVisible({ timeout: 15_000 }); + + await expect(async () => { + const text = await adversariesCount.textContent(); + expect(text?.trim()).not.toBe("---"); + expect(Number(text?.trim())).toBeGreaterThanOrEqual(0); + }).toPass({ timeout: 15_000 }); + }); + + test("should have a View Abilities button/link", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const viewAbilitiesBtn = page.locator( + 'a:has-text("Abilities"), button:has-text("Abilities"), a:has-text("View Abilities")' + ); + await expect(viewAbilitiesBtn.first()).toBeVisible({ timeout: 15_000 }); + }); + + test("should have a View Adversaries button/link", async ({ + authenticatedPage: page, + }) => { + await page.goto("/#/plugins/emu"); + await page.waitForLoadState("networkidle"); + + const viewAdversariesBtn = page.locator( + 'a:has-text("Adversaries"), button:has-text("Adversaries"), a:has-text("View Adversaries")' + ); + await expect(viewAdversariesBtn.first()).toBeVisible({ timeout: 15_000 }); + }); +});