diff --git a/page-objects/api-contract-page.ts b/page-objects/api-contract-page.ts index 4254d9d..830e158 100644 --- a/page-objects/api-contract-page.ts +++ b/page-objects/api-contract-page.ts @@ -71,6 +71,8 @@ export class ApiContractPage extends BasePage { private readonly sidebarProcessBar: (specName: string) => Locator; + readonly infoDialog: Locator; + constructor(page: Page, testInfo: TestInfo, eyes: any, specName: string) { super(page, testInfo, eyes, specName); this.specTree = page.locator("#spec-tree"); @@ -218,6 +220,8 @@ export class ApiContractPage extends BasePage { this.includeButton = this.specSection .locator("button.clear") .filter({ hasText: "Include" }); + + this.infoDialog = this.page.locator("#alert-container .alert-msg.info"); } //Function Beginning @@ -247,7 +251,7 @@ export class ApiContractPage extends BasePage { ); } - async clickRunContractTests() { + async clickRunContractTests(expectSuccess: boolean = true) { await test.step("Run Contract Tests", async () => { try { await expect(this._runButton).toBeEnabled({ timeout: 10000 }); @@ -260,7 +264,11 @@ export class ApiContractPage extends BasePage { await takeAndAttachScreenshot(this.page, "clicked-run-contract-tests"); - await this.waitForTestCompletion(); + if (expectSuccess) { + await this.waitForTestCompletion(); + } else { + await this.page.waitForTimeout(1000); + } } catch (e) { await takeAndAttachScreenshot(this.page, "error-in-run-contract-tests"); throw new Error(`Failed to click Run button:`); @@ -268,11 +276,35 @@ export class ApiContractPage extends BasePage { }); } + async waitforDialogToDismiss(status: string | RegExp) { + try { + const appeared = await this.infoDialog + .waitFor({ state: "visible", timeout: 5000 }) + .then(() => true) + .catch(() => false); + + if (!appeared) { + console.log("[INFO] Dialog did not appear — safe to continue"); + return; + } + + await expect.soft(this.infoDialog).toContainText(status, { + timeout: 10000, + }); + + await this.infoDialog.waitFor({ state: "hidden", timeout: 5000 }); + } catch (e) { + console.log("[WARN] Dialog wait issue — continuing:", e); + } + } + private async waitForTestCompletion() { await this.waitForTestsToStartRunning(); await this.waitForTestsToCompleteExecution(); + await this.waitforDialogToDismiss(/Tests? complete/i); + await takeAndAttachScreenshot(this.page, "test-completed", this.eyes); } @@ -376,6 +408,7 @@ export class ApiContractPage extends BasePage { path: string, method: string, response: string, + { captureVisual = true }: { captureVisual?: boolean } = {}, ) { const checkbox = this.exclusionCheckboxLocator(path, method, response); try { @@ -400,11 +433,13 @@ export class ApiContractPage extends BasePage { if (!isChecked) { await checkbox.click(); } - await takeAndAttachScreenshot( - this.page, - "excluded-test-" + `${path}-${method}-${response}`, - this.eyes, - ); + if (captureVisual) { + await takeAndAttachScreenshot( + this.page, + "excluded-test-" + `${path}-${method}-${response}`, + this.eyes, + ); + } } async clickExcludeButton() { @@ -442,8 +477,14 @@ export class ApiContractPage extends BasePage { testItem.path, testItem.method, testItem.response, + { captureVisual: false }, ); } + await takeAndAttachScreenshot( + this.page, + `multiple-tests-selected-${testList.length}`, + this.eyes, + ); } async getMixedOperationErrorText(): Promise { @@ -762,7 +803,9 @@ export class ApiContractPage extends BasePage { const summary = this.getPrereqErrorSummary(); const message = this.getPrereqErrorMessage(); - const isVisible = await summary.isVisible({ timeout: 3000 }).catch(() => false); + const isVisible = await summary + .isVisible({ timeout: 3000 }) + .catch(() => false); if (!isVisible) { console.log("No prerequisite error detected."); return; @@ -775,9 +818,13 @@ export class ApiContractPage extends BasePage { await summary.click(); - await expect(message).toBeVisible({ timeout: 5000 }).catch(() => { - console.warn("Detail message did not become visible after clicking summary"); - }); + await expect(message) + .toBeVisible({ timeout: 5000 }) + .catch(() => { + console.warn( + "Detail message did not become visible after clicking summary", + ); + }); await takeAndAttachScreenshot(this.page, "contract-prereq-error-expanded"); @@ -787,6 +834,6 @@ export class ApiContractPage extends BasePage { : ""; console.error(`Prerequisite error summary: ${summaryText}`); - console.error(`Prerequisite error detail: ${detailedMessage}`); + console.error(`Prerequisite error detail: ${detailedMessage}`); } } diff --git a/page-objects/example-generation-page.ts b/page-objects/example-generation-page.ts index 314151c..399cfce 100644 --- a/page-objects/example-generation-page.ts +++ b/page-objects/example-generation-page.ts @@ -11,6 +11,8 @@ import { import { takeAndAttachScreenshot } from "../utils/screenshotUtils"; import { BasePage } from "./base-page"; import { Edit } from "../utils/types/json-edit.types"; +import { SpecEditorPage } from "./spec-editor-page"; + export class ExampleGenerationPage extends BasePage { readonly openApiTabPage: OpenAPISpecTabPage; @@ -30,6 +32,8 @@ export class ExampleGenerationPage extends BasePage { private readonly specSection: Locator; private readonly specEditorSection: Locator; private readonly specTabLocator: Locator; + private readonly specEditorHelper: SpecEditorPage; + constructor(page: Page, testInfo: TestInfo, eyes: any, specName: string) { super(page, testInfo, eyes, specName); @@ -62,8 +66,10 @@ export class ExampleGenerationPage extends BasePage { this.bulkValidateBtnSelector = "button#bulk-validate"; this.inlineBtnSelector = "button#import"; this.bulkFixBtnSelector = "button#bulk-fix"; + this.specEditorHelper = new SpecEditorPage(page); } + private async openExampleGenerationTab() { console.log("Opening Example Generation tab"); return this.openApiTabPage.openExampleGenerationTab(); @@ -1134,7 +1140,7 @@ export class ExampleGenerationPage extends BasePage { await expect(editorContext.content).toBeVisible({ timeout: 15000 }); await editorContext.content.click(); - await this.loadFullEditorDocument(editorContext.scroller); + await this.specEditorHelper.loadFullEditorDocument(editorContext.scroller); await editorContext.scroller.evaluate((el) => { el.scrollTop = 0; }); @@ -1145,7 +1151,7 @@ export class ExampleGenerationPage extends BasePage { : [`${exampleName}_response`]; for (const searchTerm of targets) { - const foundByEditorApi = await this.focusTermUsingCodeMirrorApi( + const foundByEditorApi = await this.specEditorHelper.focusTermUsingCodeMirrorApi( editorContext.content, searchTerm, ); @@ -1157,12 +1163,18 @@ export class ExampleGenerationPage extends BasePage { ); if (!foundByWindowFind) { - await this.scrollToEditorSearchTerm( + await this.specEditorHelper.scrollEditorToFindTerm( editorContext.content, editorContext.scroller, editorContext.lines, searchTerm, ); + // Click the matched line for visual cursor placement + const match = editorContext.lines.filter({ hasText: searchTerm }).first(); + if ((await match.count()) > 0) { + await match.scrollIntoViewIfNeeded(); + await match.click(); + } } await this.page.waitForTimeout(250); @@ -1230,113 +1242,4 @@ export class ExampleGenerationPage extends BasePage { return window.find(term, false, false, true, false, false, false); }, searchTerm); } - - private async focusTermUsingCodeMirrorApi( - content: Locator, - searchTerm: string, - ): Promise { - return await content.evaluate((el, term) => { - const cmEditor = el.closest(".cm-editor") as any; - const view = cmEditor?.cmView?.view; - if (!view) return false; - - const fullText = view.state.doc.toString() as string; - const index = fullText.indexOf(term); - if (index === -1) return false; - - view.dispatch({ - selection: { anchor: index, head: index + term.length }, - scrollIntoView: true, - }); - return true; - }, searchTerm); - } - - private async loadFullEditorDocument(scroller: Locator): Promise { - await expect(scroller).toBeVisible({ timeout: 10000 }); - - let unchangedCount = 0; - let previousScrollHeight = -1; - - for (let i = 0; i < 60; i++) { - const metrics = await scroller.evaluate((el) => { - el.scrollTop = el.scrollHeight; - return { - scrollTop: el.scrollTop, - scrollHeight: el.scrollHeight, - clientHeight: el.clientHeight, - }; - }); - - if (metrics.scrollHeight === previousScrollHeight) { - unchangedCount += 1; - } else { - unchangedCount = 0; - } - - previousScrollHeight = metrics.scrollHeight; - - const atBottom = - metrics.scrollTop + metrics.clientHeight >= metrics.scrollHeight - 2; - if (atBottom && unchangedCount >= 2) { - break; - } - - await this.page.waitForTimeout(120); - } - } - - private async scrollToEditorSearchTerm( - content: Locator, - scroller: Locator, - lines: Locator, - searchTerm: string, - ): Promise { - await scroller.evaluate((el) => { - el.scrollTop = 0; - }); - await this.page.waitForTimeout(120); - - for (let i = 0; i < 250; i++) { - const match = lines.filter({ hasText: searchTerm }).first(); - const count = await match.count(); - - if (count > 0) { - await match.scrollIntoViewIfNeeded(); - await match.click(); - return; - } - - const moved = await scroller.evaluate((el) => { - const prev = el.scrollTop; - el.scrollTop = Math.min( - el.scrollTop + Math.max(el.clientHeight * 0.85, 120), - el.scrollHeight, - ); - return el.scrollTop > prev; - }); - - if (!moved) { - break; - } - - await this.page.waitForTimeout(80); - } - - // Fallback: some editor layouts scroll via outer containers, not .cm-scroller. - await content.hover(); - for (let i = 0; i < 280; i++) { - const visible = await lines.evaluateAll( - (els, term) => els.some((el) => el.textContent?.includes(term)), - searchTerm, - ); - if (visible) { - return; - } - await this.page.mouse.wheel(0, 900); - await this.page.waitForTimeout(60); - } - - throw new Error(`Could not find '${searchTerm}' in the spec editor`); - } } diff --git a/page-objects/service-spec-config-page.ts b/page-objects/service-spec-config-page.ts index 65859ff..b139545 100644 --- a/page-objects/service-spec-config-page.ts +++ b/page-objects/service-spec-config-page.ts @@ -2,6 +2,7 @@ import { OpenAPISpecTabPage } from "./openapi-spec-tab-page"; import { Locator, expect, type TestInfo, Page, test } from "@playwright/test"; import { takeAndAttachScreenshot } from "../utils/screenshotUtils"; import { BasePage } from "./base-page"; +import { SpecEditorPage } from "./spec-editor-page"; export interface Edit { current: { @@ -37,6 +38,8 @@ export class ServiceSpecConfigPage extends BasePage { readonly alertContainer: Locator; readonly alertTitle: Locator; readonly alertDescription: Locator; + private readonly specEditorHelper: SpecEditorPage; + constructor(page: Page, testInfo: TestInfo, eyes: any, specName: string) { super(page, testInfo, eyes, specName); @@ -78,7 +81,9 @@ export class ServiceSpecConfigPage extends BasePage { this.alertContainer = page.locator(".alert-msg.error"); this.alertTitle = this.alertContainer.locator("p"); this.alertDescription = this.alertContainer.locator("pre"); + this.specEditorHelper = new SpecEditorPage(page); } + async openSpecTab() { return this.openApiTabPage.openSpecTab(); } @@ -142,6 +147,53 @@ export class ServiceSpecConfigPage extends BasePage { }); } + async editSpecInEditor(searchText: string, replaceText: string) { + await test.step(`Edit spec in editor: '${searchText}' -> '${replaceText}'`, async () => { + const content = this.specSection.locator(".cm-content").first(); + const scroller = this.specSection.locator(".cm-scroller").first(); + const lines = this.specSection.locator(".cm-content .cm-line"); + + await expect(content).toBeVisible({ timeout: 10000 }); + + // Phase 1: scroll .cm-scroller to bottom so CodeMirror renders all lines + await this.specEditorHelper.loadFullEditorDocument(scroller); + await scroller.evaluate((el) => { el.scrollTop = 0; }); + await this.page.waitForTimeout(200); + + // Phase 2: try the CodeMirror API to jump directly to the term + const foundByApi = await this.specEditorHelper.focusTermUsingCodeMirrorApi(content, searchText); + + // Phase 3: if the API didn't work, manually scroll to find the line + if (!foundByApi) { + await this.specEditorHelper.scrollEditorToFindTerm(content, scroller, lines, searchText); + } + + const targetLine = lines.filter({ hasText: searchText }).first(); + await expect(targetLine).toBeVisible({ timeout: 10000 }); + + const originalText = await targetLine.innerText(); + const leadingSpaces = originalText.match(/^\s*/)?.[0] ?? ""; + + await targetLine.scrollIntoViewIfNeeded(); + await targetLine.click(); + + // Home twice: first Home moves past soft indent, second goes to column 0 + await this.page.keyboard.press("Home"); + await this.page.keyboard.press("Home"); + await this.page.keyboard.press("Shift+End"); + await this.page.keyboard.press("Backspace"); + await this.page.keyboard.type(leadingSpaces + replaceText.trim()); + + const safeFileName = replaceText.trim().replace(/[^a-zA-Z0-9]/g, "-"); + await takeAndAttachScreenshot( + this.page, + `spec-editor-edit-${safeFileName}`, + this.eyes, + ); + }); + } + + async deleteSpecLinesInEditor(searchText: string, lineCount: number = 1) { await test.step(`Delete ${lineCount} spec line(s) starting with '${searchText}'`, async () => { const lines = this.specSection.locator(".cm-content .cm-line"); diff --git a/page-objects/spec-editor-page.ts b/page-objects/spec-editor-page.ts new file mode 100644 index 0000000..cdf6abb --- /dev/null +++ b/page-objects/spec-editor-page.ts @@ -0,0 +1,102 @@ +import { Locator, Page, expect } from "@playwright/test"; + +export class SpecEditorPage { + constructor(private readonly page: Page) {} + async loadFullEditorDocument(scroller: Locator): Promise { + await expect(scroller).toBeVisible({ timeout: 10000 }); + + let unchangedCount = 0; + let previousScrollHeight = -1; + + for (let i = 0; i < 60; i++) { + const metrics = await scroller.evaluate((el) => { + el.scrollTop = el.scrollHeight; + return { + scrollTop: el.scrollTop, + scrollHeight: el.scrollHeight, + clientHeight: el.clientHeight, + }; + }); + + if (metrics.scrollHeight === previousScrollHeight) { + unchangedCount += 1; + } else { + unchangedCount = 0; + } + + previousScrollHeight = metrics.scrollHeight; + + const atBottom = + metrics.scrollTop + metrics.clientHeight >= metrics.scrollHeight - 2; + if (atBottom && unchangedCount >= 2) break; + + await this.page.waitForTimeout(120); + } + } + + async focusTermUsingCodeMirrorApi( + content: Locator, + searchTerm: string, + ): Promise { + return await content.evaluate((el, term) => { + const cmEditor = el.closest(".cm-editor") as any; + const view = cmEditor?.cmView?.view; + if (!view) return false; + + const fullText = view.state.doc.toString() as string; + const index = fullText.indexOf(term); + if (index === -1) return false; + + view.dispatch({ + selection: { anchor: index, head: index + term.length }, + scrollIntoView: true, + }); + return true; + }, searchTerm); + } + + async scrollEditorToFindTerm( + content: Locator, + scroller: Locator, + lines: Locator, + searchTerm: string, + ): Promise { + await scroller.evaluate((el) => { + el.scrollTop = 0; + }); + await this.page.waitForTimeout(120); + + for (let i = 0; i < 250; i++) { + const match = lines.filter({ hasText: searchTerm }).first(); + if ((await match.count()) > 0) { + await match.scrollIntoViewIfNeeded(); + return; + } + + const moved = await scroller.evaluate((el) => { + const prev = el.scrollTop; + el.scrollTop = Math.min( + el.scrollTop + Math.max(el.clientHeight * 0.85, 120), + el.scrollHeight, + ); + return el.scrollTop > prev; + }); + + if (!moved) break; + await this.page.waitForTimeout(80); + } + + await content.hover(); + for (let i = 0; i < 280; i++) { + const visible = await lines.evaluateAll( + (els, term) => els.some((el) => el.textContent?.includes(term)), + searchTerm, + ); + if (visible) return; + await this.page.mouse.wheel(0, 900); + await this.page.waitForTimeout(60); + } + + throw new Error(`Could not find '${searchTerm}' in the spec editor`); + } +} diff --git a/specmatic-studio-demo/specs/product_search_bff_v5_backward_incompatibility.yaml b/specmatic-studio-demo/specs/product_search_bff_v5_backward_incompatibility.yaml new file mode 100644 index 0000000..575460f --- /dev/null +++ b/specmatic-studio-demo/specs/product_search_bff_v5_backward_incompatibility.yaml @@ -0,0 +1,290 @@ +openapi: 3.0.0 +info: + title: Order BFF + description: Sample Order BFF + version: '5.0' + contact: + email: devs@specmatic.io +servers: + - url: http://localhost:8080 +tags: + - name: WIP + description: API still under development + - name: Product + description: Product Related APIs + - name: Order + description: Order Related APIs + - name: Monitor + description: Monitor Related APIs +paths: + /products: + summary: Create a new product + post: + description: Create a new product + operationId: createProduct + tags: + - Product + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProductBase' + responses: + '201': + description: Product created + content: + application/json: + schema: + $ref: '#/components/schemas/Id' + '202': + description: Product Accepted + headers: + Link: + schema: + $ref: '#/components/schemas/MonitorLink' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequest' + /findAvailableProducts: + parameters: + - schema: + $ref : '#/components/schemas/ProductType' + name: type + in: query + required: false + - name: pageSize + in: header + schema: + type: integer + required: true + - name: from-date + in: query + required: true + schema: + type: string + format: date + - name: to-date + in: query + required: true + schema: + type: string + format: date + get: + description: Find available products + operationId: findAvailableProducts + tags: + - Product + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Product' + '429': + description: TooManyRequests + headers: + Retry-After: + schema: + type: integer + description: The number of seconds to wait before making the next request + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequest' + /ordres: + post: + description: Create a new order + operationId: createOrder + tags: + - Order + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/OrderBase' + responses: + '201': + description: Order created + content: + application/json: + schema: + $ref: '#/components/schemas/Id' + '202': + description: Order accepted + headers: + Link: + schema: + $ref: '#/components/schemas/MonitorLink' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequest' + /monitor/{id}: + get: + description: Retrieves a Monitor by ID + operationId: retrieveMonitor + tags: + - Monitor + parameters: + - name: id + in: path + required: true + schema: + type: integer + responses: + '200': + description: Monitor status + content: + application/json: + schema: + $ref: '#/components/schemas/MonitorResponse' + '400': + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/BadRequest' +components: + schemas: + ProductInventory: + type: integer + minimum: 1 + maximum: 101 + ProductType: + type: string + title: Product Type + enum: + - book + - food + - gadget + - other + MonitorLink: + type: string + pattern: "<\\/monitor\\/\\d+>;rel=related;title=monitor" + description: Monitor link to check status + example: ;rel=related;title=monitor + Id: + type: object + properties: + id: + type: integer + required: + - id + ProductBase: + title: Product Details + type: object + properties: + name: + type: string + type: + $ref: '#/components/schemas/ProductType' + inventory: + $ref: '#/components/schemas/ProductInventory' + required: + - name + - type + - inventory + Product: + allOf: + - $ref: '#/components/schemas/Id' + - $ref: '#/components/schemas/ProductBase' + - $ref: '#/components/schemas/CreatedOn' + CreatedOn: + type: object + required: + - createdOn + properties: + createdOn: + type: string + format: date + OrderBase: + title: Order Details + type: object + properties: + productid: + type: integer + count: + type: integer + required: + - productid + - count + Order: + allOf: + - $ref: '#/components/schemas/Id' + - $ref: '#/components/schemas/OrderBase' + - type: object + properties: + status: + type: string + enum: [pending, completed, cancelled] + required: + - status + MonitorResponse: + type: object + description: Monitoring of resources + properties: + request: + $ref: '#/components/schemas/Request' + response: + $ref: '#/components/schemas/Response' + Request: + type: object + description: A response to a request + properties: + method: + type: string + body: + type: object + headers: + type: array + items: + $ref: '#/components/schemas/HeaderItem' + required: + - method + - body + - headers + Response: + type: object + description: A response to a request + properties: + statusCode: + type: integer + body: + type: object + headers: + type: array + items: + $ref: '#/components/schemas/HeaderItem' + required: + - statusCode + - body + - headers + HeaderItem: + type: object + properties: + name: + type: string + value: + type: string + BadRequest: + title: Bad Request + type: object + properties: + timestamp: + type: string + status: + type: integer + error: + type: string + message: + type: string \ No newline at end of file diff --git a/specs/openapi/execute-contract-tests/execute-contract-tests-excluded.spec.ts b/specs/openapi/execute-contract-tests/execute-contract-tests-excluded.spec.ts index 4642ab6..ccdedf2 100644 --- a/specs/openapi/execute-contract-tests/execute-contract-tests-excluded.spec.ts +++ b/specs/openapi/execute-contract-tests/execute-contract-tests-excluded.spec.ts @@ -9,18 +9,44 @@ import { verifyRightSidebarStatus, } from "../helpers/execute-contract-tests-helper"; + +type Counts = Parameters[1]; + +async function runAndVerifyCounts( + contractPage: ApiContractPage, + expectedCounts: Counts, +) { + await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); + await contractPage.clickRunContractTests(); + await verifyRightSidebarStatus( + contractPage, + "Done", + PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, + ); + + const { path } = await contractPage.getAllHeaderTotals(); + expect(path, "Path header should match unique paths in table").toBe( + await contractPage.getUniqueValuesInColumn(2), + ); + + await validateSummaryAndTableCounts(contractPage, expectedCounts); +} + + test.describe("API Contract testing with test exclusion and inclusion", () => { test( "Exclude specific tests and verify excluded tests are not executed", { tag: ["@test", "@testExclusion", "@eyes", "@expected-failure"] }, async ({ page, eyes }, testInfo) => { test.fail(true, "Needs fixing by the devs"); + const contractPage = new ApiContractPage( page, testInfo, eyes, PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, ); + await test.step(`Go to Test page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED}'`, async () => { await contractPage.openContractTestTabForSpec( testInfo, @@ -28,168 +54,56 @@ test.describe("API Contract testing with test exclusion and inclusion", () => { PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, ); }); - await test.step("Exclude single test", async () => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, - ); - await contractPage.selectTestForExclusionOrInclusion( - "/products", - "POST", - "201", - ); + await test.step("Exclude single test and verify counts decrease", async () => { + await contractPage.selectTestForExclusionOrInclusion("/products", "POST", "201"); await contractPage.clickExcludeButton(); - await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); - await contractPage.clickRunContractTests(); - await verifyRightSidebarStatus( - contractPage, - "Done", - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, - ); - - const tableHeaderTotals = await contractPage.getAllHeaderTotals(); - - expect( - tableHeaderTotals.path, - "Path header should match unique paths in table", - ).toBe(await contractPage.getUniqueValuesInColumn(2)); - - await validateSummaryAndTableCounts(contractPage, { - success: 0, - failed: 20, - total: 23, - error: 0, - notcovered: 2, - excluded: 1, + await runAndVerifyCounts(contractPage, { + success: 0, failed: 20, total: 23, error: 0, notcovered: 2, excluded: 1, }); }); - await test.step("Include single test", async () => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, - ); - await contractPage.selectTestForExclusionOrInclusion( - "/products", - "POST", - "201", - ); + await test.step("Include single test and verify counts are restored", async () => { + await contractPage.selectTestForExclusionOrInclusion("/products", "POST", "201"); await contractPage.clickIncludeButton(); - await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); - await contractPage.clickRunContractTests(); - - const tableHeaderTotals = await contractPage.getAllHeaderTotals(); - - expect( - tableHeaderTotals.path, - "Path header should match unique paths in table", - ).toBe(await contractPage.getUniqueValuesInColumn(2)); - - await validateSummaryAndTableCounts(contractPage, { - success: 12, - failed: 20, - total: 34, - error: 0, - notcovered: 2, - excluded: 0, + await runAndVerifyCounts(contractPage, { + success: 12, failed: 20, total: 34, error: 0, notcovered: 2, excluded: 0, }); }); await test.step("Exclude multiple tests across different endpoints", async () => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, - ); - await contractPage.selectMultipleTests([ { path: "/products", method: "POST", response: "201" }, { path: "/findAvailableProducts", method: "GET", response: "200" }, ]); await contractPage.clickExcludeButton(); - await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); - await contractPage.clickRunContractTests(); - - const tableHeaderTotals = await contractPage.getAllHeaderTotals(); - - expect( - tableHeaderTotals.path, - "Path header should match unique paths in table", - ).toBe(await contractPage.getUniqueValuesInColumn(2)); - - await validateSummaryAndTableCounts(contractPage, { - success: 0, - failed: 15, - total: 19, - error: 0, - notcovered: 2, - excluded: 2, + await runAndVerifyCounts(contractPage, { + success: 0, failed: 15, total: 19, error: 0, notcovered: 2, excluded: 2, }); }); - await test.step("Include multiple tests across different endpoints", async () => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_EXCLUDED, - ); + await test.step("Include multiple tests and verify all counts are restored", async () => { await contractPage.selectMultipleTests([ { path: "/products", method: "POST", response: "201" }, { path: "/findAvailableProducts", method: "GET", response: "200" }, ]); await contractPage.clickIncludeButton(); - await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); - await contractPage.clickRunContractTests(); - - const tableHeaderTotals = await contractPage.getAllHeaderTotals(); - - expect( - tableHeaderTotals.path, - "Path header should match unique paths in table", - ).toBe(await contractPage.getUniqueValuesInColumn(2)); - - await validateSummaryAndTableCounts(contractPage, { - success: 12, - failed: 20, - total: 34, - error: 0, - notcovered: 2, - excluded: 0, + await runAndVerifyCounts(contractPage, { + success: 12, failed: 20, total: 34, error: 0, notcovered: 2, excluded: 0, }); }); await test.step("Verify error when mixing inclusive and exclusive operations", async () => { - await contractPage.selectTestForExclusionOrInclusion( - "/products", - "POST", - "201", - ); + await contractPage.selectTestForExclusionOrInclusion("/products", "POST", "201"); await contractPage.clickExcludeButton(); - await contractPage.selectTestForExclusionOrInclusion( - "/products", - "POST", - "201", - ); - await contractPage.selectTestForExclusionOrInclusion( - "/products", - "POST", - "202", - ); + await contractPage.selectTestForExclusionOrInclusion("/products", "POST", "201"); + await contractPage.selectTestForExclusionOrInclusion("/products", "POST", "202"); - const actualErrorMessage = - await contractPage.getMixedOperationErrorText(); - - const expectedMessage = - "A combination of inclusive and exclusive operations have been selected, Please select only one type"; - expect(actualErrorMessage).toContain(expectedMessage); + const actualErrorMessage = await contractPage.getMixedOperationErrorText(); + expect(actualErrorMessage).toContain( + "A combination of inclusive and exclusive operations have been selected, Please select only one type", + ); }); }, ); diff --git a/specs/openapi/execute-contract-tests/execute-contract-tests-filter.spec.ts b/specs/openapi/execute-contract-tests/execute-contract-tests-filter.spec.ts index 8042382..98c71d9 100644 --- a/specs/openapi/execute-contract-tests/execute-contract-tests-filter.spec.ts +++ b/specs/openapi/execute-contract-tests/execute-contract-tests-filter.spec.ts @@ -7,25 +7,28 @@ import { ApiContractPage } from "../../../page-objects/api-contract-page"; import { verifyRightSidebarStatus } from "../helpers/execute-contract-tests-helper"; test.describe("API Contract Testing - Filtering", () => { - test( - "Verify filtering by header", - { tag: ["@test", "@filterTest", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - const contractPage = new ApiContractPage( - page, + let contractPage: ApiContractPage; + + test.beforeEach(async ({ page, eyes }, testInfo) => { + contractPage = new ApiContractPage( + page, + testInfo, + eyes, + PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_FILTER, + ); + await test.step(`Go to Test page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_FILTER}'`, async () => { + await contractPage.openContractTestTabForSpec( testInfo, eyes, PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_FILTER, ); + }); + }); - await test.step(`Go to Test page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_FILTER}'`, async () => { - await contractPage.openContractTestTabForSpec( - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_FILTER, - ); - }); - + test( + "Verify filtering by header", + { tag: ["@test", "@filterTest", "@eyes"] }, + async () => { await test.step("Enter service URL and run contract tests", async () => { await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); await contractPage.setGenerativeMode(false); diff --git a/specs/openapi/execute-contract-tests/execute-contract-tests-main.spec.ts b/specs/openapi/execute-contract-tests/execute-contract-tests-main.spec.ts index 4a3b768..4cc7e43 100644 --- a/specs/openapi/execute-contract-tests/execute-contract-tests-main.spec.ts +++ b/specs/openapi/execute-contract-tests/execute-contract-tests-main.spec.ts @@ -11,25 +11,28 @@ import { } from "../helpers/execute-contract-tests-helper"; test.describe("API Contract Testing", () => { - test( - "Run contract tests for openapi spec product_search_bff_v5.yaml with default settings", - { tag: ["@test", "@runContractTests", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - const contractPage = new ApiContractPage( - page, + let contractPage: ApiContractPage; + + test.beforeEach(async ({ page, eyes }, testInfo) => { + contractPage = new ApiContractPage( + page, + testInfo, + eyes, + PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT, + ); + await test.step(`Go to Test page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT}'`, async () => { + await contractPage.openContractTestTabForSpec( testInfo, eyes, PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT, ); + }); + }); - await test.step(`Go to Test page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT}'`, async () => { - await contractPage.openContractTestTabForSpec( - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT, - ); - }); - + test( + "Run contract tests for openapi spec product_search_bff_v5.yaml with default settings", + { tag: ["@test", "@runContractTests", "@eyes"] }, + async () => { await test.step("Enter service URL and run contract tests", async () => { await contractPage.enterServiceUrl(ORDER_BFF_SERVICE_URL); await contractPage.clickRunContractTests(); @@ -42,6 +45,7 @@ test.describe("API Contract Testing", () => { PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_DEFAULT, ); }); + await test.step("Verify test results and remarks for executed contract tests", async () => { await contractPage.verifyTestResults(); @@ -70,6 +74,7 @@ test.describe("API Contract Testing", () => { excluded: 0, }); }); + await test.step("Identify and Toggle Views for first and last failed Tests", async () => { await toggleFailedTestViewForTableandRaw(contractPage); }); diff --git a/specs/openapi/execute-contract-tests/execute-contract-tests-negative.spec.ts b/specs/openapi/execute-contract-tests/execute-contract-tests-negative.spec.ts index e43da0b..ef2bfe8 100644 --- a/specs/openapi/execute-contract-tests/execute-contract-tests-negative.spec.ts +++ b/specs/openapi/execute-contract-tests/execute-contract-tests-negative.spec.ts @@ -3,14 +3,15 @@ import { PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_NEGATIVE } from "../../specNames import { ApiContractPage } from "../../../page-objects/api-contract-page"; test.describe("API Contract Testing - Negative Scenarios", () => { + let contractPage: ApiContractPage; + test.beforeEach(async ({ page, eyes }, testInfo) => { - const contractPage = new ApiContractPage( + contractPage = new ApiContractPage( page, testInfo, eyes, PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_NEGATIVE, ); - await contractPage.openContractTestTabForSpec( testInfo, eyes, @@ -21,13 +22,7 @@ test.describe("API Contract Testing - Negative Scenarios", () => { test( "Verify error for invalid service URL", { tag: ["@test", "@negative", "@wrongServiceURL", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_NEGATIVE, - ); + async () => { const invalidUrl = "http://ww.gag.com"; await test.step("Enter invalid service URL", async () => { @@ -35,7 +30,7 @@ test.describe("API Contract Testing - Negative Scenarios", () => { }); await test.step("Run contract tests and expect error", async () => { - await contractPage.clickRunContractTests(); + await contractPage.clickRunContractTests(false); }); await test.step("Verify prerequisite error is visible", async () => { @@ -49,13 +44,7 @@ test.describe("API Contract Testing - Negative Scenarios", () => { test( "Verify error for invalid port", { tag: ["@test", "@negative", "@wrongPort", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - const contractPage = new ApiContractPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_CONTRACT_TESTS_NEGATIVE, - ); + async () => { const invalidPortUrl = "http://order-bff:9999"; await test.step("Enter service URL with invalid port", async () => { @@ -63,7 +52,7 @@ test.describe("API Contract Testing - Negative Scenarios", () => { }); await test.step("Run contract tests and expect error", async () => { - await contractPage.clickRunContractTests(); + await contractPage.clickRunContractTests(false); }); await test.step("Verify prerequisite error is visible", async () => { diff --git a/specs/openapi/generate-valid-examples/validate-post-inlined-examples.spec.ts b/specs/openapi/generate-valid-examples/generate-more-examples-for-post-validate-inline-bcc.spec.ts similarity index 53% rename from specs/openapi/generate-valid-examples/validate-post-inlined-examples.spec.ts rename to specs/openapi/generate-valid-examples/generate-more-examples-for-post-validate-inline-bcc.spec.ts index e2dc6ff..7dc6aa2 100644 --- a/specs/openapi/generate-valid-examples/validate-post-inlined-examples.spec.ts +++ b/specs/openapi/generate-valid-examples/generate-more-examples-for-post-validate-inline-bcc.spec.ts @@ -1,3 +1,4 @@ +import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page"; import { test, expect } from "../../../utils/eyesFixture"; import { PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_POST_INLINED } from "../../specNames"; import { @@ -9,9 +10,18 @@ import { verifyAndCloseInlineSuccessDialog, } from "../helpers/inline-examples-helper"; +const SPEC = PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_POST_INLINED; const PRODUCTS = "products"; const ORDRES = "ordres"; +/** All path + response-code combinations exercised by this test. */ +const POST_PATHS_AND_CODES = [ + { path: PRODUCTS, code: 201 }, + { path: PRODUCTS, code: 400 }, + { path: ORDRES, code: 201 }, + { path: ORDRES, code: 400 }, +]; + test.describe("Validate generated spec after inlining POST request examples", () => { test( "POST multiple paths, multiple response codes - Generate, validate, inline and verify updated spec", @@ -28,7 +38,7 @@ test.describe("Validate generated spec after inlining POST request examples", () page, testInfo, eyes, - PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_POST_INLINED, + SPEC, [ { path: PRODUCTS, responseCodes: [201, 400] }, { path: ORDRES, responseCodes: [201, 400] }, @@ -37,45 +47,28 @@ test.describe("Validate generated spec after inlining POST request examples", () const generatedExampleNames = await generateMoreThenValidateAndInline( examplePage, - [ - { path: PRODUCTS, code: 201 }, - { path: PRODUCTS, code: 400 }, - { path: ORDRES, code: 201 }, - { path: ORDRES, code: 400 }, - ], + POST_PATHS_AND_CODES, ); - const updatedSpecName = getUpdatedSpecName(PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_POST_INLINED); + const updatedSpecName = getUpdatedSpecName(SPEC); await verifyAndCloseInlineSuccessDialog(examplePage, updatedSpecName); await test.step("Verify inlined POST examples appear in the updated spec file", async () => { - const updatedSpecPage = await navigateToUpdatedSpec( - page, - testInfo, - eyes, - updatedSpecName, - ); + const updatedSpecPage = await navigateToUpdatedSpec(page, testInfo, eyes, updatedSpecName); - await updatedSpecPage.verifyInlinedPostExamplesInSpec( - filterExampleNames(generatedExampleNames, PRODUCTS, 201), - PRODUCTS, - 201, - ); - await updatedSpecPage.verifyInlinedPostExamplesInSpec( - filterExampleNames(generatedExampleNames, PRODUCTS, 400), - PRODUCTS, - 400, - ); - await updatedSpecPage.verifyInlinedPostExamplesInSpec( - filterExampleNames(generatedExampleNames, ORDRES, 201), - ORDRES, - 201, - ); - await updatedSpecPage.verifyInlinedPostExamplesInSpec( - filterExampleNames(generatedExampleNames, ORDRES, 400), - ORDRES, - 400, - ); + for (const { path, code } of POST_PATHS_AND_CODES) { + await updatedSpecPage.verifyInlinedPostExamplesInSpec( + filterExampleNames(generatedExampleNames, path, code), + path, + code, + ); + } + }); + await test.step("Verify inlined examples are backward compatible", async () => { + const configPage = new ServiceSpecConfigPage(page, testInfo, eyes, updatedSpecName); + await configPage.runBackwardCompatibilityTest(); + const toastText = await configPage.getAlertMessageText(); + expect(toastText).toBe("Changes are backward compatible"); }); }, ); diff --git a/specs/openapi/generate-valid-examples/generate-more-examples-validate-inline.spec.ts b/specs/openapi/generate-valid-examples/generate-more-examples-validate-inline.spec.ts new file mode 100644 index 0000000..ca6d8ca --- /dev/null +++ b/specs/openapi/generate-valid-examples/generate-more-examples-validate-inline.spec.ts @@ -0,0 +1,68 @@ +import { test } from "../../../utils/eyesFixture"; +import { PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED } from "../../specNames"; +import { + filterExampleNames, + generateMoreThenValidateAndInline, + getUpdatedSpecName, + navigateToUpdatedSpec, + setupExampleGenerationPage, + verifyAndCloseInlineSuccessDialog, +} from "../helpers/inline-examples-helper"; + +const SPEC = PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED; +const FIND_AVAILABLE_PRODUCTS = "findAvailableProducts"; +const PRODUCTS = "products"; + +const PATHS_AND_CODES = [ + { path: FIND_AVAILABLE_PRODUCTS, method: "get", code: 200 }, + { path: FIND_AVAILABLE_PRODUCTS, method: "get", code: 400 }, + { path: PRODUCTS, method: "post", code: 201 }, + { path: PRODUCTS, method: "post", code: 400 }, +]; + +test.describe("Validate generated spec after inlining GET examples", () => { + test( + "Multiple paths, multiple response codes - Generate, validate, inline and verify updated spec", + { + tag: [ + "@examples", + "@inlineExamples", + "@validateInlinedExamplesForMultiplePaths", + "@eyes", + ], + }, + async ({ page, eyes }, testInfo) => { + const examplePage = await setupExampleGenerationPage( + page, + testInfo, + eyes, + SPEC, + [ + { path: FIND_AVAILABLE_PRODUCTS, responseCodes: [200, 400] }, + { path: PRODUCTS, responseCodes: [201, 400] }, + ], + ); + + const generatedExampleNames = await generateMoreThenValidateAndInline( + examplePage, + PATHS_AND_CODES, + ); + + const updatedSpecName = getUpdatedSpecName(SPEC); + await verifyAndCloseInlineSuccessDialog(examplePage, updatedSpecName); + + await test.step("Verify inlined examples appear in the updated spec file", async () => { + const updatedSpecPage = await navigateToUpdatedSpec(page, testInfo, eyes, updatedSpecName); + + for (const { path, method, code } of PATHS_AND_CODES) { + await updatedSpecPage.verifyInlinedExamplesInSpec( + filterExampleNames(generatedExampleNames, path, code), + path, + method, + code, + ); + } + }); + }, + ); +}); diff --git a/specs/openapi/generate-valid-examples/generate-valid-examples-for-2-paths.spec.ts b/specs/openapi/generate-valid-examples/generate-valid-examples-for-multiple-paths.spec.ts similarity index 95% rename from specs/openapi/generate-valid-examples/generate-valid-examples-for-2-paths.spec.ts rename to specs/openapi/generate-valid-examples/generate-valid-examples-for-multiple-paths.spec.ts index 04088d1..d46131c 100644 --- a/specs/openapi/generate-valid-examples/generate-valid-examples-for-2-paths.spec.ts +++ b/specs/openapi/generate-valid-examples/generate-valid-examples-for-multiple-paths.spec.ts @@ -6,7 +6,7 @@ import { Page, TestInfo } from "@playwright/test"; test.describe("Example Generation", () => { test( `Generate examples for '/products' and '/monitor/(id:number)' paths of '${PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_2_PATHS}' for all response codes and methods`, - { tag: ["@examples", "@multiplePathGeneration"] }, + { tag: ["@examples", "@multiplePathGeneration", "@eyes"] }, async ({ page, eyes }, testInfo) => { console.log(`Starting test: ${testInfo.title}`); const examplePage = new ExampleGenerationPage( diff --git a/specs/openapi/generate-valid-examples/inline-1-example.spec.ts b/specs/openapi/generate-valid-examples/inline-1-example.spec.ts deleted file mode 100644 index 1e6d8e4..0000000 --- a/specs/openapi/generate-valid-examples/inline-1-example.spec.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { test, expect } from "../../../utils/eyesFixture"; -import { PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_INLINE_1 } from "../../specNames"; -import { ExampleGenerationPage } from "../../../page-objects/example-generation-page"; -import { - filterExampleNames, - getUpdatedSpecName, - navigateToUpdatedSpec, -} from "../helpers/inline-examples-helper"; - -const FIND_AVAILABLE_PRODUCTS = "findAvailableProducts"; - -test.describe("Inline examples", () => { - test( - `Inline an example for findAvailableProducts 200`, - { tag: ["@examples", "@inlineExamples", "@inline1Example", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - try { - console.log(`Starting test: ${testInfo.title}`); - const examplePage = new ExampleGenerationPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_INLINE_1, - ); - await examplePage.openExampleGenerationTabForSpec( - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_INLINE_1, - ); - - await examplePage.deleteGeneratedExamples(); - - await examplePage.generateAndValidateForPaths([ - { path: FIND_AVAILABLE_PRODUCTS, responseCodes: [200] }, - ]); - - const generatedExampleNames = await examplePage.getGeneratedExampleNames(); - - await examplePage.inlineExamples(); - - const expectedUpdatedSpecName = getUpdatedSpecName(PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_INLINE_1); - - const [actualTitle, actualMessage] = - await examplePage.getDialogTitleAndMessage(); - - expect.soft(actualTitle).toBe("Examples Inline Complete"); - expect - .soft(actualMessage) - .toBe( - `Successfully inlined examples into ${expectedUpdatedSpecName}`, - ); - - await examplePage.closeInlineSuccessDialog("Examples Inline Complete"); - - await test.step("Verify inlined examples appear in the updated spec file", async () => { - const updatedSpecPage = await navigateToUpdatedSpec( - page, - testInfo, - eyes, - expectedUpdatedSpecName, - ); - - await updatedSpecPage.verifyInlinedExamplesInSpec( - filterExampleNames(generatedExampleNames, FIND_AVAILABLE_PRODUCTS, 200), - FIND_AVAILABLE_PRODUCTS, - "get", - 200, - ); - }); - } catch (err) { - expect - .soft(err, `Unexpected error in test: ${testInfo.title}`) - .toBeUndefined(); - } - }, - ); -}); diff --git a/specs/openapi/generate-valid-examples/validate-inlined-examples.spec.ts b/specs/openapi/generate-valid-examples/validate-inlined-examples.spec.ts deleted file mode 100644 index 04578bc..0000000 --- a/specs/openapi/generate-valid-examples/validate-inlined-examples.spec.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { test, expect } from "../../../utils/eyesFixture"; -import { PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED } from "../../specNames"; -import { - filterExampleNames, - generateMoreThenValidateAndInline, - getUpdatedSpecName, - navigateToUpdatedSpec, - setupExampleGenerationPage, - verifyAndCloseInlineSuccessDialog, -} from "../helpers/inline-examples-helper"; - -const FIND_AVAILABLE_PRODUCTS = "findAvailableProducts"; -const PRODUCTS = "products"; - -test.describe("Validate generated spec after inlining GET examples", () => { - test( - "Multiple paths, multiple response codes - Generate, validate, inline and verify updated spec", - { - tag: [ - "@examples", - "@inlineExamples", - "@validateInlinedExamplesForMultiplePaths", - "@eyes", - ], - }, - async ({ page, eyes }, testInfo) => { - const examplePage = await setupExampleGenerationPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED, - [ - { path: FIND_AVAILABLE_PRODUCTS, responseCodes: [200, 400] }, - { path: PRODUCTS, responseCodes: [201, 400] }, - ], - ); - - const generatedExampleNames = await generateMoreThenValidateAndInline( - examplePage, - [ - { path: FIND_AVAILABLE_PRODUCTS, code: 200 }, - { path: FIND_AVAILABLE_PRODUCTS, code: 400 }, - { path: PRODUCTS, code: 201 }, - { path: PRODUCTS, code: 400 }, - ], - ); - - const updatedSpecName = getUpdatedSpecName( - PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED, - ); - await verifyAndCloseInlineSuccessDialog(examplePage, updatedSpecName); - - await test.step("Verify inlined examples appear in the updated spec file", async () => { - const updatedSpecPage = await navigateToUpdatedSpec( - page, - testInfo, - eyes, - updatedSpecName, - ); - - await updatedSpecPage.verifyInlinedExamplesInSpec( - filterExampleNames( - generatedExampleNames, - FIND_AVAILABLE_PRODUCTS, - 200, - ), - FIND_AVAILABLE_PRODUCTS, - "get", - 200, - ); - await updatedSpecPage.verifyInlinedExamplesInSpec( - filterExampleNames( - generatedExampleNames, - FIND_AVAILABLE_PRODUCTS, - 400, - ), - FIND_AVAILABLE_PRODUCTS, - "get", - 400, - ); - await updatedSpecPage.verifyInlinedExamplesInSpec( - filterExampleNames(generatedExampleNames, PRODUCTS, 201), - PRODUCTS, - "post", - 201, - ); - await updatedSpecPage.verifyInlinedExamplesInSpec( - filterExampleNames(generatedExampleNames, PRODUCTS, 400), - PRODUCTS, - "post", - 400, - ); - }); - }, - ); -}); diff --git a/specs/openapi/update-service-spec/backward-compatability-test.spec.ts b/specs/openapi/update-service-spec/backward-compatability-test.spec.ts deleted file mode 100644 index 56772d6..0000000 --- a/specs/openapi/update-service-spec/backward-compatability-test.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { test, expect } from "../../../utils/eyesFixture"; -import { PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY } from "../../specNames"; -import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page"; - -test.describe("API Specification", () => { - test( - "Backward Compatibility Test", - { tag: ["@spec", "@bccTest", "@eyes"] }, - async ({ page, eyes }, testInfo) => { - const configPage = new ServiceSpecConfigPage( - page, - testInfo, - eyes, - PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY, - ); - await test.step(`Go to Spec page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY}'`, async () => { - await configPage.gotoHomeAndOpenSidebar(); - await configPage.sideBar.selectSpec( - PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY, - ); - await configPage.openSpecTab(); - }); - - await test.step("Should detect and display backward compatibility errors", async () => { - await configPage.runBackwardCompatibilityTest(); - const toastText = await configPage.getAlertMessageText(); - expect(toastText).toBe("Changes are backward compatible"); - await configPage.dismissAlert(); - await expect(page.locator("#alert-container")).toBeEmpty(); - }); - }, - ); -}); diff --git a/specs/openapi/update-service-spec/backward-compatibility-test.spec.ts b/specs/openapi/update-service-spec/backward-compatibility-test.spec.ts new file mode 100644 index 0000000..3f12863 --- /dev/null +++ b/specs/openapi/update-service-spec/backward-compatibility-test.spec.ts @@ -0,0 +1,50 @@ +import { test, expect } from "../../../utils/eyesFixture"; +import { PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY } from "../../specNames"; +import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page"; +import { Page } from "playwright/test"; + +test.describe("API Specification", () => { + test( + "Backward Compatibility Test", + { tag: ["@spec", "@bccTest", "@eyes"] }, + async ({ page, eyes }, testInfo) => { + const configPage = await setupConfigPage(page, testInfo, eyes); + + await test.step("Remove summary field from /products endpoint and save", async () => { + await configPage.deleteSpecLinesInEditor( + "summary: Create a new product", + 1, + ); + }); + + await assertDialog(configPage, page); + }, + ); +}); + +async function setupConfigPage(page: Page, testInfo: any, eyes: any) { + const configPage = new ServiceSpecConfigPage( + page, + testInfo, + eyes, + PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY, + ); + await test.step(`Go to Spec page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY}'`, async () => { + await configPage.gotoHomeAndOpenSidebar(); + await configPage.sideBar.selectSpec( + PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY, + ); + await configPage.openSpecTab(); + }); + return configPage; +} + +async function assertDialog(configPage: ServiceSpecConfigPage, page: Page) { + await test.step("Should confirm removal of summary is backward compatible", async () => { + await configPage.runBackwardCompatibilityTest(); + const toastText = await configPage.getAlertMessageText(); + expect(toastText).toBe("Changes are backward compatible"); + await configPage.dismissAlert(); + await expect(page.locator("#alert-container")).toBeEmpty(); + }); +} diff --git a/specs/openapi/update-service-spec/backward-incompatible-test.spec.ts b/specs/openapi/update-service-spec/backward-incompatible-test.spec.ts new file mode 100644 index 0000000..d74bfea --- /dev/null +++ b/specs/openapi/update-service-spec/backward-incompatible-test.spec.ts @@ -0,0 +1,115 @@ +import { test, expect } from "../../../utils/eyesFixture"; +import { PRODUCT_SEARCH_BFF_SPEC_BACKWARD_INCOMPATIBLE } from "../../specNames"; +import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page"; +import { Page } from "@playwright/test"; + +test.describe("API Specification — Backward Incompatibility", () => { + test( + "Adding path parameter to existing endpoint to check backward compatibility", + { tag: ["@spec", "@bccIncompatibleTest", "@eyes"] }, + async ({ page, eyes }, testInfo) => { + const configPage = await setupConfigPage(page, testInfo, eyes); + + await test.step("Change /products to /products/{id} in the editor", async () => { + await configPage.editSpecInEditor(" /products:", " /products/{id}:"); + }); + + await assertBccFailure( + configPage, + "This API exists in the old contract but not in the new contract", + 1, + ); + }, + ); + + test( + "Removing a response status code is a backward incompatible change", + { tag: ["@spec", "@bccIncompatibleTest", "@eyes", "@expected-failure"] }, + async ({ page, eyes }, testInfo) => { + test.fail( + true, + "Error count does not match with acutal error. Needs fixing by dev", + ); + const configPage = await setupConfigPage(page, testInfo, eyes); + + await test.step("Change response status code '201' to '299' under /products in the editor", async () => { + await configPage.editSpecInEditor(" '201':", " '299':"); + }); + + await assertBccFailure( + configPage, + "This API exists in the old contract but not in the new contract", + 2, + ); + }, + ); + + test( + "Making a required parameter optional is a backward incompatible change", + { tag: ["@spec", "@bccIncompatibleTest", "@eyes", "@expected-failure"] }, + async ({ page, eyes }, testInfo) => { + test.fail( + true, + "Error count does not match with acutal error. Needs fixing by dev", + ); + const configPage = await setupConfigPage(page, testInfo, eyes); + + await test.step("Making optional Parameter required", async () => { + await configPage.editSpecInEditor( + " required: false", + " required: true", + ); + }); + + await assertBccFailure( + configPage, + 'New specification expects query param "type" in the request but it is missing from the old specification', + 3, + ); + }, + ); +}); + +async function setupConfigPage(page: Page, testInfo: any, eyes: any) { + const configPage = new ServiceSpecConfigPage( + page, + testInfo, + eyes, + PRODUCT_SEARCH_BFF_SPEC_BACKWARD_INCOMPATIBLE, + ); + await test.step(`Go to Spec page for Service Spec: '${PRODUCT_SEARCH_BFF_SPEC_BACKWARD_INCOMPATIBLE}'`, async () => { + await configPage.gotoHomeAndOpenSidebar(); + await configPage.sideBar.selectSpec( + PRODUCT_SEARCH_BFF_SPEC_BACKWARD_INCOMPATIBLE, + ); + await configPage.openSpecTab(); + }); + return configPage; +} + +async function assertBccFailure( + configPage: ServiceSpecConfigPage, + expectedErrorDetail: string, + expectedErrorCount: number, +) { + await test.step("Run Backward Compatibility test and assert failure toast", async () => { + await configPage.runBackwardCompatibilityTest(); + const toastText = await configPage.getAlertMessageText(); + expect.soft(toastText).toContain("Backward compatibility test failed"); + await configPage.dismissAlert(); + }); + + await test.step("Assert error dropdown heading shows 1 error", async () => { + await configPage.toggleBccErrorSection(true); + const { summary } = await configPage.getBccErrorDetails(); + expect + .soft(summary) + .toContain(`Backward Compatibility found ${expectedErrorCount} error`); + }); + + await test.step("Assert error detail describes the contract mismatch", async () => { + const { details } = await configPage.getBccErrorDetails(); + expect.soft(details.length).toBeGreaterThan(0); + expect.soft(details[0]).toContain(expectedErrorDetail); + }); +} diff --git a/specs/openapi/update-service-spec/fix-spec-typo.spec.ts b/specs/openapi/update-service-spec/fix-spec-typo.spec.ts index b27fa21..8eba378 100644 --- a/specs/openapi/update-service-spec/fix-spec-typo.spec.ts +++ b/specs/openapi/update-service-spec/fix-spec-typo.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "../../../utils/eyesFixture"; +import { test } from "../../../utils/eyesFixture"; import { PRODUCT_SEARCH_BFF_SPEC_FIX_TYPO } from "../../specNames"; import { ServiceSpecConfigPage } from "../../../page-objects/service-spec-config-page"; @@ -15,13 +15,10 @@ test.describe("Fix Spec Typo - Conditional Update", () => { ); await configPage.navigateToSpec(PRODUCT_SEARCH_BFF_SPEC_FIX_TYPO); - await test.step("Typo detected: Fixing /ordres to /orders", async () => { - expect(configPage.specFileContains(" /ordres:")).toBeTruthy(); - - await configPage.editSpecFile(" /ordres:", " /orders:"); - await page.reload(); - await configPage.navigateToSpec(PRODUCT_SEARCH_BFF_SPEC_FIX_TYPO); + await test.step("Typo detected: Fixing /ordres to /orders", async () => { + await configPage.editSpecInEditor(" /ordres:", " /orders:"); + await configPage.clickSaveAfterEdit(); await configPage.verifyEndpointInContractTable("/orders"); }); }, diff --git a/specs/specNames.ts b/specs/specNames.ts index 2d9f6fb..cdb5c04 100644 --- a/specs/specNames.ts +++ b/specs/specNames.ts @@ -15,6 +15,7 @@ export const PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_INLINE_ALL = "product_search_bff_v export const PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_INLINED = "product_search_bff_v5_examples_validate_inlined.yaml"; export const PRODUCT_SEARCH_BFF_SPEC_EXAMPLES_VALIDATE_POST_INLINED = "product_search_bff_v5_examples_validate_post_inlined.yaml"; export const PRODUCT_SEARCH_BFF_SPEC_BACKWARD_COMPATIBILITY = "product_search_bff_v5_backward_compatibility.yaml"; +export const PRODUCT_SEARCH_BFF_SPEC_BACKWARD_INCOMPATIBLE = "product_search_bff_v5_backward_incompatibility.yaml"; export const PRODUCT_SEARCH_BFF_SPEC_FIX_TYPO = "product_search_bff_v5_fix_typo.yaml"; export const PRODUCT_SEARCH_BFF_SPEC_SAVE_INVALID_SPEC = "product_search_bff_v5_save_invalid_spec.yaml"; export const PRODUCT_SEARCH_BFF_SPEC_UPDATE = "product_search_bff_v5_update.yaml";