diff --git a/src/appdistribution/client.ts b/src/appdistribution/client.ts index 263fbec051a..12dfd0d9b49 100644 --- a/src/appdistribution/client.ts +++ b/src/appdistribution/client.ts @@ -279,6 +279,7 @@ export class AppDistributionClient { aiInstruction?: AIInstruction, loginCredential?: LoginCredential, testCaseName?: string, + displayName?: string, ): Promise { try { const response = await this.appDistroV1AlphaClient.request({ @@ -289,6 +290,7 @@ export class AppDistributionClient { loginCredential, testCase: testCaseName, aiInstructions: aiInstruction, + displayName: displayName, }, }); return response.body; diff --git a/src/appdistribution/types.ts b/src/appdistribution/types.ts index fd8d1c1e25f..24e53778641 100644 --- a/src/appdistribution/types.ts +++ b/src/appdistribution/types.ts @@ -117,6 +117,7 @@ export interface ReleaseTest { loginCredential?: LoginCredential; testCase?: string; aiInstructions?: AIInstruction; + displayName?: string; } export interface AIInstruction { diff --git a/src/apptesting/invokeTests.spec.ts b/src/apptesting/invokeTests.spec.ts index 558f8f2cd9b..55027bc3e8e 100644 --- a/src/apptesting/invokeTests.spec.ts +++ b/src/apptesting/invokeTests.spec.ts @@ -47,7 +47,7 @@ describe("invokeTests", () => { testCase: { startUri: "https://www.example.com", displayName: "testName1", - instructions: { steps: [{ goal: "test this app", hint: "try clicking the button" }] }, + steps: [{ goal: "test this app", hint: "try clicking the button" }], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -55,7 +55,7 @@ describe("invokeTests", () => { testCase: { startUri: "https://www.example.com", displayName: "testName2", - instructions: { steps: [{ goal: "retest it", successCriteria: "a dialog appears" }] }, + steps: [{ goal: "retest it", successCriteria: "a dialog appears" }], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -67,14 +67,12 @@ describe("invokeTests", () => { { testCase: { displayName: "testName1", - instructions: { - steps: [ - { - goal: "test this app", - hint: "try clicking the button", - }, - ], - }, + steps: [ + { + goal: "test this app", + hint: "try clicking the button", + }, + ], startUri: "https://www.example.com", }, testExecution: [ @@ -88,14 +86,12 @@ describe("invokeTests", () => { { testCase: { displayName: "testName2", - instructions: { - steps: [ - { - goal: "retest it", - successCriteria: "a dialog appears", - }, - ], - }, + steps: [ + { + goal: "retest it", + successCriteria: "a dialog appears", + }, + ], startUri: "https://www.example.com", }, testExecution: [ diff --git a/src/apptesting/parseTestFiles.spec.ts b/src/apptesting/parseTestFiles.spec.ts index 3a1aa13f9ed..f9022c24e3d 100644 --- a/src/apptesting/parseTestFiles.spec.ts +++ b/src/apptesting/parseTestFiles.spec.ts @@ -31,7 +31,7 @@ describe("parseTestFiles", () => { "my_test.yaml", stringify({ defaultConfig: { route: "/mypage" }, - tests: [{ testName: "my test", steps: [{ goal: "click a button" }] }], + tests: [{ displayName: "my test", steps: [{ goal: "click a button" }] }], }), ); await expect(parseTestFiles(tempdir.name, "foo.com")).to.be.rejectedWith( @@ -45,7 +45,7 @@ describe("parseTestFiles", () => { "my_test.yaml", stringify({ defaultConfig: { route: "/mypage" }, - tests: [{ testName: "my test", steps: [{ goal: "click a button" }] }], + tests: [{ displayName: "my test", steps: [{ goal: "click a button" }] }], }), ); writeFile("my_test2.yaml", "foo"); @@ -53,15 +53,15 @@ describe("parseTestFiles", () => { expect(tests).to.eql([ { testCase: { + id: undefined, + prerequisiteTestCaseId: undefined, displayName: "my test", startUri: "http://www.foo.com/mypage", - instructions: { - steps: [ - { - goal: "click a button", - }, - ], - }, + steps: [ + { + goal: "click a button", + }, + ], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -74,17 +74,17 @@ describe("parseTestFiles", () => { expect(tests).to.eql([ { testCase: { + id: undefined, + prerequisiteTestCaseId: undefined, displayName: "Smoke test", startUri: "http://www.foo.com", - instructions: { - steps: [ - { - goal: "View the provided application", - hint: "No additional actions should be necessary", - successCriteria: "The application should load with no obvious errors", - }, - ], - }, + steps: [ + { + goal: "View the provided application", + hint: "No additional actions should be necessary", + successCriteria: "The application should load with no obvious errors", + }, + ], }, testExecution: [{ config: { browser: Browser.CHROME } }], }, @@ -94,16 +94,16 @@ describe("parseTestFiles", () => { it("parses multiple test case files", async () => { writeFile( "my_test.yaml", - stringify({ tests: [{ testName: "my test", steps: [{ goal: "click a button" }] }] }), + stringify({ tests: [{ displayName: "my test", steps: [{ goal: "click a button" }] }] }), ); writeFile( "my_test2.yaml", stringify({ defaultConfig: { browsers: ["CHROME"] }, tests: [ - { testName: "my second test", steps: [{ goal: "click a button" }] }, + { displayName: "my second test", steps: [{ goal: "click a button" }] }, { - testName: "my third test", + displayName: "my third test", testConfig: { browsers: ["firefox"], route: "/mypage" }, steps: [{ goal: "type something" }], }, @@ -115,44 +115,44 @@ describe("parseTestFiles", () => { expect(tests).to.eql([ { testCase: { + id: undefined, + prerequisiteTestCaseId: undefined, displayName: "my test", startUri: "https://www.foo.com", - instructions: { - steps: [ - { - goal: "click a button", - }, - ], - }, + steps: [ + { + goal: "click a button", + }, + ], }, testExecution: [{ config: { browser: "CHROME" } }], }, { testCase: { + id: undefined, + prerequisiteTestCaseId: undefined, displayName: "my second test", startUri: "https://www.foo.com", - instructions: { - steps: [ - { - goal: "click a button", - }, - ], - }, + steps: [ + { + goal: "click a button", + }, + ], }, testExecution: [{ config: { browser: "CHROME" } }], }, { testCase: { + id: undefined, + prerequisiteTestCaseId: undefined, displayName: "my third test", startUri: "https://www.foo.com/mypage", - instructions: { - steps: [ - { - goal: "type something", - }, - ], - }, + steps: [ + { + goal: "type something", + }, + ], }, testExecution: [{ config: { browser: "firefox" } }], }, @@ -161,10 +161,10 @@ describe("parseTestFiles", () => { }); describe("filtering", () => { - function createBasicTest(testNames: string[]) { + function createBasicTest(displayNames: string[]) { return stringify({ - tests: testNames.map((testName) => ({ - testName, + tests: displayNames.map((displayName) => ({ + displayName, steps: [{ goal: "do something" }], })), }); @@ -204,4 +204,167 @@ describe("parseTestFiles", () => { expect(await getTestCaseNames("a$", "xx")).to.eql(["axx"]); }); }); + + describe("prerequisite test cases", () => { + it("merges the steps from the prerequisite test case", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + id: "my-first-test", + displayName: "my first test", + steps: [{ goal: "do something first" }], + }, + { + displayName: "my second test", + prerequisiteTestCaseId: "my-first-test", + steps: [{ goal: "do something second" }], + }, + ], + }), + ); + + const tests = await parseTestFiles(tempdir.name, "https://www.foo.com"); + expect(tests.length).to.equal(2); + const secondTest = tests[1]; + expect(secondTest.testCase.steps).to.eql([ + { goal: "do something first" }, + { goal: "do something second" }, + ]); + }); + + it("throws an error for a non-existent prerequisite test case", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + displayName: "my second test", + prerequisiteTestCaseId: "my-first-test", + steps: [{ goal: "do something second" }], + }, + ], + }), + ); + + await expect(parseTestFiles(tempdir.name, "https://www.foo.com")).to.be.rejectedWith( + FirebaseError, + "Invalid prerequisiteTestCaseId. There is no test case with id my-first-test", + ); + }); + + it("handles an undefined prerequisite test case id", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + displayName: "my test", + steps: [{ goal: "do something" }], + }, + ], + }), + ); + + const tests = await parseTestFiles(tempdir.name, "https://www.foo.com"); + expect(tests.length).to.equal(1); + expect(tests[0].testCase.steps).to.eql([{ goal: "do something" }]); + }); + + it("works correctly with filtering", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + id: "my-first-test", + displayName: "my first test", + steps: [{ goal: "do something first" }], + }, + { + displayName: "my second test", + prerequisiteTestCaseId: "my-first-test", + steps: [{ goal: "do something second" }], + }, + ], + }), + ); + + const tests = await parseTestFiles( + tempdir.name, + "https://www.foo.com", + /* filePattern= */ "", + /* namePattern= */ "my second test", + ); + expect(tests.length).to.equal(1); + const secondTest = tests[0]; + expect(secondTest.testCase.steps).to.eql([ + { goal: "do something first" }, + { goal: "do something second" }, + ]); + }); + + it("works correctly with multiple levels of prerequisites", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + id: "my-first-test", + displayName: "my first test", + steps: [{ goal: "do something first" }], + }, + { + id: "my-second-test", + displayName: "my second test", + prerequisiteTestCaseId: "my-first-test", + steps: [{ goal: "do something second" }], + }, + { + displayName: "my third test", + prerequisiteTestCaseId: "my-second-test", + steps: [{ goal: "do something third" }], + }, + ], + }), + ); + + const tests = await parseTestFiles(tempdir.name, "https://www.foo.com"); + expect(tests.length).to.equal(3); + const thirdTest = tests[2]; + expect(thirdTest.testCase.steps).to.eql([ + { goal: "do something first" }, + { goal: "do something second" }, + { goal: "do something third" }, + ]); + }); + + it("throws error if there is a circular depedency", async () => { + writeFile( + "my_test.yaml", + stringify({ + tests: [ + { + id: "my-first-test", + displayName: "my first test", + prerequisiteTestCaseId: "my-second-test", + steps: [{ goal: "do something first" }], + }, + { + id: "my-second-test", + displayName: "my second test", + prerequisiteTestCaseId: "my-first-test", + steps: [{ goal: "do something second" }], + }, + ], + }), + ); + + await expect(parseTestFiles(tempdir.name, "https://www.foo.com")).to.be.rejectedWith( + FirebaseError, + "Detected a cycle in prerequisite test cases.", + ); + }); + }); }); diff --git a/src/apptesting/parseTestFiles.ts b/src/apptesting/parseTestFiles.ts index f366ed4af03..ba0e94060b0 100644 --- a/src/apptesting/parseTestFiles.ts +++ b/src/apptesting/parseTestFiles.ts @@ -1,76 +1,149 @@ import { dirExistsSync, fileExistsSync, listFiles } from "../fsutils"; import { join } from "path"; import { logger } from "../logger"; -import { Browser, TestCaseInvocation } from "./types"; +import { Browser, TestCaseInvocation, TestStep } from "./types"; import { readFileFromDirectory, wrappedSafeLoad } from "../utils"; import { FirebaseError, getErrMsg, getError } from "../error"; -function createFilter(pattern?: string) { - const regex = pattern ? new RegExp(pattern) : undefined; - return (s: string) => !regex || regex.test(s); -} - export async function parseTestFiles( dir: string, - targetUri: string, + targetUri?: string, filePattern?: string, namePattern?: string, ): Promise { - try { - new URL(targetUri); - } catch (ex) { - const errMsg = "Invalid URL" + (targetUri.startsWith("http") ? "" : " (must include protocol)"); - throw new FirebaseError(errMsg, { original: getError(ex) }); + if (targetUri) { + try { + new URL(targetUri); + } catch (ex) { + const errMsg = + "Invalid URL" + (targetUri.startsWith("http") ? "" : " (must include protocol)"); + throw new FirebaseError(errMsg, { original: getError(ex) }); + } } + + const files = await parseTestFilesRecursive({ testDir: dir, targetUri }); + const idToInvocation = files + .flatMap((file) => file.invocations) + .reduce( + (accumulator, invocation) => { + if (invocation.testCase.id) { + accumulator[invocation.testCase.id] = invocation; + } + return accumulator; + }, + {} as Record, + ); + const fileFilterFn = createFilter(filePattern); const nameFilterFn = createFilter(namePattern); + const filteredInvocations = files + .filter((file) => fileFilterFn(file.path)) + .flatMap((file) => file.invocations) + .filter((invocation) => nameFilterFn(invocation.testCase.displayName)); + + return filteredInvocations.map((invocation) => { + let prerequisiteTestCaseId = invocation.testCase.prerequisiteTestCaseId; + if (prerequisiteTestCaseId === undefined) { + return invocation; + } - async function parseTestFilesRecursive(testDir: string): Promise { - const items = listFiles(testDir); - const results = []; - for (const item of items) { - const path = join(testDir, item); - if (dirExistsSync(path)) { - results.push(...(await parseTestFilesRecursive(path))); - } else if (fileFilterFn(path) && fileExistsSync(path)) { - try { - const file = await readFileFromDirectory(testDir, item); - const parsedFile = wrappedSafeLoad(file.source); - const tests = parsedFile.tests; - const defaultConfig = parsedFile.defaultConfig; - if (!tests || !tests.length) { - logger.info(`No tests found in ${path}. Ignoring.`); - continue; - } - for (const rawTestDef of parsedFile.tests) { - if (!nameFilterFn(rawTestDef.testName)) continue; - const testDef = toTestDef(rawTestDef, targetUri, defaultConfig); - results.push(testDef); - } - } catch (ex) { - const errMsg = getErrMsg(ex); - const errDetails = errMsg ? `Error details: \n${errMsg}` : ""; - logger.info(`Unable to parse test file ${path}. Ignoring.${errDetails}`); + const prerequisiteSteps: TestStep[] = []; + const previousTestCaseIds = new Set(); + while (prerequisiteTestCaseId) { + if (previousTestCaseIds.has(prerequisiteTestCaseId)) { + throw new FirebaseError(`Detected a cycle in prerequisite test cases.`); + } + previousTestCaseIds.add(prerequisiteTestCaseId); + const prerequisiteTestCaseInvocation: TestCaseInvocation | undefined = + idToInvocation[prerequisiteTestCaseId]; + if (prerequisiteTestCaseInvocation === undefined) { + throw new FirebaseError( + `Invalid prerequisiteTestCaseId. There is no test case with id ${prerequisiteTestCaseId}`, + ); + } + prerequisiteSteps.unshift(...prerequisiteTestCaseInvocation.testCase.steps); + prerequisiteTestCaseId = prerequisiteTestCaseInvocation.testCase.prerequisiteTestCaseId; + } + + return { + ...invocation, + testCase: { + ...invocation.testCase, + steps: prerequisiteSteps.concat(invocation.testCase.steps), + }, + }; + }); +} + +function createFilter(pattern?: string) { + const regex = pattern ? new RegExp(pattern) : undefined; + return (s: string) => !regex || regex.test(s); +} + +interface TestCaseFile { + path: string; + invocations: TestCaseInvocation[]; +} + +async function parseTestFilesRecursive(params: { + testDir: string; + targetUri?: string; +}): Promise { + const testDir = params.testDir; + const targetUri = params.targetUri; + const items = listFiles(testDir); + const results = []; + for (const item of items) { + const path = join(testDir, item); + if (dirExistsSync(path)) { + results.push(...(await parseTestFilesRecursive({ testDir: path, targetUri }))); + } else if (fileExistsSync(path)) { + try { + const file = await readFileFromDirectory(testDir, item); + logger.debug(`Read the file ${file.source}.`); + const parsedFile = wrappedSafeLoad(file.source); + logger.debug(`Parsed the file.`); + const tests = parsedFile.tests; + logger.debug(`There are ${tests.length} tests.`); + const defaultConfig = parsedFile.defaultConfig; + if (!tests || !tests.length) { + logger.debug(`No tests found in ${path}. Ignoring.`); continue; } + const invocations = []; + for (const rawTestDef of tests) { + const invocation = toTestCaseInvocation(rawTestDef, targetUri, defaultConfig); + invocations.push(invocation); + } + results.push({ path, invocations: invocations }); + } catch (ex) { + const errMsg = getErrMsg(ex); + const errDetails = errMsg ? `Error details: \n${errMsg}` : ""; + logger.debug(`Unable to parse test file ${path}. Ignoring.${errDetails}`); + continue; } } - return results; } - return parseTestFilesRecursive(dir); + return results; } -function toTestDef(testDef: any, targetUri: string, defaultConfig: any): TestCaseInvocation { +function toTestCaseInvocation( + testDef: any, + targetUri: any, + defaultConfig: any, +): TestCaseInvocation { const steps = testDef.steps ?? []; const route = testDef.testConfig?.route ?? defaultConfig?.route ?? ""; const browsers: Browser[] = testDef.testConfig?.browsers ?? defaultConfig?.browsers ?? [Browser.CHROME]; return { testCase: { + id: testDef.id, + prerequisiteTestCaseId: testDef.prerequisiteTestCaseId, startUri: targetUri + route, - displayName: testDef.testName, - instructions: { steps }, + displayName: testDef.displayName, + steps: steps, }, testExecution: browsers.map((browser) => ({ config: { browser } })), }; diff --git a/src/apptesting/types.ts b/src/apptesting/types.ts index fc1fefca1df..e0fbe8d618b 100644 --- a/src/apptesting/types.ts +++ b/src/apptesting/types.ts @@ -57,13 +57,11 @@ export interface TestExecutionResult { } export interface TestCase { - startUri: string; + id?: string; + startUri?: string; displayName: string; - instructions: Instructions; -} - -export interface Instructions { steps: TestStep[]; + prerequisiteTestCaseId?: string; } export interface InvokeTestCasesRequest { diff --git a/src/commands/apptesting.ts b/src/commands/apptesting.ts new file mode 100644 index 00000000000..8894ab7af5a --- /dev/null +++ b/src/commands/apptesting.ts @@ -0,0 +1,124 @@ +import { requireAuth } from "../requireAuth"; +import { Command } from "../command"; +import { logger } from "../logger"; +import * as clc from "colorette"; +import { parseTestFiles } from "../apptesting/parseTestFiles"; +import * as ora from "ora"; +import { TestCaseInvocation } from "../apptesting/types"; +import { FirebaseError, getError } from "../error"; +import { marked } from "marked"; +import { AppDistributionClient } from "../appdistribution/client"; +import { Distribution, upload } from "../appdistribution/distribution"; +import { AIInstruction, ReleaseTest, TestDevice } from "../appdistribution/types"; +import { getAppName, parseTestDevices } from "../appdistribution/options-parser-util"; + +// TODO rothbutter add ability to specify devices +const defaultDevices = [ + { + model: "MediumPhone.arm", + version: "30", + locale: "en_US", + orientation: "portrait", + }, +]; + +export const command = new Command("apptesting:execute ") + .description("Run mobile automated tests written in natural language driven by AI") + .option( + "--app ", + "The app id of your Firebase web app. Optional if the project contains exactly one web app.", + ) + .option( + "--test-file-pattern ", + "Test file pattern. Only tests contained in files that match this pattern will be executed.", + ) + .option( + "--test-name-pattern ", + "Test name pattern. Only tests with names that match this pattern will be executed.", + ) + .option("--test-dir ", "Directory where tests can be found.") + .option( + "--test-devices ", + "semicolon-separated list of devices to run automated tests on, in the format 'model=,version=,locale=,orientation='. Run 'gcloud firebase test android|ios models list' to see available devices. Note: This feature is in beta.", + ) + .option( + "--test-devices-file ", + "path to file containing a list of semicolon- or newline-separated devices to run automated tests on, in the format 'model=,version=,locale=,orientation='. Run 'gcloud firebase test android|ios models list' to see available devices. Note: This feature is in beta.", + ) + .before(requireAuth) + .action(async (target: string, options: any) => { + const appName = getAppName(options); + + const testDir = options.testDir || "tests"; + const tests = await parseTestFiles( + testDir, + undefined, + options.testFilePattern, + options.testNamePattern, + ); + const testDevices = parseTestDevices(options.testDevices, options.testDevicesFile); + + if (!tests.length) { + throw new FirebaseError("No tests found"); + } + + const invokeSpinner = ora("Requesting test execution"); + + let testInvocations; + let releaseId; + try { + const client = new AppDistributionClient(); + releaseId = await upload(client, appName, new Distribution(target)); + + invokeSpinner.start(); + testInvocations = await invokeMataTests( + client, + releaseId, + tests, + !testDevices.length ? defaultDevices : testDevices, + ); + invokeSpinner.text = "Test execution requested"; + invokeSpinner.succeed(); + } catch (ex) { + invokeSpinner.fail("Failed to request test execution"); + throw ex; + } + + logger.info( + clc.bold(`\n${clc.white("===")} Running ${pluralizeTests(testInvocations.length)}`), + ); + logger.info(await marked(`View progress and results in the Firebase Console`)); + }); + +function pluralizeTests(numTests: number) { + return `${numTests} test${numTests === 1 ? "" : "s"}`; +} + +async function invokeMataTests( + client: AppDistributionClient, + releaseName: string, + testDefs: TestCaseInvocation[], + devices: TestDevice[], +) { + try { + const testInvocations: ReleaseTest[] = []; + for (const testDef of testDefs) { + const aiInstruction: AIInstruction = { + steps: testDef.testCase.steps, + }; + testInvocations.push( + await client.createReleaseTest( + releaseName, + devices, + aiInstruction, + undefined, + undefined, + testDef.testCase.displayName, + ), + ); + } + return testInvocations; + } catch (err: unknown) { + throw new FirebaseError("Test invocation failed", { original: getError(err) }); + } +} diff --git a/src/commands/index.ts b/src/commands/index.ts index ad89eff7136..8bd68688471 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -263,6 +263,7 @@ export function load(client: CLIClient): CLIClient { client.use = loadCommand("use"); if (experiments.isEnabled("apptesting")) { client.apptesting = {}; + client.apptesting.execute = loadCommand("apptesting"); client.apptesting.wata = loadCommand("apptesting-wata"); } diff --git a/templates/init/apptesting/smoke_test.yaml b/templates/init/apptesting/smoke_test.yaml index 02819c0964f..7668b5f112c 100644 --- a/templates/init/apptesting/smoke_test.yaml +++ b/templates/init/apptesting/smoke_test.yaml @@ -1,5 +1,5 @@ tests: - - testName: Smoke test + - displayName: Smoke test steps: - goal: View the provided application hint: No additional actions should be necessary