diff --git a/ci/test/ui-tests.sh b/ci/test/ui-tests.sh index 7cb919c78..a5a593793 100755 --- a/ci/test/ui-tests.sh +++ b/ci/test/ui-tests.sh @@ -9,7 +9,6 @@ set -e # stop_processes # DISPLAY: ":99" in .gitlab-ci.yml? # not sure if relevant -#xvfb-run tester ui # cleanup before recording in local folder # which is shard with normal user recordings and @@ -25,10 +24,13 @@ set -e # # rm -rf /tmp/codetracer # rm -rf /dev/shm/codetracer -# pushd ui-tests; -# xvfb-run npx playwright test -# popd; +echo "========================" +# echo "RUNNING ui e2e tests" -# stop_processes +# TODO nix-shell --command "xvfb-run just test-e2e" + +echo TODO -echo TODO +echo "========================" + +# stop_processes diff --git a/justfile b/justfile index d2b0e3b4b..d8c7d4f00 100644 --- a/justfile +++ b/justfile @@ -283,3 +283,10 @@ ls-trace-folder-for-id trace_id: # end of trace folder helpers # =========================== + +# ==== +# e2e helpers + +test-e2e: + cd ${CODETRACER_REPO_ROOT_PATH}/ui-tests && \ + env CODETRACER_OPEN_DEV_TOOLS=0 npx playwright test --reporter=list --workers=1 diff --git a/nix/shells/main.nix b/nix/shells/main.nix index e5ec0fbbd..e8e7bb219 100644 --- a/nix/shells/main.nix +++ b/nix/shells/main.nix @@ -100,6 +100,9 @@ in mdbook mdbook-alerts + # github CLI + gh + # cachix support cachix @@ -230,6 +233,7 @@ in export NIX_CODETRACER_EXE_DIR=$ROOT_PATH/src/build-debug/ export LINKS_PATH_DIR=$ROOT_PATH/src/build-debug/ + export CODETRACER_REPO_ROOT_PATH=$ROOT_PATH export PATH=$PATH:$PWD/src/build-debug/bin export PATH=$PATH:$ROOT_PATH/node_modules/.bin/ export CODETRACER_OPEN_DEV_TOOLS=1 diff --git a/src/ct/trace/record.nim b/src/ct/trace/record.nim index 49c90a40a..f7d51eb60 100644 --- a/src/ct/trace/record.nim +++ b/src/ct/trace/record.nim @@ -385,6 +385,9 @@ proc record*(args: seq[string]): Trace = # registerRecordInReportFile(reportFile, trace, outputPath) putEnv("CODETRACER_RECORDING", "") + let inUiTest = getEnv("CODETRACER_IN_UI_TEST", "") == "1" + if inUiTest: + echo fmt"> codetracer: finished with trace id: {traceId}" return trace except CatchableError as e: if sessionId != -1: diff --git a/src/frontend/index.nim b/src/frontend/index.nim index 4006a8b5b..3b8579c52 100644 --- a/src/frontend/index.nim +++ b/src/frontend/index.nim @@ -31,7 +31,7 @@ proc isCtInstalled: bool proc onClose(e: js) = - if data.config.test: + if not data.config.isNil and data.config.test: discard elif not close: # TODO refactor to use just `client.send` @@ -49,10 +49,14 @@ proc onClose(e: js) = # --caller-pid # # eventually if needed --backend-socket-host proc parseArgs = + # echo "parseArgs" + data.startOptions.screen = true data.startOptions.loading = false data.startOptions.record = false + data.startOptions.folder = electronprocess.cwd() + if electronProcess.env.hasKey(cstring"CODETRACER_TRACE_ID"): data.startOptions.traceID = electronProcess.env[cstring"CODETRACER_TRACE_ID"].parseJSInt data.startOptions.inTest = electronProcess.env[cstring"CODETRACER_TEST"] == cstring"1" @@ -61,7 +65,7 @@ proc parseArgs = else: discard - data.startOptions.folder = electronprocess.cwd() + if electronProcess.env.hasKey(cstring"CODETRACER_TEST_STRATEGY"): data.startOptions.rawTestStrategy = electronProcess.env[cstring"CODETRACER_TEST_STRATEGY"] diff --git a/ui-tests/programs/noir_example/Nargo.toml b/ui-tests/programs/noir_example/Nargo.toml new file mode 100644 index 000000000..389de66b2 --- /dev/null +++ b/ui-tests/programs/noir_example/Nargo.toml @@ -0,0 +1,6 @@ +[package] +name = "noir_example" +type = "bin" +authors = [""] + +[dependencies] \ No newline at end of file diff --git a/ui-tests/programs/noir_example/Prover.toml b/ui-tests/programs/noir_example/Prover.toml new file mode 100644 index 000000000..4c144083f --- /dev/null +++ b/ui-tests/programs/noir_example/Prover.toml @@ -0,0 +1,2 @@ +x = 0 +y = 1 diff --git a/ui-tests/programs/noir_example/src/main.nr b/ui-tests/programs/noir_example/src/main.nr new file mode 100644 index 000000000..4a8320037 --- /dev/null +++ b/ui-tests/programs/noir_example/src/main.nr @@ -0,0 +1,28 @@ +// fn run(x: Field, y: [8; Field]) -> Field { +// let length = 8; +// for item in 0 .. length { +// let item = y[i]; +// if check(x, item) { +// print("found"); +// return x; +// } else { + +// } +// } +// print(">"); +// } +// } + +fn main(x: Field, y: pub Field) { + println(f"x {x}"); // marker: PRINT_X_LINE + println(f"y {y}"); // marker: PRINT_Y_LINE + assert(x != y); // marker: ASSERT_LINE +} + +// #[test] +// fn test_main() { +// main(1, 2); + +// // Uncomment to make test fail +// // main(1, 1); +// } diff --git a/ui-tests/shell.nix b/ui-tests/shell.nix new file mode 100644 index 000000000..5323b8367 --- /dev/null +++ b/ui-tests/shell.nix @@ -0,0 +1,20 @@ +# copied and adapted from https://nix.dev/tutorials/first-steps/declarative-shell.html +# however, using unstable like flake.nix +let + + nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-unstable"; + + pkgs = import nixpkgs { config = {}; overlays = []; }; + +in + + +pkgs.mkShellNoCC { + + packages = with pkgs; [ + + xvfb-run + + ]; + +} \ No newline at end of file diff --git a/ui-tests/tests/lib/ct_helpers.ts b/ui-tests/tests/lib/ct_helpers.ts index 6d6434570..b2280d6af 100644 --- a/ui-tests/tests/lib/ct_helpers.ts +++ b/ui-tests/tests/lib/ct_helpers.ts @@ -8,7 +8,6 @@ import * as path from "node:path"; import * as childProcess from "node:child_process"; import * as process from "node:process"; -import * as fs from "node:fs"; import { test, type Page } from "@playwright/test"; import { _electron, chromium } from "playwright"; @@ -49,44 +48,49 @@ export function debugCodetracer(name: string, langExtension: string): void { } export function getTestProgramNameFromPath(filePath: string): string { - const relativePath = path.relative("programs", filePath); + if (filePath.startsWith("noir_")) { + return path.basename(filePath); + } else { + const relativePath = path.relative("programs", filePath); - // Split the relative path into segments - const pathSegments = relativePath.split(path.sep); + // Split the relative path into segments + const pathSegments = relativePath.split(path.sep); - // The first segment is the folder inside 'programs' - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - const folderName = pathSegments[0]; - return folderName; + // The first segment is the folder inside 'programs' + // eslint-disable-next-line @typescript-eslint/no-magic-numbers + const folderName = pathSegments[0]; + return folderName; + } } -export function codeTracerRun(relativeSourceFilePath: string): void { +export function ctRun(relativeSourcePath: string): void { test.beforeAll(async () => { setupLdLibraryPath(); - const sourceFilePath = path.join(testProgramsPath, relativeSourceFilePath); - const sourceFileName = path.basename( - relativeSourceFilePath, - path.extname(relativeSourceFilePath), - ); + const sourcePath = path.join(testProgramsPath, relativeSourcePath); + // const sourceFileName = path.basename( + // relativeSourceFilePath, + // path.extname(relativeSourceFilePath), + // ); // const sourceFileExtension = path.extname(sourceFilePath); - const programName = getTestProgramNameFromPath(sourceFilePath); + const programName = getTestProgramNameFromPath(relativeSourcePath); - fs.mkdirSync(testBinariesPath, { recursive: true }); + // fs.mkdirSync(testBinariesPath, { recursive: true }); - const binaryFileName = `${programName}__${sourceFileName}`; - const binaryFilePath = path.join(testBinariesPath, binaryFileName); + // const binaryFileName = `${programName}__${sourceFileName}`; + // const binaryFilePath = path.join(testBinariesPath, binaryFileName); - buildTestProgram(sourceFilePath, binaryFilePath); + // TODO: if rr-backend? + // buildTestProgram(sourceFilePath, binaryFilePath); - const traceId = recordTestProgram(binaryFilePath); + const traceId = recordTestProgram(sourcePath); const inBrowser = process.env.CODETRACER_TEST_IN_BROWSER === "1"; if (!inBrowser) { const runPid = 1; - await replayCodetracerInElectron(binaryFileName, traceId, runPid); + await replayCodetracerInElectron(programName, traceId, runPid); } else { - await replayCodetracerInBrowser(binaryFileName, traceId); + await replayCodetracerInBrowser(programName, traceId); } }); } @@ -120,19 +124,23 @@ function setupLdLibraryPath(): void { } async function replayCodetracerInElectron( - binaryFileName: string, + programName: string, traceId: number, runPid: number, ): Promise { // not clear, but maybe possible to directly augment the playwright test report? // test.info().annotations.push({type: 'something', description: `# starting codetracer for ${pattern}`}); - console.log(`# replay codetracer rr/gdb core process for ${binaryFileName}`); + console.log(`# replay codetracer rr/gdb core process for ${programName}`); const ctProcess = childProcess.spawn( codetracerPath, - ["start_core", binaryFileName, runPid.toString()], + ["start_core", `${traceId}`, runPid.toString()], { cwd: codetracerInstallDir }, ); + // ctProcess.stdout.setEncoding("utf8"); + // ctProcess.stdout.on("data", console.log); + // ctProcess.stderr.setEncoding("utf8"); + // ctProcess.stderr.on("data", console.log); ctProcess.on("close", (code) => { console.log(`child process exited with code ${code}`); @@ -148,11 +156,14 @@ async function replayCodetracerInElectron( process.env.CODETRACER_TRACE_ID = traceId.toString(); process.env.CODETRACER_IN_UI_TEST = "1"; + // console.log(ctProcess); + electronApp = await electron.launch({ executablePath: electronPath, cwd: codetracerInstallDir, args: [indexPath], }); + const firstWindow = await electronApp.firstWindow(); const firstWindowTitle = await firstWindow.title(); @@ -194,7 +205,7 @@ async function replayCodetracerInBrowser( const chromiumBrowser = await chromium.launch({ executablePath: path.join( process.env.PLAYWRIGHT_BROWSERS_PATH, - "chromium-1091", + "chromium-1134", "chrome-linux", "chrome", ), @@ -254,21 +265,26 @@ function buildTestProgram( console.log(`# codetracer built ${outputBinaryPath} succesfully`); } -function recordTestProgram(outputBinaryPath: string): number { +function recordTestProgram(recordArg: string): number { + process.env.CODETRACER_IN_UI_TEST = "1"; + // non-obvious options! // stdio: 'pipe', encoding: 'utf8' found form // https://stackoverflow.com/a/35690273/438099 const ctProcess = childProcess.spawnSync( codetracerPath, - ["record", outputBinaryPath], + ["record", recordArg], { cwd: codetracerInstallDir, stdio: "pipe", encoding: "utf-8", }, ); + // console.log(ctProcess); if (ctProcess.error !== undefined || ctProcess.status !== OK_EXIT_CODE) { - console.log(`ERROR: codetracer record: ${ctProcess.error}`); + console.log( + `ERROR: codetracer record: error: ${ctProcess.error}; status: ${ctProcess.status}`, + ); console.log(ctProcess.stderr); process.exit(ERROR_EXIT_CODE); } @@ -292,7 +308,7 @@ function recordTestProgram(outputBinaryPath: string): number { process.exit(ERROR_EXIT_CODE); } console.log( - `# codetracer recorded a trace for ${outputBinaryPath} with trace id ${maybeTraceId} succesfully`, + `# codetracer recorded a trace for ${recordArg} with trace id ${maybeTraceId} succesfully`, ); return maybeTraceId; } @@ -328,6 +344,10 @@ export async function readyOnEntryTest(): Promise { await page.locator(".location-path").click(); } +export async function loadedEventLog(): Promise { + await page.locator(".data-tables-footer-rows-count").click(); +} + export class CodetracerTestError extends Error { constructor(msg: string) { super(msg); diff --git a/ui-tests/tests/noir_example.spec.ts b/ui-tests/tests/noir_example.spec.ts new file mode 100644 index 000000000..19eb7542c --- /dev/null +++ b/ui-tests/tests/noir_example.spec.ts @@ -0,0 +1,59 @@ +import { test, expect } from "@playwright/test"; +import { + window, + page, + // wait, + // debugCodetracer, + readyOnEntryTest as readyOnEntry, + loadedEventLog, + ctRun, +} from "./lib/ct_helpers"; +import { StatusBar } from "./page_objects/status_bar"; + +ctRun("noir_example/"); + +const ENTRY_LINE = 17; + +test("we can access the browser window, not just dev tools", async () => { + const title = await window.title(); + expect(title).toBe("CodeTracer"); + await window.focus("div"); +}); + +test("correct entry status path/line", async () => { + await readyOnEntry(); + + const statusBar = new StatusBar(page, page.locator(".status-bar")); + const simpleLocation = await statusBar.location(); + expect(simpleLocation.path.endsWith("main.nr")).toBeTruthy(); + expect(simpleLocation.line).toBe(ENTRY_LINE); +}); + +// TODO: run tests serially if in the same instance (?) +// for now we're passing `--workers=1` to prevent parallelism: is this sufficient? +// maybe we should use groups instead, as we might want parallelism in the future +// (on the other hand, codetracer/backend itself might use parallelism, so we wouldn't want to parallelize tests in all cases) + +test("expected event count", async () => { + await loadedEventLog(); + + const raw = await page.$eval( + ".data-tables-footer-rows-count", + (el) => el.textContent ?? "", + ); + + expect(raw.endsWith("2")).toBeTruthy(); + expect(raw.endsWith("1")).toBeFalsy(); +}); + +test("state panel loaded initially", async () => { + await readyOnEntry(); + await expect(page.locator("#code-state-line-0")).toContainText( + "17 | println(", + ); +}); + +// TODO +// test("state panel after jump to end of run function", async () => { +// await readyOnEntry(); +// }); diff --git a/ui-tests/tests/rs_rr_gdb.spec.ts b/ui-tests/tests/rs_rr_gdb.spec.ts deleted file mode 100644 index e510f175e..000000000 --- a/ui-tests/tests/rs_rr_gdb.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { - window, - page, - // debugCodetracer, - readyOnEntryTest as readyOnEntry, - codeTracerRun, -} from "./lib/ct_helpers"; -import { StatusBar } from "./page_objects/status_bar"; - -// debugCodetracer("rr_gdb", "rs"); -codeTracerRun("/rs_rr_gdb/rr_gdb.rs"); - -const RR_GDB_ENTRY_LINE = 198; - -test("we can access the browser window, not just dev tools", async () => { - const title = await window.title(); - expect(title).toBe("editor"); - await window.focus("div"); -}); - -test("correct entry status path/line", async () => { - // TODO: actually wait for run-to-entry or status-bar appearing - // or fail after a timeout - // wait(ms) is usually flakey and bad - await readyOnEntry(); - // const waitingTimeBeforeEntryIsReadyInMs = 2_500; - // await wait(waitingTimeBeforeEntryIsReadyInMs); - - const statusBar = new StatusBar(page, page.locator(".status-bar")); - const simpleLocation = await statusBar.location(); - expect(simpleLocation.path.endsWith("rr_gdb.rs")).toBeTruthy(); - expect(simpleLocation.line).toBe(RR_GDB_ENTRY_LINE); -}); - -test("expected event count", async () => { - await readyOnEntry(); - - const raw = await page.$eval( - ".data-tables-footer-rows-count", - (el) => el.textContent ?? "", - ); - - //await wait(3000); - - expect(raw.endsWith("15")).toBeTruthy(); - // let a = 155; - // console.log(a); - // expect((raw ?? "").endsWith("16")).toBeFalsy(); - expect(raw.endsWith("16")).toBeFalsy(); - // expect(raw != null && raw.endsWith("15")).toBeTruthy(); - // expect(simpleLocation.line).toBe(RR_GDB_ENTRY_LINE); -}); - -test("state panel loaded initially", async () => { - await readyOnEntry(); - await expect(page.locator("#code-state-line-0")).toContainText( - "198 | fn main() {", - ); -}); - -test("state panel after jump to end of run function", async () => { - await readyOnEntry(); -});