diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41f96b40..950d857a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ jobs: build_tip_of_tree_v12: name: Build test (against tip-of-tree core develop) runs-on: ubuntu-latest + if: false steps: - name: Checkout experimentation plugin uses: actions/checkout@v4 @@ -59,8 +60,7 @@ jobs: build: name: Build test (against pinned v12) - # Don't run pinned version checks for PRs. - if: ${{ !github.base_ref }} + if: false runs-on: ubuntu-latest steps: - name: Checkout experimentation plugin @@ -81,6 +81,7 @@ jobs: name: Eslint check timeout-minutes: 5 runs-on: ubuntu-latest + if: false steps: - uses: actions/checkout@v4 @@ -99,6 +100,7 @@ jobs: name: Prettier check timeout-minutes: 5 runs-on: ubuntu-latest + if: false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 711d2f61..3b2cb802 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ permissions: jobs: webdriverio_tests_tip_of_tree_v12: name: WebdriverIO tests (against tip-of-tree core develop) - timeout-minutes: 10 + # timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: @@ -33,7 +33,7 @@ jobs: uses: actions/checkout@v4 with: repository: 'google/blockly' - ref: 'develop' + ref: 'main' path: core-blockly - name: Use Node.js 20.x @@ -60,15 +60,30 @@ jobs: npm link blockly cd .. - - name: Run tests + - name: Build tests + run: | + cd main + npm run wdio:clean + + - name: Run tests (up to 50 times) run: | cd main - npm run test:ci + for i in $(seq 1 50); + do + echo "Attempt $i" + npm run wdio:run:ci + done + + - name: Upload test failure screenshots + if: always() + uses: actions/upload-artifact@v5 + with: + name: test-failure-screenshots + path: main/test/webdriverio/test/failures webdriverio_tests: name: WebdriverIO tests (against pinned v12) - # Don't run pinned version checks for PRs. - if: ${{ !github.base_ref }} + if: false timeout-minutes: 10 runs-on: ${{ matrix.os }} diff --git a/package.json b/package.json index 663b312d..48ef9ab6 100644 --- a/package.json +++ b/package.json @@ -15,16 +15,16 @@ "prepublishOnly": "npm login --registry https://wombat-dressing-room.appspot.com", "start": "blockly-scripts start", "test": "npm run test:mocha && npm run test:wdio", - "test:ci": "npm run test:mocha && npm run test:wdio:ci", + "test:ci": "npm run test:wdio:ci", "test:mocha": "blockly-scripts test", "test:wdio": "npm run wdio:clean && npm run wdio:run", "test:wdio:ci": "npm run wdio:clean && npm run wdio:run:ci", "wdio:build": "npm run wdio:build:app && npm run wdio:build:tests", "wdio:build:app": "cd test/webdriverio && webpack", "wdio:build:tests": "tsc -p ./test/webdriverio/test/tsconfig.json", - "wdio:clean": "cd test/webdriverio/test && rm -rf dist", - "wdio:run": "npm run wdio:build && cd test/webdriverio/test && npx mocha dist", - "wdio:run:ci": "npm run wdio:build && cd test/webdriverio/test && npx mocha --timeout 30000 dist" + "wdio:clean": "cd test/webdriverio/test && rm -rf dist && rm -rf failures", + "wdio:run": "npm run wdio:build && cd test/webdriverio/test && mkdir -p failures && npx mocha dist", + "wdio:run:ci": "npm run wdio:build && cd test/webdriverio/test && mkdir -p failures && npx mocha --timeout 30000 dist" }, "main": "./dist/index.js", "module": "./src/index.js", diff --git a/test/webdriverio/test/actions_test.ts b/test/webdriverio/test/actions_test.ts index 56c5b85b..49be65c9 100644 --- a/test/webdriverio/test/actions_test.ts +++ b/test/webdriverio/test/actions_test.ts @@ -20,6 +20,8 @@ import { sendKeyAndWait, keyRight, contextMenuItems, + checkForFailures, + idle, } from './test_setup.js'; const isDarwin = process.platform === 'darwin'; @@ -96,13 +98,17 @@ suite('Menus test', function () { testFileLocations.MORE_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Menu action via keyboard on block opens menu', async function () { // Navigate to draw_circle_1. await focusOnBlock(this.browser, 'draw_circle_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, Key.Return]); chai.assert.deepEqual( @@ -126,7 +132,7 @@ suite('Menus test', function () { await focusOnBlock(this.browser, 'text_print_1'); await this.browser.keys(Key.ArrowRight); await this.browser.keys([Key.Ctrl, Key.Return]); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.deepEqual( await contextMenuItems(this.browser), @@ -154,9 +160,9 @@ suite('Menus test', function () { await moveToToolboxCategory(this.browser, 'Math'); // Move to flyout. await keyRight(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await rightClickOnFlyoutBlockType(this.browser, 'math_number'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.deepEqual( await contextMenuItems(this.browser), @@ -192,11 +198,11 @@ suite('Menus test', function () { test('Escape key dismisses menu', async function () { await tabNavigateToWorkspace(this.browser); await focusOnBlock(this.browser, 'draw_circle_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await this.browser.keys([Key.Ctrl, Key.Return]); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await this.browser.keys(Key.Escape); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.isTrue( await contextMenuExists(this.browser, 'Duplicate', /* reverse= */ true), @@ -207,9 +213,9 @@ suite('Menus test', function () { test('Clicking workspace dismisses menu', async function () { await tabNavigateToWorkspace(this.browser); await clickBlock(this.browser, 'draw_circle_1', {button: 'right'}); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await focusWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.isTrue( await contextMenuExists(this.browser, 'Duplicate', /* reverse= */ true), diff --git a/test/webdriverio/test/basic_test.ts b/test/webdriverio/test/basic_test.ts index 276f6835..a7fda2ad 100644 --- a/test/webdriverio/test/basic_test.ts +++ b/test/webdriverio/test/basic_test.ts @@ -22,6 +22,8 @@ import { keyRight, keyUp, keyDown, + checkForFailures, + idle, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -37,6 +39,10 @@ suite('Keyboard navigation on Blocks', function () { ); }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Default workspace', async function () { const blockCount = await this.browser.execute(() => { return Blockly.getMainWorkspace().getAllBlocks(false).length; @@ -47,7 +53,7 @@ suite('Keyboard navigation on Blocks', function () { test('Selected block', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser, 14); @@ -58,7 +64,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from statement block selects next block across stacks', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai @@ -68,7 +74,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from statement block selects previous block', async function () { await focusOnBlock(this.browser, 'simple_circle_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyUp(this.browser); chai @@ -78,7 +84,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from parent block selects first child block', async function () { await focusOnBlock(this.browser, 'p5_setup_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai .expect(await getCurrentFocusedBlockId(this.browser)) @@ -87,7 +93,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from child block selects parent block', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyUp(this.browser); chai .expect(await getCurrentFocusedBlockId(this.browser)) @@ -96,7 +102,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from block selects first field', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai @@ -108,7 +114,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from block selects first inline input', async function () { await focusOnBlock(this.browser, 'simple_circle_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai.assert.equal( @@ -119,7 +125,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from inline input selects statement block', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyUp(this.browser); chai.assert.equal( @@ -130,7 +136,7 @@ suite('Keyboard navigation on Blocks', function () { test('Left from first inline input selects block', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -141,7 +147,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from first inline input selects second inline input', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai.assert.equal( @@ -152,7 +158,7 @@ suite('Keyboard navigation on Blocks', function () { test('Left from second inline input selects first inline input', async function () { await focusOnBlock(this.browser, 'math_number_3'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -163,7 +169,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from last inline input selects next block', async function () { await focusOnBlock(this.browser, 'colour_picker_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai @@ -173,7 +179,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from inline input selects next block', async function () { await focusOnBlock(this.browser, 'colour_picker_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai @@ -183,7 +189,7 @@ suite('Keyboard navigation on Blocks', function () { test("Down from inline input selects block's child block", async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai @@ -193,7 +199,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from text block selects shadow block then field', async function () { await focusOnBlock(this.browser, 'text_print_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai.assert.equal(await getCurrentFocusedBlockId(this.browser), 'text_1'); @@ -235,9 +241,13 @@ suite('Keyboard navigation on Fields', function () { ); }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Up from first field selects block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyUp(this.browser); chai.assert.equal( @@ -248,7 +258,7 @@ suite('Keyboard navigation on Fields', function () { test('Left from first field selects block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -259,7 +269,7 @@ suite('Keyboard navigation on Fields', function () { test('Right from first field selects second field', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai @@ -271,7 +281,7 @@ suite('Keyboard navigation on Fields', function () { test('Left from second field selects first field', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'HEIGHT'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyLeft(this.browser); chai @@ -283,7 +293,7 @@ suite('Keyboard navigation on Fields', function () { test('Right from second field selects next block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'HEIGHT'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyRight(this.browser); chai @@ -293,7 +303,7 @@ suite('Keyboard navigation on Fields', function () { test('Down from field selects next block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai @@ -303,7 +313,7 @@ suite('Keyboard navigation on Fields', function () { test("Down from field selects block's child block", async function () { await focusOnBlockField(this.browser, 'controls_repeat_1', 'TIMES'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await keyDown(this.browser); chai @@ -314,7 +324,7 @@ suite('Keyboard navigation on Fields', function () { test('Do not navigate while field editor is open', async function () { // Open a field editor dropdown await focusOnBlockField(this.browser, 'logic_boolean_1', 'BOOL'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to navigate to a different block @@ -327,7 +337,7 @@ suite('Keyboard navigation on Fields', function () { test('Do not reopen field editor when handling enter to make a choice inside the editor', async function () { // Open colour picker await focusOnBlockField(this.browser, 'colour_picker_1', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Move right to pick a new colour. diff --git a/test/webdriverio/test/block_comment_test.ts b/test/webdriverio/test/block_comment_test.ts index f4c50169..008d9058 100644 --- a/test/webdriverio/test/block_comment_test.ts +++ b/test/webdriverio/test/block_comment_test.ts @@ -14,6 +14,7 @@ import { testFileLocations, keyRight, PAUSE_TIME, + checkForFailures, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -34,6 +35,10 @@ suite('Block comment navigation', function () { }); }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Activating a block comment icon focuses the comment', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); await keyRight(this.browser); diff --git a/test/webdriverio/test/clipboard_test.ts b/test/webdriverio/test/clipboard_test.ts index b204a2ee..512c96bd 100644 --- a/test/webdriverio/test/clipboard_test.ts +++ b/test/webdriverio/test/clipboard_test.ts @@ -18,6 +18,8 @@ import { blockIsPresent, getFocusedBlockType, sendKeyAndWait, + checkForFailures, + idle, } from './test_setup.js'; import {Key, KeyAction, PointerAction, WheelAction} from 'webdriverio'; @@ -28,7 +30,11 @@ suite('Clipboard test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Copy and paste while block selected', async function () { @@ -112,7 +118,7 @@ suite('Clipboard test', function () { test('Do not cut block while field editor is open', async function () { // Open a field editor await focusOnBlockField(this.browser, 'draw_circle_1_color', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to cut block while field editor is open @@ -166,7 +172,7 @@ async function performActionWhileDraggingBlock( .move(blockX, blockY), action, ]); - await browser.pause(PAUSE_TIME); + await idle(browser); } /** diff --git a/test/webdriverio/test/delete_test.ts b/test/webdriverio/test/delete_test.ts index a379c273..4d66b310 100644 --- a/test/webdriverio/test/delete_test.ts +++ b/test/webdriverio/test/delete_test.ts @@ -18,6 +18,8 @@ import { sendKeyAndWait, keyRight, focusOnBlockField, + checkForFailures, + idle, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -31,12 +33,16 @@ suite('Deleting Blocks', function () { testFileLocations.NAVIGATION_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Deleting block selects parent block', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'controls_if_2')) @@ -55,7 +61,7 @@ suite('Deleting Blocks', function () { test('Cutting block selects parent block', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'controls_if_2')) @@ -74,7 +80,7 @@ suite('Deleting Blocks', function () { test('Deleting block also deletes children and inputs', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -93,7 +99,7 @@ suite('Deleting Blocks', function () { test('Cutting block also removes children and inputs', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -112,7 +118,7 @@ suite('Deleting Blocks', function () { test('Deleting inline input selects parent block', async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -131,7 +137,7 @@ suite('Deleting Blocks', function () { test('Cutting inline input selects parent block', async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -155,11 +161,11 @@ suite('Deleting Blocks', function () { // We want deleting a block to focus the workspace, whatever that // means at the time. await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // The test workspace doesn't already contain a stranded block, so add one. await moveToToolboxCategory(this.browser, 'Math'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Move to flyout. await keyRight(this.browser); // Select number block. @@ -179,11 +185,11 @@ suite('Deleting Blocks', function () { test('Cutting stranded block selects top block', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // The test workspace doesn't already contain a stranded block, so add one. await moveToToolboxCategory(this.browser, 'Math'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Move to flyout. await keyRight(this.browser); // Select number block. @@ -204,7 +210,7 @@ suite('Deleting Blocks', function () { test('Do not delete block while field editor is open', async function () { // Open a field editor await focusOnBlockField(this.browser, 'colour_picker_1', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to delete block while field editor is open diff --git a/test/webdriverio/test/duplicate_test.ts b/test/webdriverio/test/duplicate_test.ts index 879ef964..db7cc848 100644 --- a/test/webdriverio/test/duplicate_test.ts +++ b/test/webdriverio/test/duplicate_test.ts @@ -15,6 +15,8 @@ import { testFileLocations, testSetup, sendKeyAndWait, + checkForFailures, + idle, } from './test_setup.js'; suite('Duplicate test', function () { @@ -24,7 +26,11 @@ suite('Duplicate test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Duplicate block', async function () { @@ -61,7 +67,7 @@ suite('Duplicate test', function () { (workspace as Blockly.WorkspaceSvg).getTopComments()[0], ); }, text); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Duplicate. await sendKeyAndWait(this.browser, 'd'); diff --git a/test/webdriverio/test/flyout_test.ts b/test/webdriverio/test/flyout_test.ts index 4e438f6d..b199b1e8 100644 --- a/test/webdriverio/test/flyout_test.ts +++ b/test/webdriverio/test/flyout_test.ts @@ -20,6 +20,9 @@ import { getCurrentFocusNodeId, getCurrentFocusedBlockId, tabNavigateToToolbox, + checkForFailures, + idle, + setSynchronizeCoreBlocklyRendering, } from './test_setup.js'; suite('Toolbox and flyout test', function () { @@ -29,7 +32,11 @@ suite('Toolbox and flyout test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Tab navigating to toolbox should open flyout', async function () { @@ -144,7 +151,7 @@ suite('Toolbox and flyout test', function () { test('Tabbing to the workspace should close the flyout', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // The flyout should be closed since it lost focus. const flyoutIsOpen = await checkIfFlyoutIsOpen(this.browser); @@ -291,8 +298,9 @@ suite('Toolbox and flyout test', function () { } }); test('callbackkey is activated with enter', async function () { + setSynchronizeCoreBlocklyRendering(false); await tabNavigateToToolbox(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // First thing in the toolbox is the first button // Press Enter to activate it. @@ -304,8 +312,9 @@ suite('Toolbox and flyout test', function () { }); test('callbackKey is activated with enter', async function () { + setSynchronizeCoreBlocklyRendering(false); await tabNavigateToToolbox(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Navigate to second button. // Press Enter to activate it. diff --git a/test/webdriverio/test/insert_test.ts b/test/webdriverio/test/insert_test.ts index 6517cb46..449b3913 100644 --- a/test/webdriverio/test/insert_test.ts +++ b/test/webdriverio/test/insert_test.ts @@ -22,6 +22,8 @@ import { blockIsPresent, keyUp, tabNavigateToToolbox, + checkForFailures, + idle, } from './test_setup.js'; suite('Insert test', function () { @@ -31,7 +33,11 @@ suite('Insert test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Insert and cancel with block selection', async function () { @@ -145,7 +151,11 @@ suite('Insert test with more blocks', function () { testFileLocations.MORE_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Does not bump immovable input blocks on insert', async function () { diff --git a/test/webdriverio/test/keyboard_mode_test.ts b/test/webdriverio/test/keyboard_mode_test.ts index 1c0f4d8e..a794f32f 100644 --- a/test/webdriverio/test/keyboard_mode_test.ts +++ b/test/webdriverio/test/keyboard_mode_test.ts @@ -15,6 +15,8 @@ import { tabNavigateToWorkspace, clickBlock, sendKeyAndWait, + checkForFailures, + idle, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -37,7 +39,7 @@ suite( this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Reset the keyboard navigation state between tests. await this.browser.execute(() => { @@ -48,8 +50,12 @@ suite( await tabNavigateToWorkspace(this.browser); }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('T to open toolbox enables keyboard mode', async function () { - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, 't'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -57,14 +63,14 @@ suite( test('M for move mode enables keyboard mode', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, 'm'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); }); test('W for workspace cursor enables keyboard mode', async function () { - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, 'w'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -72,7 +78,7 @@ suite( test('X to disconnect enables keyboard mode', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, 'x'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -81,7 +87,7 @@ suite( test('Copy does not change keyboard mode state', async function () { // Make sure we're on a copyable block so that copy occurs await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, 'c']); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); @@ -90,7 +96,7 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, 'c']); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -99,7 +105,7 @@ suite( test('Delete does not change keyboard mode state', async function () { // Make sure we're on a deletable block so that delete occurs await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Backspace); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); @@ -110,7 +116,7 @@ suite( // Focus a different deletable block await focusOnBlock(this.browser, 'controls_if_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); await sendKeyAndWait(this.browser, Key.Backspace); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -121,10 +127,10 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Right click a block await clickBlock(this.browser, 'controls_if_1', {button: 'right'}); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); }); @@ -134,7 +140,7 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Drag a block const element = await getBlockElementById(this.browser, 'controls_if_1'); @@ -147,7 +153,7 @@ suite( ); }); await element.dragAndDrop({x: 10, y: 10}); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); }); diff --git a/test/webdriverio/test/move_test.ts b/test/webdriverio/test/move_test.ts index 72212051..24f209fb 100644 --- a/test/webdriverio/test/move_test.ts +++ b/test/webdriverio/test/move_test.ts @@ -16,12 +16,14 @@ import { sendKeyAndWait, keyDown, contextMenuItems, + checkForFailures, + idle, } from './test_setup.js'; suite('Move start tests', function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); // Clear the workspace and load start blocks. setup(async function () { @@ -29,7 +31,11 @@ suite('Move start tests', function () { testFileLocations.MOVE_START_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); // When a move of a statement block begins, it is expected that only @@ -181,9 +187,9 @@ suite('Move start tests', function () { }); suite('Statement move tests', function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); // Clear the workspace and load start blocks. setup(async function () { @@ -191,7 +197,11 @@ suite('Statement move tests', function () { testFileLocations.MOVE_STATEMENT_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); /** Serialized simple statement block with no statement inputs. */ @@ -362,9 +372,9 @@ suite('Statement move tests', function () { }); suite(`Value expression move tests`, function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); /** Serialized simple reporter value block with no inputs. */ const VALUE_SIMPLE = { @@ -474,7 +484,11 @@ suite(`Value expression move tests`, function () { ), this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); suite('Constrained moves of a simple reporter block', function () { diff --git a/test/webdriverio/test/mutator_test.ts b/test/webdriverio/test/mutator_test.ts index b604c752..18eafe1c 100644 --- a/test/webdriverio/test/mutator_test.ts +++ b/test/webdriverio/test/mutator_test.ts @@ -17,6 +17,8 @@ import { sendKeyAndWait, keyRight, keyDown, + checkForFailures, + idle, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -32,7 +34,7 @@ suite('Mutator navigation', function () { ); this.openMutator = async () => { await focusOnBlock(this.browser, 'controls_if_1'); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Navigate to the mutator icon await keyRight(this.browser); // Activate the icon @@ -40,6 +42,10 @@ suite('Mutator navigation', function () { }; }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Enter opens mutator', async function () { await this.openMutator(); @@ -96,7 +102,7 @@ suite('Mutator navigation', function () { await sendKeyAndWait(this.browser, 't'); // Navigate down to the second block in the flyout await keyDown(this.browser); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); // Hit enter to enter insert mode await sendKeyAndWait(this.browser, Key.Enter); // Hit enter again to lock it into place on the connection diff --git a/test/webdriverio/test/scroll_test.ts b/test/webdriverio/test/scroll_test.ts index c340714b..d611f866 100644 --- a/test/webdriverio/test/scroll_test.ts +++ b/test/webdriverio/test/scroll_test.ts @@ -9,26 +9,34 @@ import * as chai from 'chai'; import {Key} from 'webdriverio'; import { sendKeyAndWait, + keyUp, keyDown, keyRight, PAUSE_TIME, tabNavigateToWorkspace, testFileLocations, testSetup, + checkForFailures, + idle, } from './test_setup.js'; suite('Scrolling into view', function () { // Disable timeouts when non-zero PAUSE_TIME is used to watch tests run. if (PAUSE_TIME) this.timeout(0); - // Resize browser to provide predictable small window size for scrolling. + // Resize browser to provide predictable small viewport size for scrolling. // // N.B. that this is called only one per suite, not once per test. suiteSetup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); + // Note that a viewport is used here over adjusting window size to ensure + // consistency across platforms and environments. + // await this.browser.setViewport({ + // width: 800, height: 600, devicePixelRatio: 1 + // }); this.windowSize = await this.browser.getWindowSize(); await this.browser.setWindowSize(800, 600); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); }); // Restore original browser window size. @@ -44,10 +52,15 @@ suite('Scrolling into view', function () { await testSetup(testFileLocations.BASE, this.timeout()); }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Insert scrolls new block into view', async function () { - // Increase timeout to 10s for this longer test. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test. + this.timeout(PAUSE_TIME ? 0 : 30000); + // setPauseTime(0); await tabNavigateToWorkspace(this.browser); // Separate the two top-level blocks by moving p5_draw_1 further down. @@ -64,14 +77,19 @@ suite('Scrolling into view', function () { ).getBoundingRectangleWithoutChildren(), ); }); + // Pause to allow scrolling to stabilize before proceeding. + await idle(this.browser); // Insert and confirm the test block which should be scrolled into view. await sendKeyAndWait(this.browser, 't'); await keyRight(this.browser); - await sendKeyAndWait(this.browser, Key.Enter, 2); + await sendKeyAndWait(this.browser, Key.Enter); + await keyDown(this.browser); + await keyUp(this.browser); + await sendKeyAndWait(this.browser, Key.Enter); // Assert new block has been scrolled into the viewport. - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); const inViewport = await this.browser.execute(() => { const workspace = Blockly.getMainWorkspace() as Blockly.WorkspaceSvg; const block = workspace.getBlocksByType( diff --git a/test/webdriverio/test/stack_navigation.ts b/test/webdriverio/test/stack_navigation.ts index b26f273f..726cc996 100644 --- a/test/webdriverio/test/stack_navigation.ts +++ b/test/webdriverio/test/stack_navigation.ts @@ -13,13 +13,19 @@ import { testFileLocations, testSetup, sendKeyAndWait, + checkForFailures, + idle, } from './test_setup.js'; suite('Stack navigation', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.COMMENTS, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Next', async function () { diff --git a/test/webdriverio/test/styling_test.ts b/test/webdriverio/test/styling_test.ts index 78a99018..cd33a3b6 100644 --- a/test/webdriverio/test/styling_test.ts +++ b/test/webdriverio/test/styling_test.ts @@ -14,6 +14,8 @@ import { tabNavigateToWorkspace, testFileLocations, testSetup, + checkForFailures, + idle, } from './test_setup.js'; import * as chai from 'chai'; @@ -24,7 +26,11 @@ suite('Styling test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); async function strokeColorEquals( diff --git a/test/webdriverio/test/test_setup.ts b/test/webdriverio/test/test_setup.ts index 1ddbac97..8c2c00ab 100644 --- a/test/webdriverio/test/test_setup.ts +++ b/test/webdriverio/test/test_setup.ts @@ -44,6 +44,8 @@ let driver: webdriverio.Browser | null = null; */ export const PAUSE_TIME = 0; +var synchronizeCoreBlocklyRendering: boolean = true; + /** * Start up WebdriverIO and load the test page. This should only be * done once, to avoid constantly popping browser windows open and @@ -119,8 +121,16 @@ export async function testSetup( playgroundUrl: string, wdioWaitTimeoutMs: number, ): Promise { + // Reset back to default state between tests. + synchronizeCoreBlocklyRendering = true; + if (!driver) { driver = await driverSetup(wdioWaitTimeoutMs); + } else if (process.env.CI) { + // If running in CI force a session reload to ensure no browser state can + // leak across test suites (since this can sometimes cause complex combined + // failures in CI). + await driver.reloadSession(); } await driver.url(playgroundUrl); // Wait for the workspace to exist and be rendered. @@ -130,6 +140,16 @@ export async function testSetup( return driver; } +export async function checkForFailures( + browser: WebdriverIO.Browser, + testTitle: string, + testState: string | undefined +) { + if (testState === 'failed') { + await browser.saveScreenshot(`failures/${testTitle}.png`); + } +} + /** * Replaces OS-specific path with POSIX style path. * @@ -192,6 +212,63 @@ export async function getSelectedBlockId(browser: WebdriverIO.Browser) { }); } +/** + * Idles the browser for PAUSE_TIME, also possibly synchronizing rendering in + * core Blockly. + * + * This generally should always be preferred over calling browser.pause() + * directly. + * + * See setSynchronizeCoreBlocklyRendering() for additional details on + * configuring how this function behaves. + * + * @param browser The active WebdriverIO Browser object. + */ +export async function idle(browser: WebdriverIO.Browser, synchronizeRendering: boolean = true) { + if (synchronizeCoreBlocklyRendering) { + // First, attempt to synchronize on rendering to ensure that Blockly is + // fully rendered before pausing for browser execution. This works around + // potential bugs when running in headless mode that can cause + // requestAnimationFrame to not call back (and cause state inconsistencies + // in block positions and sizes per #770). + await browser.execute(() => { + const workspace = Blockly.getMainWorkspace() as Blockly.WorkspaceSvg; + // Queue re-rendering all blocks. + workspace.render(); + // Flush the rendering queue (this is a slight hack to leverage + // BlockSvg.render() directly blocking on rendering finishing). + const blocks = workspace.getTopBlocks(); + if (blocks.length > 0) { + blocks[0].render(); + } + }); + } + await browser.pause(PAUSE_TIME); +} + +/** + * Configures whether to synchronize core Blockly's rendering system when trying + * to idle tests to wait for operations to complete. + * + * This is enabled by default and changes the behavior of idle() which is used + * both directly in tests and indirectly via the many test helpers in this file. + * + * Synchronization is useful because it ensures Blockly is fully rendered and + * stable before proceeding with the test (which can sometimes be desynchronized + * if the headless test environment drops a request for animation rendering + * frame which has been observed in CI environments). + * + * Synchronization must be disabled in certain tests, particularly those that + * can trigger alert dialogs since, when open, these will always cause any + * browser.execute() calls to fail the test (and rendering synchronization + * relies on this WebdriverIO mechanism). + * + * @param enableSynchronization + */ +export async function setSynchronizeCoreBlocklyRendering(enableSynchronization: boolean) { + synchronizeCoreBlocklyRendering = enableSynchronization; +} + /** * Clicks in the workspace to focus it. * @@ -202,7 +279,7 @@ export async function focusWorkspace(browser: WebdriverIO.Browser) { '#blocklyDiv > div > svg.blocklySvg > g', ); await workspaceElement.click({x: 100}); - await browser.pause(PAUSE_TIME); + await idle(browser); } /** @@ -291,7 +368,7 @@ export async function focusOnBlock( if (!block) throw new Error(`No block found with ID: ${blockId}.`); Blockly.getFocusManager().focusNode(block); }, blockId); - await browser.pause(PAUSE_TIME); + await idle(browser); } /** @@ -314,7 +391,7 @@ export async function focusOnWorkspaceComment( } Blockly.getFocusManager().focusNode(comment); }, commentId); - await browser.pause(PAUSE_TIME); + await idle(browser); } /** @@ -346,7 +423,7 @@ export async function focusOnBlockField( blockId, fieldName, ); - await browser.pause(PAUSE_TIME); + await idle(browser); } /** @@ -479,7 +556,7 @@ export async function tabNavigateToWorkspace( // there's no straightforward way to do that; see // https://stackoverflow.com/q/51518855/4969945 await browser.execute(() => document.getElementById('focusableDiv')?.focus()); - await browser.pause(PAUSE_TIME); + await idle(browser); // Navigate to workspace. if (hasToolbox) await tabNavigateForward(browser); if (hasFlyout) await tabNavigateForward(browser); @@ -576,10 +653,11 @@ export async function sendKeyAndWait( // Send all keys in one call if no pauses needed. keys = Array(times).fill(keys).flat(); await browser.keys(keys); + await idle(browser); } else { for (let i = 0; i < times; i++) { await browser.keys(keys); - await browser.pause(PAUSE_TIME); + await idle(browser); } } } @@ -725,13 +803,13 @@ export async function clickBlock( blockId, findableId, ); - await browser.pause(PAUSE_TIME); + await idle(browser); // In the test context, get the WebdriverIO Element that we've identified. const elem = await browser.$(`#${findableId}`); await elem.click(clickOptions); - await browser.pause(PAUSE_TIME); + await idle(browser); // In the browser context, remove the ID. await browser.execute((elemId) => { @@ -751,5 +829,5 @@ export async function rightClickOnFlyoutBlockType( ) { const elem = await browser.$(`.blocklyFlyout .${blockType}`); await elem.click({button: 'right'}); - await browser.pause(PAUSE_TIME); + await idle(browser); } diff --git a/test/webdriverio/test/toast_test.ts b/test/webdriverio/test/toast_test.ts index d73ca3d6..04a0a42f 100644 --- a/test/webdriverio/test/toast_test.ts +++ b/test/webdriverio/test/toast_test.ts @@ -6,7 +6,7 @@ import * as chai from 'chai'; import * as Blockly from 'blockly/core'; -import {PAUSE_TIME, testFileLocations, testSetup} from './test_setup.js'; +import {PAUSE_TIME, testFileLocations, testSetup, checkForFailures, idle} from './test_setup.js'; suite('HTML toasts', function () { // Disable timeouts when non-zero PAUSE_TIME is used to watch tests run. @@ -15,7 +15,11 @@ suite('HTML toasts', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await idle(this.browser); + }); + + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); }); test('Can be displayed', async function () { diff --git a/test/webdriverio/test/workspace_comment_test.ts b/test/webdriverio/test/workspace_comment_test.ts index 7f9e4a75..22ec5569 100644 --- a/test/webdriverio/test/workspace_comment_test.ts +++ b/test/webdriverio/test/workspace_comment_test.ts @@ -20,6 +20,7 @@ import { keyUp, contextMenuItems, PAUSE_TIME, + checkForFailures, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -69,6 +70,10 @@ suite('Workspace comment navigation', function () { }; }); + teardown(async function() { + await checkForFailures(this.browser, this.currentTest!.title, this.currentTest?.state); + }); + test('Navigate forward from block to workspace comment', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); await keyDown(this.browser); @@ -151,7 +156,7 @@ suite('Workspace comment navigation', function () { return Blockly.getMainWorkspace() .getCommentById(commentId) ?.isCollapsed(); - }, this.commentId1); + }, this.commentId1 as string); chai.assert.isTrue(collapsed); }); @@ -172,7 +177,7 @@ suite('Workspace comment navigation', function () { const commentText = await this.browser.execute((commentId) => { return Blockly.getMainWorkspace().getCommentById(commentId)?.getText(); - }, this.commentId1); + }, this.commentId1 as string); chai.assert.equal(commentText, 'Comment oneHello world'); });