From 5fb9faa286b5d9356db65056d978359446a944cf Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sun, 10 Nov 2024 13:00:02 +0000 Subject: [PATCH 01/12] Initial k6 load test --- k6/_util.js | 14 ++++++++++++++ k6/root.js | 15 +++++++++++++++ package.json | 1 + yarn.lock | 5 +++++ 4 files changed, 35 insertions(+) create mode 100644 k6/_util.js create mode 100644 k6/root.js diff --git a/k6/_util.js b/k6/_util.js new file mode 100644 index 000000000..a9a7c800c --- /dev/null +++ b/k6/_util.js @@ -0,0 +1,14 @@ +export const url = (path) => `${__ENV.BASE_URL ?? "http://localhost:3000"}/${path}`; + +/** @type {import('k6/options').Options} */ +export const defaultOptions = { + vus: 10, + duration: '30s', +}; + +/** @type {import('k6/options').Options} */ +export const debugOptions = { + vus: 1, + iterations: 1, + httpDebug: 'full', +}; diff --git a/k6/root.js b/k6/root.js new file mode 100644 index 000000000..0f44d1f42 --- /dev/null +++ b/k6/root.js @@ -0,0 +1,15 @@ +/* eslint-disable import/no-unresolved */ +import { check } from 'k6'; +import http from 'k6/http'; +/* eslint-enable import/no-unresolved */ +import { defaultOptions, url } from './_util.js'; + +export const options = defaultOptions; + +export default () => { + const rootUrl = url(''); + check(http.get(rootUrl), { + 'status was 200': (res) => res.status === 200, + 'includes expected text': (res) => res.body?.toString()?.includes('Hello WORLD!'), + }); +}; diff --git a/package.json b/package.json index 54641cf43..1959deb3d 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@tsconfig/node14": "^14.1.2", "@types/fs-extra": "^11.0.4", "@types/jest": "^29.5.12", + "@types/k6": "^0.54.2", "@types/lockfile": "^1.0.4", "@types/touch": "^3.1.5", "babel-jest": "^29.7.0", diff --git a/yarn.lock b/yarn.lock index d3cb39e1e..a55246a69 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1789,6 +1789,11 @@ dependencies: "@types/node" "*" +"@types/k6@^0.54.2": + version "0.54.2" + resolved "https://registry.yarnpkg.com/@types/k6/-/k6-0.54.2.tgz#944d6e20881d0fed3123742654ec8a12e175ea49" + integrity sha512-B5LPxeQm97JnUTpoKNE1UX9jFp+JiJCAXgZOa2P7aChxVoPQXKfWMzK+739xHq3lPkKj1aV+HeOxkP56g/oWBg== + "@types/lockfile@^1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@types/lockfile/-/lockfile-1.0.4.tgz#9d6a6d1b6dbd4853cecc7f334bc53ea0ff363b8e" From 441a8d73392c8c62b06d8044d644d22e5617dc44 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sun, 10 Nov 2024 14:19:43 +0000 Subject: [PATCH 02/12] Add streaming load test using k6 --- k6/_util.js | 39 ++++++++++++++----- k6/root.js | 2 +- k6/streaming.js | 30 ++++++++++++++ .../pages/stream_async_components.html.erb | 7 +++- .../StreamAsyncComponents.jsx | 16 ++++---- 5 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 k6/streaming.js diff --git a/k6/_util.js b/k6/_util.js index a9a7c800c..45aa9dee9 100644 --- a/k6/_util.js +++ b/k6/_util.js @@ -1,14 +1,35 @@ export const url = (path) => `${__ENV.BASE_URL ?? "http://localhost:3000"}/${path}`; -/** @type {import('k6/options').Options} */ -export const defaultOptions = { - vus: 10, - duration: '30s', +/** @type {(envVar: string) => boolean} */ +const envToBoolean = (envVar) => { +// eslint-disable-next-line no-undef -- __ENV is provided by k6 + const value = __ENV[envVar]; + return !!value && ['true', '1', 'yes'].includes(value.toLowerCase()); }; -/** @type {import('k6/options').Options} */ -export const debugOptions = { - vus: 1, - iterations: 1, - httpDebug: 'full', +/** @type {(isBrowser: boolean, isDebug?: boolean) => import('k6/options').Options} */ +export const defaultOptions = (isBrowser, isDebug = envToBoolean('DEBUG_K6')) => { + const baseOptions = isDebug ? + { + vus: 1, + iterations: 1, + httpDebug: isBrowser ? undefined : 'full', + } : + { + vus: 10, + duration: '30s', + }; + return isBrowser ? { + scenarios: { + browser: { + ...baseOptions, + executor: 'shared-iterations', + options: { + browser: { + type: 'chromium', + }, + }, + }, + } + } : baseOptions; }; diff --git a/k6/root.js b/k6/root.js index 0f44d1f42..d57018919 100644 --- a/k6/root.js +++ b/k6/root.js @@ -4,7 +4,7 @@ import http from 'k6/http'; /* eslint-enable import/no-unresolved */ import { defaultOptions, url } from './_util.js'; -export const options = defaultOptions; +export const options = defaultOptions(false); export default () => { const rootUrl = url(''); diff --git a/k6/streaming.js b/k6/streaming.js new file mode 100644 index 000000000..2afe14567 --- /dev/null +++ b/k6/streaming.js @@ -0,0 +1,30 @@ +/* eslint-disable import/no-unresolved */ +import { check } from 'k6'; +import { browser } from 'k6/browser'; +/* eslint-enable import/no-unresolved */ +import { defaultOptions, url } from './_util.js'; + +export const options = defaultOptions(true); + +export default async () => { + const streamingUrl = url('stream_async_components?delay=0'); + const page = await browser.newPage(); + try { + await page.goto(streamingUrl); + await page.waitForFunction( + () => !document.body.textContent.includes('Loading'), + { + // in milliseconds + timeout: 5000, + }, + ); + check(await page.locator('html').textContent(), { + 'has all comments': (text) => { + const commentIds = [1, 2, 3, 4]; + return commentIds.every((id) => text.includes(`Comment ${id}`)); + }, + }); + } finally { + await page.close(); + } +}; diff --git a/spec/dummy/app/views/pages/stream_async_components.html.erb b/spec/dummy/app/views/pages/stream_async_components.html.erb index 5d0de9f46..56258ac71 100644 --- a/spec/dummy/app/views/pages/stream_async_components.html.erb +++ b/spec/dummy/app/views/pages/stream_async_components.html.erb @@ -1,4 +1,9 @@ -<%= stream_react_component("StreamAsyncComponents", props: @app_props_server_render, prerender: true, trace: true, id: "StreamAsyncComponents-react-component-0") %> +<%= + props = @app_props_server_render + delay = params[:delay] + props = props.merge(baseDelayMs: delay.to_i) unless delay.nil? + stream_react_component("StreamAsyncComponents", props: props, prerender: true, trace: true, id: "StreamAsyncComponents-react-component-0") +%>

React Rails Server Streaming Server Rendered Async React Components

diff --git a/spec/dummy/client/app/ror-auto-load-components/StreamAsyncComponents.jsx b/spec/dummy/client/app/ror-auto-load-components/StreamAsyncComponents.jsx index a39b87431..4bcdfe2b2 100644 --- a/spec/dummy/client/app/ror-auto-load-components/StreamAsyncComponents.jsx +++ b/spec/dummy/client/app/ror-auto-load-components/StreamAsyncComponents.jsx @@ -5,11 +5,11 @@ const delayPromise = (promise, ms) => new Promise((resolve) => setTimeout(() => const cachedFetches = {}; -const AsyncPost = async () => { +const AsyncPost = async ({ delayMs }) => { console.log('Hello from AsyncPost'); const post = (cachedFetches['post'] ??= await delayPromise( fetch('https://jsonplaceholder.org/posts/1'), - 2000, + delayMs, ).then((response) => response.json())); // Uncomment to test handling of errors occuring outside of the shell @@ -26,10 +26,10 @@ const AsyncPost = async () => { ); }; -const AsyncComment = async ({ commentId }) => { +const AsyncComment = async ({ commentId, delayMs }) => { const comment = (cachedFetches[commentId] ??= await delayPromise( fetch(`https://jsonplaceholder.org/comments/${commentId}`), - 2000 + commentId * 1000, + delayMs, ).then((response) => response.json())); console.log('Hello from AsyncComment', commentId); return ( @@ -40,8 +40,8 @@ const AsyncComment = async ({ commentId }) => { ); }; -function StreamAsyncComponents(props) { - const [name, setName] = useState(props.helloWorldData.name); +function StreamAsyncComponents({ helloWorldData, baseDelayMs = 1000 }) { + const [name, setName] = useState(helloWorldData.name); // Uncomment to test error handling during rendering the shell // throw new Error('Hello from StreamAsyncComponents'); @@ -57,13 +57,13 @@ function StreamAsyncComponents(props) {

Loading...}> - +

Comments Fetched Asynchronously on Server

{[1, 2, 3, 4].map((commentId) => ( Loading Comment {commentId}...}> - + ))} From 573cd19b57b13c85a188703bf6c2b18e350ae73b Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 11 Nov 2024 08:01:07 +0000 Subject: [PATCH 03/12] Fix ESLint for K6 --- .eslintrc | 25 +++++++++++++++++++++++-- k6/_util.js | 1 - k6/root.js | 2 -- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.eslintrc b/.eslintrc index 530f9cde1..2df093c6f 100644 --- a/.eslintrc +++ b/.eslintrc @@ -8,8 +8,8 @@ extends: - prettier globals: - __DEBUG_SERVER_ERRORS__: true - __SERVER_ERRORS__: true + __DEBUG_SERVER_ERRORS__: writable + __SERVER_ERRORS__: writable parserOptions: # We have @babel/eslint-parser from eslint-config-shakacode, but don't use Babel in the main project @@ -89,3 +89,24 @@ overrides: - error - patterns: - "../*" + - files: "k6/**/*.{js,ts}" + globals: + # See https://github.com/grafana/k6-docs/blob/65d9f8a9a53f57a4f416763f8020d0c7f40eb976/docs/sources/.eslintrc.js + __ENV: readonly + __VU: readonly + __ITER: readonly + console: readonly + open: readonly + window: readonly + setInterval: readonly + clearInterval: readonly + setTimeout: readonly + clearTimeout: readonly + rules: + "import/extensions": + - error + - ignorePackages + "import/no-unresolved": + - error + # k6 and k6/... have to be installed globally and can't be resolved by ESLint + - { ignore: ["k6.*"] } diff --git a/k6/_util.js b/k6/_util.js index 45aa9dee9..a111ce517 100644 --- a/k6/_util.js +++ b/k6/_util.js @@ -2,7 +2,6 @@ export const url = (path) => `${__ENV.BASE_URL ?? "http://localhost:3000"}/${pat /** @type {(envVar: string) => boolean} */ const envToBoolean = (envVar) => { -// eslint-disable-next-line no-undef -- __ENV is provided by k6 const value = __ENV[envVar]; return !!value && ['true', '1', 'yes'].includes(value.toLowerCase()); }; diff --git a/k6/root.js b/k6/root.js index d57018919..35ff248d2 100644 --- a/k6/root.js +++ b/k6/root.js @@ -1,7 +1,5 @@ -/* eslint-disable import/no-unresolved */ import { check } from 'k6'; import http from 'k6/http'; -/* eslint-enable import/no-unresolved */ import { defaultOptions, url } from './_util.js'; export const options = defaultOptions(false); From 47f60cdbd4bc718233296d6613a8d61981904912 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 11 Nov 2024 08:39:01 +0000 Subject: [PATCH 04/12] Switch the streaming load test from browser to request API --- k6/streaming.js | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/k6/streaming.js b/k6/streaming.js index 2afe14567..cae8f750d 100644 --- a/k6/streaming.js +++ b/k6/streaming.js @@ -1,30 +1,19 @@ -/* eslint-disable import/no-unresolved */ import { check } from 'k6'; -import { browser } from 'k6/browser'; -/* eslint-enable import/no-unresolved */ +import http from 'k6/http'; import { defaultOptions, url } from './_util.js'; -export const options = defaultOptions(true); +export const options = defaultOptions(false); -export default async () => { - const streamingUrl = url('stream_async_components?delay=0'); - const page = await browser.newPage(); - try { - await page.goto(streamingUrl); - await page.waitForFunction( - () => !document.body.textContent.includes('Loading'), - { - // in milliseconds - timeout: 5000, - }, - ); - check(await page.locator('html').textContent(), { - 'has all comments': (text) => { - const commentIds = [1, 2, 3, 4]; - return commentIds.every((id) => text.includes(`Comment ${id}`)); - }, - }); - } finally { - await page.close(); - } +export default () => { + const streamingUrl = url('stream_async_components?delay=5'); + check(http.get(streamingUrl), { + 'status was 200': (res) => res.status === 200, + 'has all comments': (res) => { + const body = res.html().text(); + const commentIds = [1, 2, 3, 4]; + const hasAllComments = commentIds.every((commentId) => body.includes(`Comment ${commentId}`)); + const hasFailedRequests = !!body.match(/Request to .+ failed/i); + return hasAllComments && !hasFailedRequests; + }, + }); }; From fc4ed67c88513a2030f9648eeb8338ff1e783d2d Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 11 Nov 2024 12:43:42 +0000 Subject: [PATCH 05/12] Move _util to lib/ --- k6/_util.js | 34 ---------------------------------- k6/lib/util.js | 32 ++++++++++++++++++++++++++++++++ k6/root.js | 4 ++-- k6/streaming.js | 4 ++-- 4 files changed, 36 insertions(+), 38 deletions(-) delete mode 100644 k6/_util.js create mode 100644 k6/lib/util.js diff --git a/k6/_util.js b/k6/_util.js deleted file mode 100644 index a111ce517..000000000 --- a/k6/_util.js +++ /dev/null @@ -1,34 +0,0 @@ -export const url = (path) => `${__ENV.BASE_URL ?? "http://localhost:3000"}/${path}`; - -/** @type {(envVar: string) => boolean} */ -const envToBoolean = (envVar) => { - const value = __ENV[envVar]; - return !!value && ['true', '1', 'yes'].includes(value.toLowerCase()); -}; - -/** @type {(isBrowser: boolean, isDebug?: boolean) => import('k6/options').Options} */ -export const defaultOptions = (isBrowser, isDebug = envToBoolean('DEBUG_K6')) => { - const baseOptions = isDebug ? - { - vus: 1, - iterations: 1, - httpDebug: isBrowser ? undefined : 'full', - } : - { - vus: 10, - duration: '30s', - }; - return isBrowser ? { - scenarios: { - browser: { - ...baseOptions, - executor: 'shared-iterations', - options: { - browser: { - type: 'chromium', - }, - }, - }, - } - } : baseOptions; -}; diff --git a/k6/lib/util.js b/k6/lib/util.js new file mode 100644 index 000000000..3ce2c5667 --- /dev/null +++ b/k6/lib/util.js @@ -0,0 +1,32 @@ +export const url = (path) => `${__ENV.BASE_URL ?? 'http://localhost:3000'}/${path}`; + +/** @type {(envVar: string) => boolean} */ +const envToBoolean = (envVar) => { + const value = __ENV[envVar]; + return !!value && ['true', '1', 'yes'].includes(value.toLowerCase()); +}; + +/** @type {(env?: { isBrowser?: boolean, isDebug?: boolean }) => import('k6/options').Options} */ +export const defaultOptions = ({ isBrowser = false, isDebug = envToBoolean('DEBUG_K6') } = {}) => { + const baseOptions = isDebug + ? { + vus: 1, + iterations: 1, + httpDebug: isBrowser ? undefined : 'full', + } + : { + vus: 10, + duration: '30s', + }; + return isBrowser + ? { + ...baseOptions, + executor: 'shared-iterations', + options: { + browser: { + type: 'chromium', + }, + }, + } + : baseOptions; +}; diff --git a/k6/root.js b/k6/root.js index 35ff248d2..a47a6eb86 100644 --- a/k6/root.js +++ b/k6/root.js @@ -1,8 +1,8 @@ import { check } from 'k6'; import http from 'k6/http'; -import { defaultOptions, url } from './_util.js'; +import { defaultOptions, url } from './lib/util.js'; -export const options = defaultOptions(false); +export const options = defaultOptions(); export default () => { const rootUrl = url(''); diff --git a/k6/streaming.js b/k6/streaming.js index cae8f750d..adec6fbfb 100644 --- a/k6/streaming.js +++ b/k6/streaming.js @@ -1,8 +1,8 @@ import { check } from 'k6'; import http from 'k6/http'; -import { defaultOptions, url } from './_util.js'; +import { defaultOptions, url } from './lib/util.js'; -export const options = defaultOptions(false); +export const options = defaultOptions(); export default () => { const streamingUrl = url('stream_async_components?delay=5'); From 48c639aeac811b306e111682924581b6ed31c0d3 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 11 Nov 2024 15:04:17 +0000 Subject: [PATCH 06/12] Format k6/** --- k6/root.js | 26 +++++++++++++------------- k6/streaming.js | 38 +++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/k6/root.js b/k6/root.js index a47a6eb86..7dd964da4 100644 --- a/k6/root.js +++ b/k6/root.js @@ -1,13 +1,13 @@ -import { check } from 'k6'; -import http from 'k6/http'; -import { defaultOptions, url } from './lib/util.js'; - -export const options = defaultOptions(); - -export default () => { - const rootUrl = url(''); - check(http.get(rootUrl), { - 'status was 200': (res) => res.status === 200, - 'includes expected text': (res) => res.body?.toString()?.includes('Hello WORLD!'), - }); -}; +import { check } from 'k6'; +import http from 'k6/http'; +import { defaultOptions, url } from './lib/util.js'; + +export const options = defaultOptions(); + +export default () => { + const rootUrl = url(''); + check(http.get(rootUrl), { + 'status was 200': (res) => res.status === 200, + 'includes expected text': (res) => res.body?.toString()?.includes('Hello WORLD!'), + }); +}; diff --git a/k6/streaming.js b/k6/streaming.js index adec6fbfb..db2e7dd1d 100644 --- a/k6/streaming.js +++ b/k6/streaming.js @@ -1,19 +1,19 @@ -import { check } from 'k6'; -import http from 'k6/http'; -import { defaultOptions, url } from './lib/util.js'; - -export const options = defaultOptions(); - -export default () => { - const streamingUrl = url('stream_async_components?delay=5'); - check(http.get(streamingUrl), { - 'status was 200': (res) => res.status === 200, - 'has all comments': (res) => { - const body = res.html().text(); - const commentIds = [1, 2, 3, 4]; - const hasAllComments = commentIds.every((commentId) => body.includes(`Comment ${commentId}`)); - const hasFailedRequests = !!body.match(/Request to .+ failed/i); - return hasAllComments && !hasFailedRequests; - }, - }); -}; +import { check } from 'k6'; +import http from 'k6/http'; +import { defaultOptions, url } from './lib/util.js'; + +export const options = defaultOptions(); + +export default () => { + const streamingUrl = url('stream_async_components?delay=5'); + check(http.get(streamingUrl), { + 'status was 200': (res) => res.status === 200, + 'has all comments': (res) => { + const body = res.html().text(); + const commentIds = [1, 2, 3, 4]; + const hasAllComments = commentIds.every((commentId) => body.includes(`Comment ${commentId}`)); + const hasFailedRequests = !!body.match(/Request to .+ failed/i); + return hasAllComments && !hasFailedRequests; + }, + }); +}; From d48e6fe0d3e41d7df56461e5e416c1c9b3a91266 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sun, 17 Nov 2024 14:16:00 +0000 Subject: [PATCH 07/12] Update check for rendered components --- .eslintrc | 4 ---- k6/lib/util.js | 3 ++- k6/root.js | 13 ++++++++++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.eslintrc b/.eslintrc index 2df093c6f..deae43217 100644 --- a/.eslintrc +++ b/.eslintrc @@ -7,10 +7,6 @@ extends: - eslint-config-shakacode - prettier -globals: - __DEBUG_SERVER_ERRORS__: writable - __SERVER_ERRORS__: writable - parserOptions: # We have @babel/eslint-parser from eslint-config-shakacode, but don't use Babel in the main project requireConfigFile: false diff --git a/k6/lib/util.js b/k6/lib/util.js index 3ce2c5667..648ece170 100644 --- a/k6/lib/util.js +++ b/k6/lib/util.js @@ -21,7 +21,8 @@ export const defaultOptions = ({ isBrowser = false, isDebug = envToBoolean('DEBU return isBrowser ? { ...baseOptions, - executor: 'shared-iterations', + // See https://github.com/grafana/k6-learn/blob/main/Modules/III-k6-Intermediate/08-Setting-load-profiles-with-executors.md + executor: isDebug ? 'shared-iterations' : 'constant-vus', options: { browser: { type: 'chromium', diff --git a/k6/root.js b/k6/root.js index 7dd964da4..674913eab 100644 --- a/k6/root.js +++ b/k6/root.js @@ -8,6 +8,17 @@ export default () => { const rootUrl = url(''); check(http.get(rootUrl), { 'status was 200': (res) => res.status === 200, - 'includes expected text': (res) => res.body?.toString()?.includes('Hello WORLD!'), + 'renders components successfully': (res) => { + const body = res.html().text(); + return Object.entries({ + // This is visible on the page in the browser 4 times, but for some reason, the one under + // "Server Rendered React Component Without Redux" is missing in `body`. + 'Hello, Mr. Server Side Rendering!': 3, + 'Hello, Mrs. Client Side Rendering!': 2, + 'Hello, Mrs. Client Side Hello Again!': 1, + 'Hello ES5, Mrs. Client Side Rendering!': 1, + 'Hello WORLD! Will this work?? YES! Time to visit Maui': 1, + }).every(([text, count]) => body.split(text).length >= count + 1); + }, }); }; From 6b11238c00f80bf20371a8458956cd87dfc4b9bd Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Sun, 17 Nov 2024 15:44:12 +0000 Subject: [PATCH 08/12] Improve defaultOptions --- k6/lib/util.js | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/k6/lib/util.js b/k6/lib/util.js index 648ece170..9adf2e786 100644 --- a/k6/lib/util.js +++ b/k6/lib/util.js @@ -6,23 +6,36 @@ const envToBoolean = (envVar) => { return !!value && ['true', '1', 'yes'].includes(value.toLowerCase()); }; -/** @type {(env?: { isBrowser?: boolean, isDebug?: boolean }) => import('k6/options').Options} */ -export const defaultOptions = ({ isBrowser = false, isDebug = envToBoolean('DEBUG_K6') } = {}) => { +/** + * @param {boolean} [inScenario=isBrowser] Is this used as `scenarios: { : defaultOptions(...) }`? + * @param {boolean} [isBrowser=false] Is this a browser test? + * @param {boolean} [isDebug=env.DEBUG_K6] Are we running in debug mode? + * @return {import('k6/options').Options} + * */ +export const defaultOptions = ({ + isBrowser = false, + isDebug = envToBoolean('DEBUG_K6'), + // Browser tests options can only be set inside `scenarios` + // https://grafana.com/docs/k6/latest/using-k6-browser/ + inScenario = isBrowser, +} = {}) => { const baseOptions = isDebug ? { vus: 1, iterations: 1, - httpDebug: isBrowser ? undefined : 'full', + httpDebug: inScenario ? undefined : 'full', } : { vus: 10, duration: '30s', }; + if (inScenario) { + // See https://github.com/grafana/k6-learn/blob/main/Modules/III-k6-Intermediate/08-Setting-load-profiles-with-executors.md + baseOptions.executor = isDebug ? 'shared-iterations' : 'constant-vus'; + } return isBrowser ? { ...baseOptions, - // See https://github.com/grafana/k6-learn/blob/main/Modules/III-k6-Intermediate/08-Setting-load-profiles-with-executors.md - executor: isDebug ? 'shared-iterations' : 'constant-vus', options: { browser: { type: 'chromium', From 84a5e20d62683cf894c56d908e7df2f4f14e6349 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 18 Nov 2024 13:54:18 +0000 Subject: [PATCH 09/12] Add benchmarking to docs --- CONTRIBUTING.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a24acfd6d..07c9a4ae5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -293,12 +293,33 @@ If you run `rspec` at the top level, you'll see this message: `require': cannot After running a test, you can view the coverage results in SimpleCov reports by opening `coverage/index.html`. +### Benchmarking +You'll need to [install `k6`](https://grafana.com/docs/k6/latest/set-up/install-k6/) first. + +The benchmarking scripts are in `k6` directory, so you can run, for example: +```sh +k6 run k6/root.js +``` +to exercise the main page of the dummy app, or `k6/streaming.js` for the streaming page. + +For significant changes, please make sure to run all benchmarks before and after, and include the results in the PR. +Later they will be added to CI, if we can make sure the results there are stable enough. + +If you add significant new functionality and add a page in the dummy app showing it, consider adding the corresponding benchmark as well. + +#### Debugging the benchmark scripts +You can add `-e DEBUG_K6=true` to run only a single iteration while showing HTTP requests and responses. See +[How to debug k6 load testing scripts](https://github.com/grafana/k6-learn/blob/main/Modules/III-k6-Intermediate/01-How-to-debug-k6-load-testing-scripts.md) for more suggestions if needed. + ### Debugging Start the sample app like this for some debug printing: ```sh -TRACE_REACT_ON_RAILS=true && foreman start -f Procfile.dev +TRACE_REACT_ON_RAILS=true overmind start -f Procfile.dev ``` +Using `overmind` instead of `foreman` lets you restart separate processes, connect with them when stopped with a `debugger` call, and so on. +See https://railsnotes.xyz/blog/overmind-better-bin-dev-for-your-procfile-dev for more details. +If you don't need that, you can use `bin/dev` or `foreman` as well. # Releasing Contact Justin Gordon, justin@shakacode.com From b59fd67cde93a2b5c042969ebdb4a6b550922eb7 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 18 Nov 2024 17:55:11 +0000 Subject: [PATCH 10/12] Switch streaming back to browser tests --- .eslintrc | 4 ++++ CONTRIBUTING.md | 5 ++++- k6/streaming.js | 43 +++++++++++++++++++++++++++++-------------- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.eslintrc b/.eslintrc index deae43217..437612212 100644 --- a/.eslintrc +++ b/.eslintrc @@ -106,3 +106,7 @@ overrides: - error # k6 and k6/... have to be installed globally and can't be resolved by ESLint - { ignore: ["k6.*"] } + no-use-before-define: + - error + # We want to allow function.name in options, and place them at the top of the file + - { functions: false } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07c9a4ae5..466c0424d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -300,7 +300,10 @@ The benchmarking scripts are in `k6` directory, so you can run, for example: ```sh k6 run k6/root.js ``` -to exercise the main page of the dummy app, or `k6/streaming.js` for the streaming page. +to exercise the main page of the dummy app. + +For browser tests like `k6/streaming.js`, it's recommended to include `K6_BROWSER_LOG=fatal` for actual measurement after verifying the tests work. +It avoids unnecessary overhead from outputting console messages. For significant changes, please make sure to run all benchmarks before and after, and include the results in the PR. Later they will be added to CI, if we can make sure the results there are stable enough. diff --git a/k6/streaming.js b/k6/streaming.js index db2e7dd1d..1b6448fec 100644 --- a/k6/streaming.js +++ b/k6/streaming.js @@ -1,19 +1,34 @@ import { check } from 'k6'; -import http from 'k6/http'; +import { browser } from 'k6/browser'; import { defaultOptions, url } from './lib/util.js'; -export const options = defaultOptions(); +const streamingUrl = url('stream_async_components?delay=5'); -export default () => { - const streamingUrl = url('stream_async_components?delay=5'); - check(http.get(streamingUrl), { - 'status was 200': (res) => res.status === 200, - 'has all comments': (res) => { - const body = res.html().text(); - const commentIds = [1, 2, 3, 4]; - const hasAllComments = commentIds.every((commentId) => body.includes(`Comment ${commentId}`)); - const hasFailedRequests = !!body.match(/Request to .+ failed/i); - return hasAllComments && !hasFailedRequests; - }, - }); +export const options = { + scenarios: { + browser: defaultOptions({ isBrowser: true }), + }, +}; + +export default async () => { + const page = await browser.newPage(); + try { + const response = await page.goto(streamingUrl); + check(response, { + 'status was 200': (res) => res.status() === 200, + }); + await page.waitForFunction(() => !document.body.textContent.includes('Loading'), { + // in milliseconds + timeout: 5000, + }); + check(await page.locator('html').textContent(), { + 'has all comments': (text) => { + // can't define commentIds as a constant outside, this runs in browser context + const commentIds = [1, 2, 3, 4]; + return commentIds.every((id) => text.includes(`Comment ${id}`)); + }, + }); + } finally { + await page.close(); + } }; From a25cc547603c5706343249310877e2a078857781 Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Mon, 18 Nov 2024 18:52:59 +0000 Subject: [PATCH 11/12] Add spec/dummy/bin/prod --- CONTRIBUTING.md | 6 +++++- spec/dummy/bin/prod | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100755 spec/dummy/bin/prod diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 466c0424d..d7d0876f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -294,7 +294,11 @@ If you run `rspec` at the top level, you'll see this message: `require': cannot After running a test, you can view the coverage results in SimpleCov reports by opening `coverage/index.html`. ### Benchmarking -You'll need to [install `k6`](https://grafana.com/docs/k6/latest/set-up/install-k6/) first. +You'll need to [install `k6`](https://grafana.com/docs/k6/latest/set-up/install-k6/) first and start the dummy app in production mode (using [bin/prod](https://github.com/shakacode/react_on_rails_pro/blob/master/spec/dummy/bin/prod)). +You can remove even more overhead by using +```sh +bin/prod &> /dev/null +``` The benchmarking scripts are in `k6` directory, so you can run, for example: ```sh diff --git a/spec/dummy/bin/prod b/spec/dummy/bin/prod new file mode 100755 index 000000000..3c844bf0f --- /dev/null +++ b/spec/dummy/bin/prod @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +RAILS_ENV=production NODE_ENV=production overmind start -f Procfile.static From 72f5115df1a990f3d5fa57f1411fef67c9863dcf Mon Sep 17 00:00:00 2001 From: Alexey Romanov Date: Tue, 19 Nov 2024 08:39:47 +0000 Subject: [PATCH 12/12] Add thresholds for failing test --- k6/lib/util.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/k6/lib/util.js b/k6/lib/util.js index 9adf2e786..1ab08f146 100644 --- a/k6/lib/util.js +++ b/k6/lib/util.js @@ -10,6 +10,7 @@ const envToBoolean = (envVar) => { * @param {boolean} [inScenario=isBrowser] Is this used as `scenarios: { : defaultOptions(...) }`? * @param {boolean} [isBrowser=false] Is this a browser test? * @param {boolean} [isDebug=env.DEBUG_K6] Are we running in debug mode? + * @param {import('k6/options').Options} [otherOptions] Other options to merge in * @return {import('k6/options').Options} * */ export const defaultOptions = ({ @@ -18,16 +19,25 @@ export const defaultOptions = ({ // Browser tests options can only be set inside `scenarios` // https://grafana.com/docs/k6/latest/using-k6-browser/ inScenario = isBrowser, + ...otherOptions } = {}) => { + const thresholds = { + checks: ['rate>0.90'], + http_req_failed: ['rate<0.05'], + }; const baseOptions = isDebug ? { vus: 1, iterations: 1, httpDebug: inScenario ? undefined : 'full', + thresholds, + ...otherOptions, } : { vus: 10, duration: '30s', + thresholds, + ...otherOptions, }; if (inScenario) { // See https://github.com/grafana/k6-learn/blob/main/Modules/III-k6-Intermediate/08-Setting-load-profiles-with-executors.md