diff --git a/packages/wdio-types/src/Options.ts b/packages/wdio-types/src/Options.ts index cae45dd9306..c9b79135fde 100644 --- a/packages/wdio-types/src/Options.ts +++ b/packages/wdio-types/src/Options.ts @@ -1,11 +1,35 @@ +import type http from 'node:http' +import type https from 'node:https' +import type { URL } from 'node:url' + import type { Hooks, ServiceEntry } from './Services.js' import type { ReporterEntry } from './Reporters.js' export type WebDriverLogTypes = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'silent' export type SupportedProtocols = 'webdriver' | 'devtools' | './protocol-stub.js' +export type Agents = { http?: unknown, https?: unknown } export type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'HEAD' | 'DELETE' | 'OPTIONS' | 'TRACE' | 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete' | 'options' | 'trace' +export interface RequestLibOptions { + agent?: Agents + followRedirect?: boolean + headers?: Record + https?: Record + json?: Record + method?: Method + responseType?: 'json' | 'buffer' | 'text' + retry?: { limit: number, methods?: Method[], calculateDelay?: (retryOptions: { computedValue: number }) => number } + searchParams?: Record | URLSearchParams + throwHttpErrors?: boolean + timeout?: { response: number } + url?: URL + path?: string + username?: string + password?: string + body?: unknown +} + export interface RequestLibResponse { statusCode: number body?: Body @@ -104,14 +128,29 @@ export interface WebDriver extends Connection { headers?: { [name: string]: string } + /** + * Allows you to use a custom http/https/http2 [agent](https://www.npmjs.com/package/got#agent) to make requests. + * + * @default + * ```js + * { + * http: new http.Agent({ keepAlive: true }), + * https: new https.Agent({ keepAlive: true }) + * } + * ``` + */ + agent?: { + http: http.Agent, + https: https.Agent + } /** * Function intercepting [HTTP request options](https://github.com/sindresorhus/got#options) before a WebDriver request is made. */ - transformRequest?: (requestOptions: RequestInit) => RequestInit + transformRequest?: (requestOptions: RequestLibOptions) => RequestLibOptions /** * Function intercepting HTTP response objects after a WebDriver response has arrived. */ - transformResponse?: (response: RequestLibResponse, requestOptions: RequestInit) => RequestLibResponse + transformResponse?: (response: RequestLibResponse, requestOptions: RequestLibOptions) => RequestLibResponse /** * Appium direct connect options (see: https://appiumpro.com/editions/86-connecting-directly-to-appium-hosts-in-distributed-environments) diff --git a/packages/webdriver/package.json b/packages/webdriver/package.json index a2fdf82ad42..d45c13393cf 100644 --- a/packages/webdriver/package.json +++ b/packages/webdriver/package.json @@ -36,14 +36,16 @@ "url": "https://github.com/webdriverio/webdriverio/issues" }, "dependencies": { + "@testplane/wdio-config": "workspace:*", + "@testplane/wdio-logger": "workspace:*", "@testplane/wdio-protocols": "workspace:*", "@testplane/wdio-types": "workspace:*", "@testplane/wdio-utils": "workspace:*", "@types/node": "^20.1.0", "@types/ws": "^8.5.3", - "@testplane/wdio-config": "workspace:*", - "@testplane/wdio-logger": "workspace:*", "deepmerge-ts": "^7.0.3", + "got": "12.6.1", + "ky": "0.33.0", "undici": "6.12.0", "ws": "^8.8.0" } diff --git a/packages/webdriver/src/browser.ts b/packages/webdriver/src/browser.ts index ac1ae949f4e..fc5cb8cfe90 100644 --- a/packages/webdriver/src/browser.ts +++ b/packages/webdriver/src/browser.ts @@ -3,7 +3,7 @@ import type { WebSocket } from 'ws' import WebDriver from './index.js' import { BrowserSocket } from './bidi/socket.js' -import { FetchRequest } from './request/web.js' +import { WebRequest } from './request/web.js' export default WebDriver export * from './index.js' @@ -13,7 +13,7 @@ import { environment } from './environment.js' const log = logger('webdriver') environment.value = { - Request: FetchRequest, + Request: WebRequest, Socket: BrowserSocket, createBidiConnection: (webSocketUrl: string, options: unknown) => { log.info(`Connecting to webSocketUrl ${webSocketUrl}`) diff --git a/packages/webdriver/src/command.ts b/packages/webdriver/src/command.ts index e2439810de0..9b87ab16b6c 100644 --- a/packages/webdriver/src/command.ts +++ b/packages/webdriver/src/command.ts @@ -130,20 +130,17 @@ export default function ( * - `deleteSession` calls which should go through in case we retry the command. * - requests that don't require a session. */ - const { isAborted, abortSignal, cleanup } = manageSessionAbortions.call(this) + const { isAborted, cleanup } = manageSessionAbortions.call(this) const requiresSession = endpointUri.includes('/:sessionId/') if (isAborted && command !== 'deleteSession' && requiresSession) { throw new Error(`Trying to run command "${commandCallStructure(command, args)}" after session has been deleted, aborting request without executing it`) } - const request = new environment.value.Request(method, endpoint, body, abortSignal, isHubCommand, { - onPerformance: (data) => this.emit('request.performance', data), - onRequest: (data) => this.emit('request.start', data), - onResponse: (data) => this.emit('request.end', data), - onRetry: (data) => this.emit('request.retry', data) - }) + const request = new environment.value.Request(method, endpoint, body, isHubCommand) + request.on('performance', (...args) => this.emit('request.performance', ...args)) this.emit('command', { command, method, endpoint, body }) log.info('COMMAND', commandCallStructure(command, args)) + /** * use then here so we can better unit test what happens before and after the request */ diff --git a/packages/webdriver/src/constants.ts b/packages/webdriver/src/constants.ts index 1b232764c7b..a3d845591ff 100644 --- a/packages/webdriver/src/constants.ts +++ b/packages/webdriver/src/constants.ts @@ -1,6 +1,6 @@ -import type { Options } from '@testplane/wdio-types' - import { environment } from './environment.js' + +import type { Options } from '@testplane/wdio-types' import type { RemoteConfig } from './types.js' export const DEFAULTS: Options.Definition> = { @@ -92,6 +92,12 @@ export const DEFAULTS: Options.Definition> = { type: 'number', default: 3 }, + /** + * Override default agent + */ + agent: { + type: 'object' + }, /** * Override default agent */ @@ -109,7 +115,7 @@ export const DEFAULTS: Options.Definition> = { */ transformRequest: { type: 'function', - default: (requestOptions: RequestInit) => requestOptions + default: (requestOptions: Options.RequestLibOptions) => requestOptions }, /** * Function transforming the response object after it is received @@ -144,5 +150,10 @@ export const DEFAULTS: Options.Definition> = { } } +export const REG_EXPS = { + commandName: /.*\/session\/[0-9a-f-]+\/(.*)/, + execFn: /return \(([\s\S]*)\)\.apply\(null, arguments\)/ +} + export const ELEMENT_KEY = 'element-6066-11e4-a52e-4f735466cecf' export const SHADOW_ELEMENT_KEY = 'shadow-6066-11e4-a52e-4f735466cecf' diff --git a/packages/webdriver/src/environment.ts b/packages/webdriver/src/environment.ts index e563ab2bd27..0aa2794e466 100644 --- a/packages/webdriver/src/environment.ts +++ b/packages/webdriver/src/environment.ts @@ -1,8 +1,8 @@ import type WebSocket from 'ws' import type { BrowserSocket } from './bidi/socket.js' -import type { FetchRequest } from './request/web.js' - +import type { WebRequest } from './request/web.js' +import type { NodeJSRequest } from './request/node.js' /** * @internal */ @@ -14,7 +14,7 @@ export interface EnvironmentVariables { } export interface EnvironmentDependencies { - Request: typeof FetchRequest, + Request: typeof WebRequest | typeof NodeJSRequest, Socket: typeof BrowserSocket, createBidiConnection: (wsUrl?: string, options?: unknown) => Promise, variables: EnvironmentVariables diff --git a/packages/webdriver/src/node.ts b/packages/webdriver/src/node.ts index 8c16dc113c8..609b9249eb2 100644 --- a/packages/webdriver/src/node.ts +++ b/packages/webdriver/src/node.ts @@ -2,8 +2,8 @@ import os from 'node:os' import ws from 'ws' import WebDriver from './index.js' -import { FetchRequest } from './request/node.js' -import { FetchRequest as WebFetchRequest } from './request/web.js' +import { NodeJSRequest } from './request/node.js' +import { WebRequest } from './request/web.js' import { createBidiConnection } from './node/bidi.js' import type { BrowserSocket } from './bidi/socket.js' @@ -22,11 +22,11 @@ environment.value = { */ process.env.WDIO_USE_NATIVE_FETCH || /** - * For unit tests we use the WebFetchRequest implementation as we can better mock the + * For unit tests we use the WebRequest implementation as we can better mock the * requests in the unit tests. */ process.env.WDIO_UNIT_TESTS - ) ? WebFetchRequest : FetchRequest, + ) ? WebRequest : NodeJSRequest, Socket: ws as unknown as typeof BrowserSocket, createBidiConnection, variables: { @@ -34,4 +34,3 @@ environment.value = { PROXY_URL: process.env.HTTP_PROXY || process.env.HTTPS_PROXY } } - diff --git a/packages/webdriver/src/request/constants.ts b/packages/webdriver/src/request/constants.ts deleted file mode 100644 index 5e2621121ce..00000000000 --- a/packages/webdriver/src/request/constants.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** - * retrieved from https://github.com/sindresorhus/ky/blob/3ba40cc6333cf1847c02c51744e22ab7c04407f5/source/utils/normalize.ts#L10 - */ -export const RETRYABLE_STATUS_CODES = [408, 413, 429, 500, 502, 503, 504] -/** - * retrieved from https://github.com/sindresorhus/got/blob/89b7fdfd4e7ea4e76258f50b70ae8a1d2aea8125/source/core/options.ts#L392C1-L399C37 - */ -export const RETRYABLE_ERROR_CODES = [ - 'ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', - 'ENETUNREACH', 'EAI_AGAIN', - // additional error codes we like to retry - 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET' -] - -export const REG_EXPS = { - commandName: /.*\/session\/[0-9a-f-]+\/(.*)/, - execFn: /return \(([\s\S]*)\)\.apply\(null, arguments\)/ -} diff --git a/packages/webdriver/src/request/error.ts b/packages/webdriver/src/request/error.ts deleted file mode 100644 index b42d28f5613..00000000000 --- a/packages/webdriver/src/request/error.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { transformCommandLogResult } from '@testplane/wdio-utils' - -import { REG_EXPS } from './constants.js' - -abstract class WebDriverError extends Error { - abstract url: URL - abstract opts: RequestInit - - /** - * return timeout error with information about the executing command on which the test hangs - */ - computeErrorMessage() { - const cmdName = this.#getExecCmdName() - const cmdArgs = this.#getExecCmdArgs(this.opts) - - const cmdInfoMsg = `when running "${cmdName}" with method "${this.opts.method}"` - const cmdArgsMsg = cmdArgs ? ` and args ${cmdArgs}` : '' - - return `WebDriverError: ${this.message} ${cmdInfoMsg}${cmdArgsMsg}` - } - - #getExecCmdName(): string { - const { href } = this.url - const res = href.match(REG_EXPS.commandName) || [] - - return res[1] || href - } - - #getExecCmdArgs(requestOptions: RequestInit): string { - const { body: cmdJson } = requestOptions as unknown as { body: Record } - - if (typeof cmdJson !== 'object') { - return '' - } - - const transformedRes = transformCommandLogResult(cmdJson) - - if (typeof transformedRes === 'string') { - return transformedRes - } - - if (typeof cmdJson.script === 'string') { - const scriptRes = cmdJson.script.match(REG_EXPS.execFn) || [] - - return `"${scriptRes[1] || cmdJson.script}"` - } - - return Object.keys(cmdJson).length ? `"${JSON.stringify(cmdJson)}"` : '' - } -} - -export class WebDriverRequestError extends WebDriverError { - url: URL - opts: RequestInit - - statusCode?: number - body?: unknown - code?: string - - constructor (err: Error, url: URL, opts: RequestInit) { - let message = err.message - if (err.message === 'fetch failed') { - message = `Failed to fetch [${opts.method}] ${url.href}: please make sure you have a WebDriver compatible server running on ${url.origin}` - } - - super(message) - this.url = url - this.opts = opts - - const errorCode = typeof err.cause === 'object' && err.cause && 'code' in err.cause && typeof err.cause.code === 'string' - ? err.cause.code - : 'code' in err && typeof err.code === 'string' - ? err.code - : undefined - if (errorCode) { - this.code = errorCode - this.message = errorCode === 'UND_ERR_CONNECT_TIMEOUT' - ? 'Request timed out! Consider increasing the "connectionRetryTimeout" option.' - : 'Request failed with error code ' + errorCode - } - - this.message = this.computeErrorMessage() - } -} - -export class WebDriverResponseError extends WebDriverError { - url: URL - opts: RequestInit - constructor (response: unknown, url: URL, opts: RequestInit) { - const errorObj: { message?: string, error?: string, class?: string, name?: string } = !response || typeof response !== 'object' || !('body' in response) || !response.body - ? new Error('Response has empty body') - : typeof response.body === 'string' && response.body.length - ? new Error(response.body) - : typeof response.body !== 'object' - ? new Error('Unknown error') - : 'value' in response.body && response.body.value - ? response.body.value - : response.body - - /** - * e.g. in Firefox or Safari, error are following the following structure: - * ``` - * { - * value: { - * error: '...', - * message: '...', - * stacktrace: '...' - * } - * } - * ``` - */ - let errorMessage = errorObj.message || errorObj.error || errorObj.class || 'unknown error' - - /** - * Improve Chromedriver's error message for an invalid selector - * - * Chrome: - * error: 'invalid argument' - * message: 'invalid argument: invalid locator\n (Session info: chrome=122.0.6261.94)' - * Firefox: - * error: 'invalid selector' - * message: 'Given xpath expression "//button" is invalid: NotSupportedError: Operation is not supported' - * Safari: - * error: 'timeout' - * message: '' - */ - if (typeof errorMessage === 'string' && errorMessage.includes('invalid locator')) { - const requestOptions = opts.body as unknown as { using: string; value: string } - errorMessage = ( - `The selector "${requestOptions.value}" used with strategy "${requestOptions.using}" is invalid!` - ) - } - - super(errorMessage) - if (errorObj.error) { - this.name = errorObj.error - } else if (errorMessage && errorMessage.includes('stale element reference')) { - this.name = 'stale element reference' - } else { - this.name = errorObj.name || 'WebDriver Error' - } - - Error.captureStackTrace(this, WebDriverResponseError) - this.url = url - this.opts = opts - this.message = this.computeErrorMessage() - } -} diff --git a/packages/webdriver/src/request/index.ts b/packages/webdriver/src/request/index.ts new file mode 100644 index 00000000000..847aad667ec --- /dev/null +++ b/packages/webdriver/src/request/index.ts @@ -0,0 +1,302 @@ +import path from 'node:path' +import { EventEmitter } from 'node:events' +import { WebDriverProtocol } from '@testplane/wdio-protocols' +import { URL } from 'node:url' +import logger from '@testplane/wdio-logger' +import { transformCommandLogResult } from '@testplane/wdio-utils' +import type { Options } from '@testplane/wdio-types' + +import { isSuccessfulResponse, getErrorFromResponseBody, getTimeoutError } from '../utils.js' +import pkg from '../../package.json' with { type: 'json' } + +import type { WebDriverResponse, RequestLibOptions, RequestLibResponse, RequestOptions } from './types.js' + +type Agents = Options.Agents + +const RETRY_METHODS = [ + 'GET', + 'POST', + 'PUT', + 'HEAD', + 'DELETE', + 'OPTIONS', + 'TRACE' +] as Options.Method[] + +export class RequestLibError extends Error { + statusCode?: number + body?: unknown + code?: string +} + +export const COMMANDS_WITHOUT_RETRY = [ + findCommandPathByName('performActions'), +] +const MAX_RETRY_TIMEOUT = 100 // 100ms +const DEFAULT_HEADERS = { + 'Content-Type': 'application/json; charset=utf-8', + 'Connection': 'keep-alive', + 'Accept': 'application/json', + 'User-Agent': 'webdriver/' + pkg.version +} + +const log = logger('webdriver') + +export default abstract class WebDriverRequest extends EventEmitter { + body?: Record + method: string + endpoint: string + isHubCommand: boolean + requiresSessionId: boolean + defaultAgents?: Agents + defaultOptions: RequestLibOptions = { + followRedirect: true, + responseType: 'json', + throwHttpErrors: false + } + + constructor (method: string, endpoint: string, body?: Record, isHubCommand: boolean = false) { + super() + this.body = body + this.method = method + this.endpoint = endpoint + this.isHubCommand = isHubCommand + this.requiresSessionId = Boolean(this.endpoint.match(/:sessionId/)) + } + + async makeRequest (options: RequestOptions, sessionId?: string) { + let fullRequestOptions: RequestLibOptions = Object.assign( + { method: this.method }, + this.defaultOptions, + await this._createOptions(options, sessionId) + ) + if (typeof options.transformRequest === 'function') { + fullRequestOptions = options.transformRequest(fullRequestOptions) + } + + this.emit('request', fullRequestOptions) + return this._request(fullRequestOptions, options.transformResponse, options.connectionRetryCount, 0) + } + + protected async _createOptions (options: RequestOptions, sessionId?: string, isBrowser: boolean = false): Promise { + const agent = isBrowser ? undefined : (options.agent || this.defaultAgents) + const searchParams = isBrowser ? + undefined : + (typeof options.queryParams === 'object' ? options.queryParams : {}) + const requestOptions: RequestLibOptions = { + https: {}, + agent, + headers: { + ...DEFAULT_HEADERS, + ...(typeof options.headers === 'object' ? options.headers : {}) + }, + searchParams, + retry: { + limit: options.connectionRetryCount as number, + /** + * this enables request retries for all commands except for the + * ones defined in `COMMANDS_WITHOUT_RETRY` since they have their + * own retry mechanism. Including a request based retry mechanism + * here also ensures we retry if e.g. a connection to the server + * can't be established at all. + */ + ...(COMMANDS_WITHOUT_RETRY.includes(this.endpoint) + ? {} + : { + methods: RETRY_METHODS, + calculateDelay: ({ computedValue }) => Math.min(MAX_RETRY_TIMEOUT, computedValue / 10) + } + ), + }, + timeout: { response: options.connectionRetryTimeout as number } + } + + /** + * only apply body property if existing + */ + if (this.body && (Object.keys(this.body).length || this.method === 'POST')) { + const contentLength = Buffer.byteLength(JSON.stringify(this.body), 'utf8') + requestOptions.json = this.body + requestOptions.headers!['Content-Length'] = `${contentLength}` + } + + /** + * if we don't have a session id we set it here, unless we call commands that don't require session ids, for + * example /sessions. The call to /sessions is not connected to a session itself and it therefore doesn't + * require it + */ + let endpoint = this.endpoint + if (this.requiresSessionId) { + if (!sessionId) { + throw new Error('A sessionId is required for this command') + } + endpoint = endpoint.replace(':sessionId', sessionId) + } + + requestOptions.url = new URL(`${options.protocol}://${options.hostname}:${options.port}${this.isHubCommand ? this.endpoint : path.join(options.path || '', endpoint)}`) + + /** + * send authentication credentials only when creating new session + */ + if (this.endpoint === '/session' && options.user && options.key) { + requestOptions.username = options.user + requestOptions.password = options.key + } + + /** + * if the environment variable "STRICT_SSL" is defined as "false", it doesn't require SSL certificates to be valid. + * Or the requestOptions has strictSSL for an environment which cannot get the environment variable correctly like on an Electron app. + */ + requestOptions.https!.rejectUnauthorized = !( + options.strictSSL === false || + process.env.STRICT_SSL === 'false' || + process.env.strict_ssl === 'false' + ) + + return requestOptions + } + + protected async _libRequest(url: URL, options: RequestLibOptions): Promise { // eslint-disable-line @typescript-eslint/no-unused-vars + throw new Error('This function must be implemented') + } + + protected _libPerformanceNow(): number { + throw new Error('This function must be implemented') + } + + private async _request ( + fullRequestOptions: RequestLibOptions, + transformResponse?: (response: RequestLibResponse, requestOptions: RequestLibOptions) => RequestLibResponse, + totalRetryCount = 0, + retryCount = 0 + ): Promise { + log.info(`[${fullRequestOptions.method}] ${(fullRequestOptions.url as URL).href}`) + + if (fullRequestOptions.json && Object.keys(fullRequestOptions.json).length) { + log.info('DATA', transformCommandLogResult(fullRequestOptions.json)) + } + + const { url, ...requestLibOptions } = fullRequestOptions + const startTime = this._libPerformanceNow() + let response = await this._libRequest(url!, requestLibOptions) + .catch((err: RequestLibError) => err) + const durationMillisecond = this._libPerformanceNow() - startTime + + /** + * handle retries for requests + * @param {Error} error error object that causes the retry + * @param {string} msg message that is being shown as warning to user + */ + const retry = (error: Error, msg: string) => { + /** + * stop retrying if totalRetryCount was exceeded or there is no reason to + * retry, e.g. if sessionId is invalid + */ + if (retryCount >= totalRetryCount || error.message.includes('invalid session id')) { + log.error(`Request failed with status ${response.statusCode} due to ${error}`) + this.emit('response', { error }) + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) + throw error + } + + ++retryCount + this.emit('retry', { error, retryCount }) + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) + log.warn(msg) + log.info(`Retrying ${retryCount}/${totalRetryCount}`) + return this._request(fullRequestOptions, transformResponse, totalRetryCount, retryCount) + } + + /** + * handle request errors + */ + if (response instanceof Error) { + /** + * handle timeouts + */ + if ((response as RequestLibError).code === 'ETIMEDOUT') { + const error = getTimeoutError(response, fullRequestOptions) + + return retry(error, 'Request timed out! Consider increasing the "connectionRetryTimeout" option.') + } + + /** + * throw if request error is unknown + */ + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: false, error: response, retryCount }) + throw response + } + + if (typeof transformResponse === 'function') { + response = transformResponse(response, fullRequestOptions) as RequestLibResponse + } + + const error = getErrorFromResponseBody(response.body, fullRequestOptions.json) + + /** + * retry connection refused errors + */ + if (error.message === 'java.net.ConnectException: Connection refused: connect') { + return retry(error, 'Connection to Selenium Standalone server was refused.') + } + + /** + * hub commands don't follow standard response formats + * and can have empty bodies + */ + if (this.isHubCommand) { + /** + * if body contains HTML the command was called on a node + * directly without using a hub, therefore throw + */ + if (typeof response.body === 'string' && response.body.startsWith('')) { + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) + return Promise.reject(new Error('Command can only be called to a Selenium Hub')) + } + + return { value: response.body || null } + } + + /** + * Resolve only if successful response + */ + if (isSuccessfulResponse(response.statusCode, response.body)) { + this.emit('response', { result: response.body }) + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: true, retryCount }) + return response.body as WebDriverResponse + } + + /** + * stop retrying as this will never be successful. + * we will handle this at the elementErrorHandler + */ + if (error.name === 'stale element reference') { + log.warn('Request encountered a stale element - terminating request') + this.emit('response', { error }) + this.emit('performance', { request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) + throw error + } + + /** + * Move out of bounds errors can be excluded from the request retry mechanism as + * it likely does not changes anything and the error is handled within the command. + */ + if (error.name === 'move target out of bounds') { + throw error + } + + return retry(error, `Request failed with status ${response.statusCode} due to ${error.message}`) + } +} + +function findCommandPathByName (commandName: string) { + const command = Object.entries(WebDriverProtocol).find( + ([, command]) => Object.values(command).find( + (cmd) => cmd.command === commandName)) + + if (!command) { + throw new Error(`Couldn't find command "${commandName}"`) + } + + return command[0] +} diff --git a/packages/webdriver/src/request/node.ts b/packages/webdriver/src/request/node.ts index 31b769753ed..a8b6120a84c 100644 --- a/packages/webdriver/src/request/node.ts +++ b/packages/webdriver/src/request/node.ts @@ -1,37 +1,42 @@ import dns from 'node:dns' -import { fetch, Agent, type RequestInit as UndiciRequestInit, ProxyAgent } from 'undici' +import http from 'node:http' +import https from 'node:https' +import { performance } from 'node:perf_hooks' +import type { URL } from 'node:url' -import { environment } from '../environment.js' -import { WebDriverRequest } from './request.js' -import type { RequestOptions } from './types.js' +import got, { type OptionsOfTextResponseBody } from 'got' +import type { Options } from '@testplane/wdio-types' + +import WebDriverRequest, { RequestLibError } from './index.js' // As per this https://github.com/node-fetch/node-fetch/issues/1624#issuecomment-1407717012 we are setting ipv4first as default IP resolver. // This can be removed when we drop Node18 support. dns.setDefaultResultOrder('ipv4first') -/** - * Node implementation of WebDriverRequest using undici fetch - */ -export class FetchRequest extends WebDriverRequest { - fetch (url: URL, opts: RequestInit) { - return fetch(url, opts as UndiciRequestInit) as unknown as Promise - } +const agents: Options.Agents = { + http: new http.Agent({ keepAlive: true }), + https: new https.Agent({ keepAlive: true }) +} - async createOptions (options: RequestOptions, sessionId?: string, isBrowser: boolean = false) { - const { url, requestOptions } = await super.createOptions(options, sessionId, isBrowser) +export class NodeJSRequest extends WebDriverRequest { + constructor (method: string, endpoint: string, body?: Record, isHubCommand: boolean = false) { + super(method, endpoint, body, isHubCommand) + this.defaultAgents = agents + } - /** - * Use a proxy agent if we have a proxy url set - */ - const dispatcher = environment.value.variables.PROXY_URL - ? new ProxyAgent(environment.value.variables.PROXY_URL) - : new Agent({ - connectTimeout: options.connectionRetryTimeout, - headersTimeout: options.connectionRetryTimeout, - bodyTimeout: options.connectionRetryTimeout, - }) + protected async _libRequest (url: URL, opts: Options.RequestLibOptions) { + try { + return (await got(url, opts as OptionsOfTextResponseBody)) as Options.RequestLibResponse + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (!(err instanceof Error)) { + throw new RequestLibError(err.message || err) + } + throw err + } + } - ;(requestOptions as UndiciRequestInit).dispatcher = dispatcher - return { url, requestOptions } + protected _libPerformanceNow(): number { + return performance.now() } } diff --git a/packages/webdriver/src/request/polyfill.ts b/packages/webdriver/src/request/polyfill.ts deleted file mode 100644 index db83e72eb5d..00000000000 --- a/packages/webdriver/src/request/polyfill.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Polyfill for AbortSignal.any() - * - * Creates a new AbortSignal that aborts when any of the given signals abort. - * - * @param signals - An array of AbortSignal objects - * @returns A new AbortSignal that aborts when any of the input signals abort - */ -if (!AbortSignal.any) { - AbortSignal.any = function (signals: AbortSignal[]): AbortSignal { - // Validate input - if (!signals || !Array.isArray(signals)) { - throw new TypeError('AbortSignal.any requires an array of AbortSignal objects') - } - - // Create a new controller for our combined signal - const controller = new AbortController() - - // If any signal is already aborted, abort immediately - if (signals.some(signal => signal.aborted)) { - controller.abort() - return controller.signal - } - - // Set up listeners for each signal - const listeners = signals.map(signal => { - const listener = () => { - // When any signal aborts, abort our controller - // and forward the abort reason if available - if ('reason' in signal && signal.reason !== undefined) { - controller.abort(signal.reason) - } else { - controller.abort() - } - - // Clean up other listeners when one signal aborts - cleanup() - } - - signal.addEventListener('abort', listener) - return { signal, listener } - }) - - // Function to remove all event listeners - const cleanup = () => { - listeners.forEach(({ signal, listener }) => { - signal.removeEventListener('abort', listener) - }) - } - - // Make sure to clean up if our combined signal is aborted - controller.signal.addEventListener('abort', cleanup) - - return controller.signal - } -} diff --git a/packages/webdriver/src/request/request.ts b/packages/webdriver/src/request/request.ts deleted file mode 100644 index 5af5c4b380d..00000000000 --- a/packages/webdriver/src/request/request.ts +++ /dev/null @@ -1,288 +0,0 @@ -import logger from '@testplane/wdio-logger' -import { transformCommandLogResult, sleep } from '@testplane/wdio-utils' -import type { Options } from '@testplane/wdio-types' - -import { WebDriverResponseError, WebDriverRequestError } from './error.js' -import { RETRYABLE_STATUS_CODES, RETRYABLE_ERROR_CODES } from './constants.js' -import type { WebDriverResponse, RequestLibResponse, RequestOptions, RequestEventHandler } from './types.js' - -import { isSuccessfulResponse } from '../utils.js' -import { DEFAULTS } from '../constants.js' -import pkg from '../../package.json' with { type: 'json' } - -import './polyfill.js' - -const ERRORS_TO_EXCLUDE_FROM_RETRY = [ - 'detached shadow root', - 'move target out of bounds' -] - -const DEFAULT_HEADERS = { - 'Content-Type': 'application/json; charset=utf-8', - 'Connection': 'keep-alive', - 'Accept': 'application/json', - 'User-Agent': 'webdriver/' + pkg.version -} - -const log = logger('webdriver') - -export abstract class WebDriverRequest { - protected abstract fetch(url: URL, opts: RequestInit): Promise - - body?: Record - method: string - endpoint: string - isHubCommand: boolean - requiresSessionId: boolean - eventHandler: RequestEventHandler - abortSignal?: AbortSignal - constructor ( - method: string, - endpoint: string, - body?: Record, - abortSignal?: AbortSignal, - isHubCommand: boolean = false, - eventHandler: RequestEventHandler = {} - ) { - this.body = body - this.method = method - this.endpoint = endpoint - this.isHubCommand = isHubCommand - this.requiresSessionId = Boolean(this.endpoint.match(/:sessionId/)) - this.eventHandler = eventHandler - this.abortSignal = abortSignal - } - - async makeRequest (options: RequestOptions, sessionId?: string) { - const { url, requestOptions } = await this.createOptions(options, sessionId) - this.eventHandler.onRequest?.(requestOptions) - return this._request(url, requestOptions, options.transformResponse, options.connectionRetryCount, 0) - } - - async createOptions (options: RequestOptions, sessionId?: string, isBrowser: boolean = false): Promise<{url: URL; requestOptions: RequestInit;}> { - const timeout = options.connectionRetryTimeout || DEFAULTS.connectionRetryTimeout.default as number - const requestOptions: RequestInit = { - method: this.method, - redirect: 'follow', - signal: AbortSignal.any([ - AbortSignal.timeout(timeout), - ...(this.abortSignal ? [this.abortSignal] : []) - ]) - } - - const requestHeaders: HeadersInit = new Headers({ - ...DEFAULT_HEADERS, - ...(typeof options.headers === 'object' ? options.headers : {}) - }) - - const searchParams = isBrowser ? undefined : (typeof options.queryParams === 'object' ? options.queryParams : undefined) - - /** - * only apply body property if existing - */ - if (this.body && (Object.keys(this.body).length || this.method === 'POST')) { - const contentLength = Buffer.byteLength(JSON.stringify(this.body), 'utf8') - requestOptions.body = this.body as unknown as BodyInit - requestHeaders.set('Content-Length', `${contentLength}`) - } - - /** - * if we don't have a session id we set it here, unless we call commands that don't require session ids, for - * example /sessions. The call to /sessions is not connected to a session itself and it therefore doesn't - * require it - */ - let endpoint = this.endpoint - if (this.requiresSessionId) { - if (!sessionId) { - throw new Error('A sessionId is required for this command') - } - endpoint = endpoint.replace(':sessionId', sessionId) - } - - const url = new URL(`${options.protocol}://${options.hostname}:${options.port}${this.isHubCommand ? this.endpoint : `${options.path || ''}/${endpoint}`.replace(/(\/){2,}/g, '/')}`) - - if (searchParams) { - url.search = new URLSearchParams(searchParams).toString() - } - - /** - * send authentication credentials only when creating new session - */ - if (this.endpoint === '/session' && options.user && options.key) { - requestHeaders.set('Authorization', 'Basic ' + btoa(options.user + ':' + options.key)) - } - - requestOptions.headers = requestHeaders - - return { - url, - requestOptions: typeof options.transformRequest === 'function' - ? options.transformRequest(requestOptions) - : requestOptions - } - } - - protected async _libRequest (url: URL, opts: RequestInit): Promise { - try { - const response = await this.fetch(url, { - method: opts.method, - body: JSON.stringify(opts.body), - headers: opts.headers as Record, - signal: opts.signal, - redirect: opts.redirect - }) - - // Cloning the response to prevent body unusable error - const resp = response.clone() - - return { - statusCode: resp.status, - body: await resp.json() ?? {}, - } as Options.RequestLibResponse - } catch (err) { - if (!(err instanceof Error)) { - throw new WebDriverRequestError( - new Error(`Failed to fetch ${url.href}: ${(err as Error).message || err || 'Unknown error'}`), - url, - opts - ) - } - - throw new WebDriverRequestError(err, url, opts) - } - } - - protected async _request ( - url: URL, - fullRequestOptions: RequestInit, - transformResponse?: (response: RequestLibResponse, requestOptions: RequestInit) => RequestLibResponse, - totalRetryCount = 0, - retryCount = 0 - ): Promise { - log.info(`[${fullRequestOptions.method}] ${(url as URL).href}`) - - if (fullRequestOptions.body && Object.keys(fullRequestOptions.body).length) { - log.info('DATA', transformCommandLogResult(fullRequestOptions.body)) - } - - const { ...requestLibOptions } = fullRequestOptions - const startTime = performance.now() - let response = await this._libRequest(url!, requestLibOptions) - .catch((err: WebDriverRequestError) => err) - const durationMillisecond = performance.now() - startTime - - /** - * handle retries for requests - * @param {Error} error error object that causes the retry - * @param {string} msg message that is being shown as warning to user - */ - const retry = async (error: Error) => { - /** - * stop retrying if totalRetryCount was exceeded or there is no reason to - * retry, e.g. if sessionId is invalid - */ - if (retryCount >= totalRetryCount || error.message.includes('invalid session id')) { - log.error(error.message) - this.eventHandler.onResponse?.({ error }) - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) - throw error - } - - if (retryCount > 0) { - /* - * Exponential backoff with a minimum of 500ms and a maximum of 10_000ms. - */ - await sleep(Math.min(10000, 250 * Math.pow(2, retryCount))) - } - - ++retryCount - - this.eventHandler.onRetry?.({ error, retryCount }) - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) - log.warn(error.message) - log.info(`Retrying ${retryCount}/${totalRetryCount}`) - return this._request(url, fullRequestOptions, transformResponse, totalRetryCount, retryCount) - } - - /** - * handle request errors - */ - if (response instanceof Error) { - const resError = response as WebDriverRequestError - - /** - * retry failed requests, only if: - * - the abort signal is not aborted - * - the error code or status code is retryable - */ - if ( - !(this.abortSignal && this.abortSignal.aborted) && - ( - (resError.code && RETRYABLE_ERROR_CODES.includes(resError.code)) || - (resError.statusCode && RETRYABLE_STATUS_CODES.includes(resError.statusCode)) - ) - ) { - return retry(resError) - } - - /** - * throw if request error is unknown - */ - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: false, error: response, retryCount }) - throw response - } - - if (typeof transformResponse === 'function') { - response = transformResponse(response, fullRequestOptions) as RequestLibResponse - } - - /** - * Resolve only if successful response - */ - if (isSuccessfulResponse(response.statusCode, response.body)) { - this.eventHandler.onResponse?.({ result: response.body }) - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: true, retryCount }) - return response.body as WebDriverResponse - } - - const error = new WebDriverResponseError(response, url, fullRequestOptions) - - /** - * hub commands don't follow standard response formats - * and can have empty bodies - */ - if (this.isHubCommand) { - /** - * if body contains HTML the command was called on a node - * directly without using a hub, therefore throw - */ - if (typeof response.body === 'string' && response.body.startsWith('')) { - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) - return Promise.reject(new Error('Command can only be called to a Selenium Hub')) - } - - return { value: response.body || null } - } - - /** - * stop retrying as this will never be successful. - * we will handle this at the elementErrorHandler - */ - if (error.name === 'stale element reference') { - log.warn('Request encountered a stale element - terminating request') - this.eventHandler.onResponse?.({ error }) - this.eventHandler.onPerformance?.({ request: fullRequestOptions, durationMillisecond, success: false, error, retryCount }) - throw error - } - - /** - * some errors can be excluded from the request retry mechanism as - * it likely does not changes anything and the error is handled within the command. - */ - if (ERRORS_TO_EXCLUDE_FROM_RETRY.includes(error.name)) { - throw error - } - - return retry(error) - } -} diff --git a/packages/webdriver/src/request/types.ts b/packages/webdriver/src/request/types.ts index 654a8a2d496..5a072b3d74a 100644 --- a/packages/webdriver/src/request/types.ts +++ b/packages/webdriver/src/request/types.ts @@ -27,7 +27,30 @@ export interface RequestEventHandler { onPerformance?: (ev: RequestPerformanceEvent) => void } -export type RequestStartEvent = RequestInit +export type RequestStartEvent = RequestLibOptions export type RequestEndEvent = { result?: unknown, error?: Error } export type RequestRetryEvent = { error: Error, retryCount: number } export type RequestPerformanceEvent = { request: RequestInit, durationMillisecond: number, success: boolean, error?: Error, retryCount: number } + +export type Agents = { http?: unknown, https?: unknown } + +export type Method = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'HEAD' | 'DELETE' | 'OPTIONS' | 'TRACE' | 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete' | 'options' | 'trace' + +export interface RequestLibOptions { + agent?: Agents + followRedirect?: boolean + headers?: Record + https?: Record + json?: Record + method?: Method + responseType?: 'json' | 'buffer' | 'text' + retry?: { limit: number, methods?: Method[], calculateDelay?: (retryOptions: { computedValue: number }) => number } + searchParams?: Record | URLSearchParams + throwHttpErrors?: boolean + timeout?: { response: number } + url?: URL + path?: string + username?: string + password?: string + body?: unknown +} diff --git a/packages/webdriver/src/request/web.ts b/packages/webdriver/src/request/web.ts index 7fb66e081df..1d7867f5233 100644 --- a/packages/webdriver/src/request/web.ts +++ b/packages/webdriver/src/request/web.ts @@ -1,10 +1,58 @@ -import { WebDriverRequest } from './request.js' - -/** - * Cross platform implementation of a fetch based request using native fetch - */ -export class FetchRequest extends WebDriverRequest { - fetch (url: URL, opts: RequestInit) { - return fetch(url, opts) +import type { Options as KyOptions } from 'ky' +import ky from 'ky' +import logger from '@testplane/wdio-logger' +import WebDriverRequest from './index.js' +import type { RequestOptions, RequestLibOptions } from './types.js' + +const log = logger('webdriver') + +const UNSUPPORTED_OPTS: Array = [ + 'agent', + 'responseType', + 'searchParams', +] + +export class WebRequest extends WebDriverRequest { + constructor (method: string, endpoint: string, body?: Record, isHubCommand: boolean = false) { + super(method, endpoint, body, isHubCommand) + } + + protected async _createOptions (options: RequestOptions, sessionId?: string): Promise { + return super._createOptions(options, sessionId, true) + } + + protected async _libRequest (url: URL, options: RequestLibOptions) { + const kyOptions: KyOptions = {} + + for (const opt of Object.keys(options) as Array) { + if ( + typeof options[opt] !== 'undefined' && + UNSUPPORTED_OPTS.includes(opt) && + options[opt] !== this.defaultOptions[opt] + ) { + log.info(`Browser-based webdriver does not support the '${String(opt)}' option; behavior may be unexpected`) + continue + } + // @ts-expect-error + kyOptions[opt] = options[opt] + } + + if (options.username && options.password) { + const encodedAuth = Buffer.from(`${options.username}:${options.password}`, 'utf8').toString('base64') + kyOptions.headers = { + ...kyOptions.headers, + Authorization: `Basic ${encodedAuth}` + } + } + + const res = await ky(url, kyOptions) + return { + statusCode: res.status, + body: await res.json(), + } + } + + protected _libPerformanceNow(): number { + return performance.now() } } diff --git a/packages/webdriver/src/utils.ts b/packages/webdriver/src/utils.ts index c579cf9eb78..d5fd70d9052 100644 --- a/packages/webdriver/src/utils.ts +++ b/packages/webdriver/src/utils.ts @@ -8,12 +8,15 @@ import { SauceLabsProtocol, SeleniumProtocol, GeckoProtocol, WebDriverBidiProtocol } from '@testplane/wdio-protocols' import { CAPABILITY_KEYS } from '@testplane/wdio-protocols' +import { transformCommandLogResult } from '@testplane/wdio-utils' import type { Options } from '@testplane/wdio-types' -import command from './command.js' import { environment } from './environment.js' +import command from './command.js' import { BidiHandler } from './bidi/handler.js' +import { REG_EXPS } from './constants.js' import type { Event } from './bidi/localTypes.js' +import type { WebDriverResponse } from './request/types.js' import type { Client, JSONWPCommandError, SessionFlags, RemoteConfig } from './types.js' const log = logger('webdriver') @@ -79,6 +82,7 @@ export async function startWebDriverSession (params: RemoteConfig): Promise<{ se } validateCapabilities(w3cCaps.alwaysMatch) + const sessionRequest = new environment.value.Request( 'POST', '/session', @@ -275,6 +279,68 @@ export function getPrototype ({ isW3C, isChromium, isFirefox, isMobile, isSauce, return prototype } +/** + * helper method to determine the error from webdriver response + * @param {Object} body body object + * @return {Object} error + */ +export function getErrorFromResponseBody (body: unknown, requestOptions: unknown) { + if (!body) { + return new Error('Response has empty body') + } + + if (typeof body === 'string' && body.length) { + return new Error(body) + } + + if (typeof body !== 'object') { + return new Error('Unknown error') + } + + return new CustomRequestError(body as WebDriverResponse, requestOptions) +} + +//Exporting for testability +export class CustomRequestError extends Error { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + constructor (body: WebDriverResponse, requestOptions: any) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const errorObj = (body.value || body) as any + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let errorMessage = errorObj.message || errorObj.class || 'unknown error' + + /** + * Improve Chromedriver's error message for an invalid selector + * + * Chrome: + * error: 'invalid argument' + * message: 'invalid argument: invalid locator\n (Session info: chrome=122.0.6261.94)' + * Firefox: + * error: 'invalid selector' + * message: 'Given xpath expression "//button" is invalid: NotSupportedError: Operation is not supported' + * Safari: + * error: 'timeout' + * message: '' + */ + if (typeof errorObj.message === 'string' && errorObj.message.includes('invalid locator')) { + errorMessage = ( + `The selector "${requestOptions.value}" used with strategy "${requestOptions.using}" is invalid!` + ) + } + + super(errorMessage) + if (errorObj.error) { + this.name = errorObj.error + } else if (errorObj.message && errorObj.message.includes('stale element reference')) { + this.name = 'stale element reference' + } else { + this.name = errorObj.name || 'WebDriver Error' + } + + Error.captureStackTrace(this, CustomRequestError) + } +} + /** * return all supported flags and return them in a format so we can attach them * to the instance protocol @@ -386,6 +452,49 @@ export const getSessionError = (err: JSONWPCommandError, params: Partial { + const cmdName = getExecCmdName(requestOptions) + const cmdArgs = getExecCmdArgs(requestOptions) + + const cmdInfoMsg = `when running "${cmdName}" with method "${requestOptions.method}"` + const cmdArgsMsg = cmdArgs ? ` and args ${cmdArgs}` : '' + + const timeoutErr = new Error(`${error.message} ${cmdInfoMsg}${cmdArgsMsg}`) + return Object.assign(timeoutErr, error) +} + +function getExecCmdName(requestOptions: Options.RequestLibOptions): string { + const { href } = requestOptions.url as URL + const res = href.match(REG_EXPS.commandName) || [] + + return res[1] || href +} + +function getExecCmdArgs(requestOptions: Options.RequestLibOptions): string { + const { json: cmdJson } = requestOptions + + if (typeof cmdJson !== 'object') { + return '' + } + + const transformedRes = transformCommandLogResult(cmdJson) + + if (typeof transformedRes === 'string') { + return transformedRes + } + + if (typeof cmdJson.script === 'string') { + const scriptRes = cmdJson.script.match(REG_EXPS.execFn) || [] + + return `"${scriptRes[1] || cmdJson.script}"` + } + + return Object.keys(cmdJson).length ? `"${JSON.stringify(cmdJson)}"` : '' +} + /** * Enhance the monad with WebDriver Bidi primitives if a connection can be established successfully * @param socketUrl url to bidi interface diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b6b72e2fab..09518aef20b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1517,6 +1517,12 @@ importers: deepmerge-ts: specifier: ^7.0.3 version: 7.1.4 + got: + specifier: 12.6.1 + version: 12.6.1 + ky: + specifier: 0.33.0 + version: 0.33.0 undici: specifier: 6.12.0 version: 6.12.0 @@ -10018,6 +10024,10 @@ packages: kolorist@1.8.0: resolution: {integrity: sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==} + ky@0.33.0: + resolution: {integrity: sha512-peKzuOlN/q3Q3jOgi4t0cp6DOgif5rVnmiSIsjsmkiOcdnSjkrKSUqQmRWYCTqjUtR9b3xQQr8aj7KwSW1r49A==} + engines: {node: '>=14.16'} + ky@0.33.3: resolution: {integrity: sha512-CasD9OCEQSFIam2U8efFK81Yeg8vNMTBUqtMOHlrcWQHqUX3HeCl9Dr31u4toV7emlH8Mymk5+9p0lL6mKb/Xw==} engines: {node: '>=14.16'} @@ -25012,6 +25022,8 @@ snapshots: kolorist@1.8.0: {} + ky@0.33.0: {} + ky@0.33.3: optional: true @@ -27741,7 +27753,6 @@ snapshots: typed-query-selector: 2.12.0 ws: 8.18.0 transitivePeerDependencies: - - bare-buffer - bufferutil - supports-color - utf-8-validate