diff --git a/__tests__/respect/local-json-server/.gitignore b/__tests__/respect/local-json-server/.gitignore new file mode 100644 index 0000000000..e55a99aef3 --- /dev/null +++ b/__tests__/respect/local-json-server/.gitignore @@ -0,0 +1 @@ +fake-db.json \ No newline at end of file diff --git a/__tests__/respect/local-json-server/fake-db.json b/__tests__/respect/local-json-server/fake-db.json index 6a7ce6ab71..2feb21015d 100644 --- a/__tests__/respect/local-json-server/fake-db.json +++ b/__tests__/respect/local-json-server/fake-db.json @@ -1,20 +1,3 @@ { - "items": [ - { - "id": "a959", - "value": 9 - }, - { - "id": "1768", - "value": 6 - }, - { - "id": "6e43", - "value": 10 - }, - { - "id": "8cf3", - "value": 7 - } - ] + "items": [] } diff --git a/__tests__/respect/max-steps/__snapshots__/max-steps.test.ts.snap b/__tests__/respect/max-steps/__snapshots__/max-steps.test.ts.snap new file mode 100644 index 0000000000..697ce63ae2 --- /dev/null +++ b/__tests__/respect/max-steps/__snapshots__/max-steps.test.ts.snap @@ -0,0 +1,165 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should quit an infinite loop on RESPECT_MAX_STEPS 1`] = ` +"──────────────────────────────────────────────────────────────────────────────── + + Running workflow arazzo.yaml / infinite + + ✗ GET /ping - step pre-step + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action continue for the step pre-step + + Running child workflow for the step with-nested + Running workflow arazzo.yaml / nested-workflow + + ✗ GET /ping - step step-1 + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action next-step for the step step-1 + + ✗ GET /ping - step step-2 + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + ✗ GET /ping - step ping + +    Request URL: https://bad-api-url.com/api/ping + +    ✗ failed network request + + Running failure action infinite-loop for the step ping + + + + + + + + + + +  Failed tests info: + +  Workflow name: infinite + +    stepId - pre-step +    ✗ failed network request +      fetch failed +       +    stepId - step-1 +    ✗ failed network request +      fetch failed +       +    stepId - step-2 +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ failed network request +      fetch failed +       +    stepId - ping +    ✗ unexpected error +    Reason: Max steps (10) reached +  Summary for arazzo.yaml +   +  Workflows: 1 failed, 1 total +  Steps: 11 failed, 11 total +  Checks: 11 failed, 11 total +  Time: ms + + +┌─────────────────────────────────────────────────────┬────────────┬─────────┬─────────┬──────────┬─────────┐ +│ Filename │ Workflows │ Passed │ Failed │ Warnings │ Skipped │ +├─────────────────────────────────────────────────────┼────────────┼─────────┼─────────┼──────────┼─────────┤ +│ x arazzo.yaml │ 1 │ 0 │ 1 │ - │ - │ +└─────────────────────────────────────────────────────┴────────────┴─────────┴─────────┴──────────┴─────────┘ + + + + Tests exited with error +" +`; diff --git a/__tests__/respect/max-steps/arazzo.yaml b/__tests__/respect/max-steps/arazzo.yaml new file mode 100644 index 0000000000..55ab2214e8 --- /dev/null +++ b/__tests__/respect/max-steps/arazzo.yaml @@ -0,0 +1,47 @@ +arazzo: 1.0.1 +info: + title: Arazzo + version: 1.0.0 + description: Test infinite loop +sourceDescriptions: + - name: oas + type: openapi + url: ./oas.yaml +workflows: + - workflowId: infinite + steps: + - stepId: pre-step + description: Pre-step + operationId: ping + onFailure: + - name: continue + type: goto + stepId: with-nested + + - stepId: with-nested + description: With nested workflow + workflowId: nested-workflow + + - stepId: ping + description: Ping the API + operationId: ping + onFailure: + - name: infinite-loop + type: goto + stepId: ping + + - workflowId: nested-workflow + steps: + - stepId: step-1 + description: Step 1 + operationId: ping + onFailure: + - name: next-step + type: goto + stepId: step-2 + - stepId: step-2 + description: Step 2 + operationId: ping + onFailure: + - name: end + type: end diff --git a/__tests__/respect/max-steps/max-steps.test.ts b/__tests__/respect/max-steps/max-steps.test.ts new file mode 100644 index 0000000000..df7e892468 --- /dev/null +++ b/__tests__/respect/max-steps/max-steps.test.ts @@ -0,0 +1,20 @@ +import { getParams, getCommandOutput } from '../utils'; +import { join } from 'path'; + +test('should quit an infinite loop on RESPECT_MAX_STEPS', () => { + const indexEntryPoint = join(process.cwd(), 'packages/cli/lib/index.js'); + const fixturesPath = join(__dirname, 'arazzo.yaml'); + const args = getParams(indexEntryPoint, [ + 'respect', + fixturesPath, + '--verbose', + '--workflow', + 'infinite', + ]); + + const result = getCommandOutput(args, { + RESPECT_MAX_STEPS: '10', + }); + + expect(result).toMatchSnapshot(); +}); diff --git a/__tests__/respect/max-steps/oas.yaml b/__tests__/respect/max-steps/oas.yaml new file mode 100644 index 0000000000..8cbf7446ed --- /dev/null +++ b/__tests__/respect/max-steps/oas.yaml @@ -0,0 +1,18 @@ +openapi: 3.1.0 +info: + title: Arazzo + version: 1.0.0 + description: Arazzo API + license: + name: MIT +servers: + - url: https://bad-api-url.com/api +paths: + /ping: + get: + security: [] + summary: Ping + operationId: ping + responses: + '200': + description: A list of users diff --git a/__tests__/respect/severity-off-level/__snapshots__/severity-off-level.test.ts.snap b/__tests__/respect/severity-off-level/__snapshots__/severity-off-level.test.ts.snap index ed3809abb2..73ed738c26 100644 --- a/__tests__/respect/severity-off-level/__snapshots__/severity-off-level.test.ts.snap +++ b/__tests__/respect/severity-off-level/__snapshots__/severity-off-level.test.ts.snap @@ -346,7 +346,7 @@ exports[`should use off severity level 1`] = ` ┌─────────────────────────────────────────────────────────────┬────────────┬─────────┬─────────┬──────────┬─────────┐ │ Filename │ Workflows │ Passed │ Failed │ Warnings │ Skipped │ ├─────────────────────────────────────────────────────────────┼────────────┼─────────┼─────────┼──────────┼─────────┤ -│ ✓ severity-level.yaml │ 2 │ 2 │ - │ - │ 2 │ +│ ✓ severity-level.yaml │ 2 │ 2 │ - │ - │ - │ └─────────────────────────────────────────────────────────────┴────────────┴─────────┴─────────┴──────────┴─────────┘ diff --git a/__tests__/respect/utils.ts b/__tests__/respect/utils.ts index bf3fd788ed..fa4f18433c 100644 --- a/__tests__/respect/utils.ts +++ b/__tests__/respect/utils.ts @@ -7,7 +7,7 @@ export function getParams(indexEntryPoint: string, args: string[] = []): string[ return [indexEntryPoint, ...args]; } -export function getCommandOutput(args: string[]) { +export function getCommandOutput(args: string[], env?: Record) { const result = spawnSync('node', args, { encoding: 'utf-8', stdio: 'pipe', @@ -16,6 +16,7 @@ export function getCommandOutput(args: string[]) { NODE_ENV: 'test', NO_COLOR: 'TRUE', FORCE_COLOR: '0', + ...env, }, }); diff --git a/packages/respect-core/src/handlers/run.ts b/packages/respect-core/src/handlers/run.ts index 7e1756269d..1ecc4ec9ad 100644 --- a/packages/respect-core/src/handlers/run.ts +++ b/packages/respect-core/src/handlers/run.ts @@ -87,11 +87,15 @@ export async function handleRun({ argv, collectSpecData }: CommandArgs { }); expect(displayChecks).toHaveBeenCalled(); - expect(checkCriteria).toHaveBeenCalledTimes(5); + expect(checkCriteria).toHaveBeenCalledTimes(3); }); it('should result with an error when onFailure step criteria with retry StepId and WorkflowId provided', async () => { @@ -1790,7 +1790,7 @@ describe('runStep', () => { }); expect(displayChecks).toHaveBeenCalled(); - expect(checkCriteria).toHaveBeenCalledTimes(5); + expect(checkCriteria).toHaveBeenCalledTimes(3); }); it('should execute onFailure step criteria with successful retry', async () => { diff --git a/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts b/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts index fb8ffbce22..6443e9c1cd 100644 --- a/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts +++ b/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts @@ -9,26 +9,23 @@ export function calculateTotals(workflows: WorkflowExecutionResult[]): Calculate let totalRequests = 0; let totalWarnings = 0; let totalSkipped = 0; - const failedWorkflows = new Set(); - const workflowsWithSkippedStepsChecks = new Set(); + let failedWorkflows = 0; const workflowsWithWarningsStepsChecks = new Set(); - const failedSteps = new Set(); + let failedSteps = 0; const skippedSteps = new Set(); const warningsSteps = new Set(); for (const workflow of workflows) { const steps = flattenNestedSteps(workflow.executedSteps); + let hasFailedSteps = false; for (const step of steps) { - if (step.response) { - // we count request only if we received a response to prevent counting network errors or step with broken inputs - totalRequests++; - } + totalRequests++; if (step.retriesLeft && step.retriesLeft !== 0) { continue; // do not count retried steps as a step } totalSteps++; - + let hasFailedChecks = false; for (const check of step.checks) { totalChecks++; if (!check.passed) { @@ -40,30 +37,35 @@ export function calculateTotals(workflows: WorkflowExecutionResult[]): Calculate break; case 'off': totalSkipped++; - workflowsWithSkippedStepsChecks.add(workflow.workflowId); skippedSteps.add(workflow.workflowId + ':' + step.stepId); break; default: failedChecks++; - failedWorkflows.add(workflow.workflowId); - failedSteps.add(workflow.workflowId + ':' + step.stepId); + hasFailedChecks = true; } } } + if (hasFailedChecks) { + failedSteps++; + hasFailedSteps = true; + } + } + if (hasFailedSteps) { + failedWorkflows++; } } return { workflows: { - passed: totalWorkflows - failedWorkflows.size, - failed: failedWorkflows.size, + passed: totalWorkflows - failedWorkflows, + failed: failedWorkflows, warnings: workflowsWithWarningsStepsChecks.size, - skipped: workflowsWithSkippedStepsChecks.size, + skipped: 0, total: totalWorkflows, }, steps: { - passed: totalSteps - failedSteps.size, - failed: failedSteps.size, + passed: totalSteps - failedSteps, + failed: failedSteps, warnings: warningsSteps.size, skipped: skippedSteps.size, total: totalSteps, diff --git a/packages/respect-core/src/modules/cli-output/json-logs.ts b/packages/respect-core/src/modules/cli-output/json-logs.ts index 73ed9dba95..4453924e9d 100644 --- a/packages/respect-core/src/modules/cli-output/json-logs.ts +++ b/packages/respect-core/src/modules/cli-output/json-logs.ts @@ -4,6 +4,7 @@ import type { TestContext, JsonLogs, WorkflowExecutionResult, + WorkflowExecutionResultJson, Step, StepExecutionResult, Check, @@ -28,21 +29,23 @@ export function composeJsonLogsFiles( const { secretFields } = fileResult.ctx; files[fileResult.file] = maskSecrets( - executedWorkflows.map((workflow) => { - const steps = workflow.executedSteps.map((step) => - composeJsonSteps(step, workflow.workflowId, fileResult.ctx) - ); + { + totalRequests: fileResult.totalRequests, + executedWorkflows: executedWorkflows.map((workflow) => { + const steps = workflow.executedSteps.map((step) => + composeJsonSteps(step, workflow.workflowId, fileResult.ctx) + ); - const result = { - ...workflow, - executedSteps: steps, - status: fileResult.hasProblems ? 'error' : fileResult.hasWarnings ? 'warn' : 'success', - totalTimeMs: fileResult.totalTimeMs, - totalRequests: fileResult.totalRequests, - }; + const result: WorkflowExecutionResultJson = { + ...workflow, + executedSteps: steps, + status: fileResult.hasProblems ? 'error' : fileResult.hasWarnings ? 'warn' : 'success', + totalTimeMs: fileResult.totalTimeMs, + }; - return result; - }), + return result; + }), + }, secretFields || new Set() ); } diff --git a/packages/respect-core/src/modules/cli-output/mask-secrets.ts b/packages/respect-core/src/modules/cli-output/mask-secrets.ts index c0cdb4e988..508838fa09 100644 --- a/packages/respect-core/src/modules/cli-output/mask-secrets.ts +++ b/packages/respect-core/src/modules/cli-output/mask-secrets.ts @@ -1,12 +1,15 @@ -export function maskSecrets(target: { [x: string]: any } | string, secretValues: Set) { +export function maskSecrets( + target: T, + secretValues: Set +): T { const maskValue = (): string => '*'.repeat(8); if (typeof target === 'string') { - let maskedString = target; + let maskedString = target as string; secretValues.forEach((secret) => { maskedString = maskedString.split(secret).join('*'.repeat(secret.length)); }); - return maskedString; + return maskedString as T; } const masked = JSON.parse(JSON.stringify(target)); diff --git a/packages/respect-core/src/modules/flow-runner/run-step.ts b/packages/respect-core/src/modules/flow-runner/run-step.ts index a37eb3493b..3639103dc8 100644 --- a/packages/respect-core/src/modules/flow-runner/run-step.ts +++ b/packages/respect-core/src/modules/flow-runner/run-step.ts @@ -32,6 +32,9 @@ import type { ParameterWithoutIn } from '../config-parser'; const logger = DefaultLogger.getInstance(); +const MAX_STEPS = parseInt(process.env.RESPECT_MAX_STEPS || '500', 10); +let stepsRun = 0; + export async function runStep({ step, ctx, @@ -142,6 +145,17 @@ export async function runStep({ } ctx.executedSteps.push(step); + stepsRun++; + if (stepsRun > MAX_STEPS) { + step.checks.push({ + name: CHECKS.UNEXPECTED_ERROR, + message: `Max steps (${MAX_STEPS}) reached`, + passed: false, + severity: ctx.severity['UNEXPECTED_ERROR'], + }); + return { shouldEnd: true }; + } + if (resolvedParameters && resolvedParameters.length) { // When the step in context does not specify a workflowId the `in` field MUST be specified. const parameterWithoutIn = resolvedParameters.find((parameter: Parameter) => { diff --git a/packages/respect-core/src/types.ts b/packages/respect-core/src/types.ts index a0dc301933..e22bfc4245 100644 --- a/packages/respect-core/src/types.ts +++ b/packages/respect-core/src/types.ts @@ -238,6 +238,11 @@ export interface WorkflowExecutionResult { invocationContext?: string; } +export type WorkflowExecutionResultJson = Omit & { + executedSteps: (StepExecutionResult | WorkflowExecutionResult)[]; + status: ExecutionStatus; +}; + export type TestContext = RuntimeExpressionContext & { executedSteps: (Step | WorkflowExecutionResult)[]; arazzo: string; @@ -294,7 +299,13 @@ export type StepCallContext = { }; export type JsonLogs = { - files: Record; + files: Record< + string, + { + totalRequests: number; + executedWorkflows: WorkflowExecutionResultJson[]; + } + >; status: string; totalTime: number; };