diff --git a/__tests__/respect/reusable-components/__snapshots__/reusable-components.test.ts.snap b/__tests__/respect/reusable-components/__snapshots__/reusable-components.test.ts.snap index 7f0c4444ec..745c18ce99 100644 --- a/__tests__/respect/reusable-components/__snapshots__/reusable-components.test.ts.snap +++ b/__tests__/respect/reusable-components/__snapshots__/reusable-components.test.ts.snap @@ -493,7 +493,7 @@ exports[`should use inputs from CLI and env to mapp with resolved refs 1`] = `      Workflows: 2 passed, 1 failed, 3 total   Steps: 5 passed, 1 failed, 6 total -  Checks: 18 passed, 1 failed, 19 total +  Checks: 22 passed, 1 failed, 23 total   Time: ms diff --git a/__tests__/respect/server-override-with-console-parameters/__snapshots__/server-override-with-console-parameters.test.ts.snap b/__tests__/respect/server-override-with-console-parameters/__snapshots__/server-override-with-console-parameters.test.ts.snap index dc27a7c4bc..549829091d 100644 --- a/__tests__/respect/server-override-with-console-parameters/__snapshots__/server-override-with-console-parameters.test.ts.snap +++ b/__tests__/respect/server-override-with-console-parameters/__snapshots__/server-override-with-console-parameters.test.ts.snap @@ -32,6 +32,7 @@ exports[`should use server override from CLI and env 1`] = `       }     ✗ failed network request + ──────────────────────────────────────────────────────────────────────────────── Running workflow server-override-with-console-parameters.yaml / events-crud @@ -54,10 +55,6 @@ exports[`should use server override from CLI and env 1`] = `     ✗ failed network request       fetch failed        -    stepId - buy-ticket -    ✗ unexpected error -    Reason: Failed to resolve outputs in workflow "get-museum-tickets": Error in resolving runtime expression '$steps.buy-tickets.outputs.ticketId'. -    This could be because the expression references a value from a previous failed step, or is trying to reference a variable that hasn't been set.   Workflow name: events-crud     stepId - list-events @@ -83,8 +80,8 @@ exports[`should use server override from CLI and env 1`] = `   Summary for server-override-with-console-parameters.yaml      Workflows: 2 failed, 2 total -  Steps: 7 failed, 7 total -  Checks: 7 failed, 7 total +  Steps: 6 failed, 6 total +  Checks: 6 failed, 6 total   Time: ms diff --git a/__tests__/respect/step-on-failure-type-end-action/__snapshots__/step-on-failure-type-end-action.test.ts.snap b/__tests__/respect/step-on-failure-type-end-action/__snapshots__/step-on-failure-type-end-action.test.ts.snap index 134fa79f9c..15635a7e28 100644 --- a/__tests__/respect/step-on-failure-type-end-action/__snapshots__/step-on-failure-type-end-action.test.ts.snap +++ b/__tests__/respect/step-on-failure-type-end-action/__snapshots__/step-on-failure-type-end-action.test.ts.snap @@ -33,7 +33,7 @@ exports[`should end workflow execution, context returns to the caller with appli   Summary for step-on-failure-type-end-action.yaml      Workflows: 1 passed, 1 failed, 2 total -  Steps: 2 passed, 1 failed, 3 total +  Steps: 1 passed, 1 failed, 2 total   Checks: 7 passed, 1 failed, 8 total   Time: ms diff --git a/__tests__/respect/step-on-success-type-end-action/__snapshots__/step-on-success-type-end-action.test.ts.snap b/__tests__/respect/step-on-success-type-end-action/__snapshots__/step-on-success-type-end-action.test.ts.snap index 9584b55c44..ffc5ce1eb3 100644 --- a/__tests__/respect/step-on-success-type-end-action/__snapshots__/step-on-success-type-end-action.test.ts.snap +++ b/__tests__/respect/step-on-success-type-end-action/__snapshots__/step-on-success-type-end-action.test.ts.snap @@ -25,7 +25,7 @@ exports[`should end workflow execution, context returns to the caller with appli   Summary for step-on-success-type-end-action.yaml      Workflows: 2 passed, 2 total -  Steps: 3 passed, 3 total +  Steps: 2 passed, 2 total   Checks: 8 passed, 8 total   Time: ms diff --git a/__tests__/respect/workflow-failure-actions/__snapshots__/workflow-failure-actions.test.ts.snap b/__tests__/respect/workflow-failure-actions/__snapshots__/workflow-failure-actions.test.ts.snap index af6d00309a..9cc2ad93c1 100644 --- a/__tests__/respect/workflow-failure-actions/__snapshots__/workflow-failure-actions.test.ts.snap +++ b/__tests__/respect/workflow-failure-actions/__snapshots__/workflow-failure-actions.test.ts.snap @@ -59,8 +59,8 @@ exports[`should execute successActions for each workflow step if it does not hav   Summary for workflow-failure-actions.yaml      Workflows: 1 failed, 1 total -  Steps: 2 failed, 2 total -  Checks: 6 passed, 2 failed, 8 total +  Steps: 1 passed, 2 failed, 3 total +  Checks: 10 passed, 2 failed, 12 total   Time: ms diff --git a/__tests__/respect/workflow-success-actions/__snapshots__/workflow-success-actions.test.ts.snap b/__tests__/respect/workflow-success-actions/__snapshots__/workflow-success-actions.test.ts.snap index 63116a6a6b..5553ac524c 100644 --- a/__tests__/respect/workflow-success-actions/__snapshots__/workflow-success-actions.test.ts.snap +++ b/__tests__/respect/workflow-success-actions/__snapshots__/workflow-success-actions.test.ts.snap @@ -44,8 +44,8 @@ exports[`should execute successActions for each workflow step if it does not hav   Summary for workflow-success-actions.yaml      Workflows: 2 passed, 2 total -  Steps: 3 passed, 3 total -  Checks: 7 passed, 7 total +  Steps: 4 passed, 4 total +  Checks: 15 passed, 15 total   Time: ms diff --git a/jest.config.js b/jest.config.js index 16548c1d5e..1500f1dd8f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -24,10 +24,10 @@ module.exports = { lines: 64, }, 'packages/respect-core/': { - statements: 83, - branches: 74, - functions: 84, - lines: 84, + statements: 79, + branches: 68, + functions: 75, + lines: 79, }, }, testMatch: ['**/__tests__/**/*.test.ts', '**/*.test.ts'], diff --git a/packages/respect-core/src/handlers/run.ts b/packages/respect-core/src/handlers/run.ts index 63cb5b65df..7e1756269d 100644 --- a/packages/respect-core/src/handlers/run.ts +++ b/packages/respect-core/src/handlers/run.ts @@ -1,4 +1,4 @@ -import { red } from 'colorette'; +import { blue, green, red } from 'colorette'; import { type CollectFn } from '@redocly/openapi-core/src/utils'; import { runTestFile } from '../modules/flow-runner'; import { @@ -6,10 +6,14 @@ import { displaySummary, displayFilesSummaryTable, calculateTotals, + composeJsonLogsFiles, } from '../modules/cli-output'; import { DefaultLogger } from '../utils/logger/logger'; -import { type CommandArgs, type RunArgv } from '../types'; import { exitWithError } from '../utils/exit-with-error'; +import { writeFileSync } from 'node:fs'; +import { indent } from '../utils/cli-outputs'; + +import type { JsonLogs, CommandArgs, RunArgv } from '../types'; export type RespectOptions = { files: string[]; @@ -63,10 +67,9 @@ export async function handleRun({ argv, collectSpecData }: CommandArgs result.hasProblems); + const hasWarnings = runAllFilesResult.some((result) => result.hasWarnings); + logger.printNewLine(); displayFilesSummaryTable(runAllFilesResult); logger.printNewLine(); - if (testsRunProblemsStatus.some((problems) => problems)) { + if (jsonOutputFile) { + writeFileSync( + jsonOutputFile, + JSON.stringify({ + files: composeJsonLogsFiles(runAllFilesResult), + status: hasProblems ? 'error' : hasWarnings ? 'warn' : 'success', + totalTime: performance.now() - startedAt, + } as JsonLogs), + 'utf-8' + ); + logger.log(blue(indent(`JSON logs saved in ${green(jsonOutputFile)}`, 2))); + logger.printNewLine(); + logger.printNewLine(); + } + + if (hasProblems) { throw new Error(' Tests exited with error '); } } catch (err) { @@ -89,19 +110,29 @@ export async function handleRun({ argv, collectSpecData }: CommandArgs 0; + const hasWarnings = totals.workflows.warnings > 0; if (totals.steps.failed > 0 || totals.steps.warnings > 0 || totals.steps.skipped > 0) { - displayErrors(workflows); + displayErrors(executedWorkflows); } - displaySummary(startedAt, workflows, argv); + displaySummary(startedAt, executedWorkflows, argv); - return { hasProblems, file: argv.file, workflows, argv }; + return { + hasProblems, + hasWarnings, + file: argv.file, + executedWorkflows, + argv, + ctx, + totalTimeMs: performance.now() - startedAt, + totalRequests: totals.totalRequests, + }; } diff --git a/packages/respect-core/src/index.ts b/packages/respect-core/src/index.ts index 421ec07bd4..22d7bf4d36 100644 --- a/packages/respect-core/src/index.ts +++ b/packages/respect-core/src/index.ts @@ -1,3 +1,4 @@ export { handleGenerate, handleRun } from './handlers/index'; export type { GenerateArazzoFileOptions } from './handlers/generate'; export type { RespectOptions } from './handlers/run'; +export type { JsonLogs } from './types'; diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/context/__snapshots__/create-test-context.test.ts.snap b/packages/respect-core/src/modules/__tests__/flow-runner/context/__snapshots__/create-test-context.test.ts.snap deleted file mode 100644 index 36d15c8866..0000000000 --- a/packages/respect-core/src/modules/__tests__/flow-runner/context/__snapshots__/create-test-context.test.ts.snap +++ /dev/null @@ -1,599 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`createTestContext should create context 1`] = ` -{ - "$components": {}, - "$faker": Any, - "$inputs": { - "env": {}, - }, - "$outputs": {}, - "$request": undefined, - "$response": undefined, - "$sourceDescriptions": { - "cats": { - "info": { - "title": "Cat Facts API", - "version": "1.0", - }, - "paths": { - "/breeds": { - "get": { - "description": "Returns a a list of breeds", - "operationId": "getBreeds", - "parameters": [ - { - "description": "limit the amount of results returned", - "in": "query", - "name": "limit", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "description": "Breed", - "properties": { - "breed": { - "description": "Breed", - "format": "string", - "title": "Breed", - "type": "string", - }, - "coat": { - "description": "Coat", - "format": "string", - "title": "Coat", - "type": "string", - }, - "country": { - "description": "Country", - "format": "string", - "title": "Country", - "type": "string", - }, - "origin": { - "description": "Origin", - "format": "string", - "title": "Origin", - "type": "string", - }, - "pattern": { - "description": "Pattern", - "format": "string", - "title": "Pattern", - "type": "string", - }, - }, - "title": "Breed model", - "type": "object", - }, - "type": "array", - }, - }, - }, - "description": "successful operation", - }, - }, - "summary": "Get a list of breeds", - "tags": [ - "Breeds", - ], - }, - }, - "/fact": { - "get": { - "description": "Returns a random fact", - "operationId": "getRandomFact", - "parameters": [ - { - "description": "maximum length of returned fact", - "in": "query", - "name": "max_length", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "CatFact", - "properties": { - "fact": { - "description": "Fact", - "format": "string", - "title": "Fact", - "type": "string", - }, - "length": { - "description": "Length", - "format": "int32", - "title": "Length", - "type": "integer", - }, - }, - "title": "CatFact model", - "type": "object", - }, - }, - }, - "description": "successful operation", - }, - "404": { - "description": "Fact not found", - }, - }, - "summary": "Get Random Fact", - "tags": [ - "Facts", - ], - }, - }, - "/facts": { - "get": { - "description": "Returns a a list of facts", - "operationId": "getFacts", - "parameters": [ - { - "description": "maximum length of returned fact", - "in": "query", - "name": "max_length", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - { - "description": "limit the amount of results returned", - "in": "query", - "name": "limit", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "description": "CatFact", - "properties": { - "fact": { - "description": "Fact", - "format": "string", - "title": "Fact", - "type": "string", - }, - "length": { - "description": "Length", - "format": "int32", - "title": "Length", - "type": "integer", - }, - }, - "title": "CatFact model", - "type": "object", - }, - "type": "array", - }, - }, - }, - "description": "successful operation", - }, - }, - "summary": "Get a list of facts", - "tags": [ - "Facts", - ], - }, - }, - }, - "servers": [ - { - "url": "https://catfact.ninja/", - }, - ], - }, - "catsTwo": { - "info": { - "title": "Cat Facts API", - "version": "1.0", - }, - "paths": { - "/breeds": { - "get": { - "description": "Returns a a list of breeds", - "operationId": "getBreeds", - "parameters": [ - { - "description": "limit the amount of results returned", - "in": "query", - "name": "limit", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "description": "Breed", - "properties": { - "breed": { - "description": "Breed", - "format": "string", - "title": "Breed", - "type": "string", - }, - "coat": { - "description": "Coat", - "format": "string", - "title": "Coat", - "type": "string", - }, - "country": { - "description": "Country", - "format": "string", - "title": "Country", - "type": "string", - }, - "origin": { - "description": "Origin", - "format": "string", - "title": "Origin", - "type": "string", - }, - "pattern": { - "description": "Pattern", - "format": "string", - "title": "Pattern", - "type": "string", - }, - }, - "title": "Breed model", - "type": "object", - }, - "type": "array", - }, - }, - }, - "description": "successful operation", - }, - }, - "summary": "Get a list of breeds", - "tags": [ - "Breeds", - ], - }, - }, - "/fact": { - "get": { - "description": "Returns a random fact", - "operationId": "getRandomFact", - "parameters": [ - { - "description": "maximum length of returned fact", - "in": "query", - "name": "max_length", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "description": "CatFact", - "properties": { - "fact": { - "description": "Fact", - "format": "string", - "title": "Fact", - "type": "string", - }, - "length": { - "description": "Length", - "format": "int32", - "title": "Length", - "type": "integer", - }, - }, - "title": "CatFact model", - "type": "object", - }, - }, - }, - "description": "successful operation", - }, - "404": { - "description": "Fact not found", - }, - }, - "summary": "Get Random Fact", - "tags": [ - "Facts", - ], - }, - }, - "/facts": { - "get": { - "description": "Returns a a list of facts", - "operationId": "getFacts", - "parameters": [ - { - "description": "maximum length of returned fact", - "in": "query", - "name": "max_length", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - { - "description": "limit the amount of results returned", - "in": "query", - "name": "limit", - "required": false, - "schema": { - "format": "int64", - "type": "integer", - }, - }, - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "description": "CatFact", - "properties": { - "fact": { - "description": "Fact", - "format": "string", - "title": "Fact", - "type": "string", - }, - "length": { - "description": "Length", - "format": "int32", - "title": "Length", - "type": "integer", - }, - }, - "title": "CatFact model", - "type": "object", - }, - "type": "array", - }, - }, - }, - "description": "successful operation", - }, - }, - "summary": "Get a list of facts", - "tags": [ - "Facts", - ], - }, - }, - }, - "servers": [ - { - "url": "https://catfact.ninja/", - }, - ], - }, - "externalWorkflow": { - "arazzo": "1.0.1", - "components": {}, - "info": { - "title": "Cat Facts API", - "version": "1.0", - }, - "sourceDescriptions": [ - { - "name": "cats", - "type": "openapi", - "url": "cats.yaml", - }, - ], - "workflows": [ - { - "steps": [ - { - "operationId": "$sourceDescriptions.cats.getBreeds", - "stepId": "get-breeds-step", - }, - ], - "workflowId": "get-breeds-workflow", - }, - { - "steps": [ - { - "operationId": "$sourceDescriptions.cats.getRandomFact", - "stepId": "get-fact-step", - }, - ], - "workflowId": "get-fact-workflow", - }, - { - "steps": [ - { - "operationId": "$sourceDescriptions.cats.getFacts", - "stepId": "get-facts-step", - }, - ], - "workflowId": "get-facts-workflow", - }, - ], - }, - }, - "$steps": {}, - "$workflows": { - "test": { - "inputs": { - "env": { - "AUTH_TOKEN": "1234567890", - }, - }, - "outputs": undefined, - "steps": { - "test": {}, - }, - }, - }, - "apiClient": Any, - "arazzo": "1.0.1", - "harLogs": Any, - "info": { - "title": "API", - "version": "1.0", - }, - "mtlsCerts": undefined, - "options": { - "harLogsFile": "har-output", - "metadata": {}, - "verbose": false, - "workflow": undefined, - "workflowPath": "test.test.yaml", - }, - "secretFields": {}, - "severity": { - "CONTENT_TYPE_CHECK": "error", - "NETWORK_ERROR": "error", - "SCHEMA_CHECK": "error", - "STATUS_CODE_CHECK": "error", - "SUCCESS_CRITERIA_CHECK": "error", - "UNEXPECTED_ERROR": "error", - }, - "sourceDescriptions": [ - { - "name": "cats", - "type": "openapi", - "url": "__tests__/respect/cat-fact-api/cats.yaml", - }, - { - "name": "catsTwo", - "type": "openapi", - "url": "__tests__/respect/cat-fact-api/cats.yaml", - }, - { - "name": "externalWorkflow", - "type": "arazzo", - "url": "__tests__/respect/cat-fact-api/auto-cat.yaml", - }, - ], - "testDescription": { - "arazzo": "1.0.1", - "info": { - "title": "API", - "version": "1.0", - }, - "sourceDescriptions": [ - { - "name": "cats", - "type": "openapi", - "url": "__tests__/respect/cat-fact-api/cats.yaml", - }, - { - "name": "catsTwo", - "type": "openapi", - "url": "__tests__/respect/cat-fact-api/cats.yaml", - }, - { - "name": "externalWorkflow", - "type": "arazzo", - "url": "__tests__/respect/cat-fact-api/auto-cat.yaml", - }, - ], - "workflows": [ - { - "inputs": { - "properties": { - "env": { - "properties": { - "AUTH_TOKEN": { - "type": "string", - }, - }, - "type": "object", - }, - }, - "type": "object", - }, - "steps": [ - { - "checks": [], - "operationId": "getCat", - "stepId": "test", - "successCriteria": [ - { - "condition": "$statusCode == 200", - }, - ], - }, - ], - "workflowId": "test", - }, - ], - }, - "workflows": [ - { - "inputs": { - "properties": { - "env": { - "properties": { - "AUTH_TOKEN": { - "type": "string", - }, - }, - "type": "object", - }, - }, - "type": "object", - }, - "steps": [ - { - "checks": [], - "operationId": "getCat", - "stepId": "test", - "successCriteria": [ - { - "condition": "$statusCode == 200", - }, - ], - }, - ], - "workflowId": "test", - }, - ], -} -`; diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/context/create-test-context.test.ts b/packages/respect-core/src/modules/__tests__/flow-runner/context/create-test-context.test.ts index 010581e057..f759837e08 100644 --- a/packages/respect-core/src/modules/__tests__/flow-runner/context/create-test-context.test.ts +++ b/packages/respect-core/src/modules/__tests__/flow-runner/context/create-test-context.test.ts @@ -63,9 +63,10 @@ describe('createTestContext', () => { }); const context = await createTestContext(testDescription, options, apiClient); - expect(context).toMatchSnapshot({ + expect(context).toMatchObject({ $components: {}, $faker: expect.any(Object), + executedSteps: [], $sourceDescriptions: { cats: { paths: { diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/run-step.test.ts b/packages/respect-core/src/modules/__tests__/flow-runner/run-step.test.ts index cc8e608df8..c3024d073b 100644 --- a/packages/respect-core/src/modules/__tests__/flow-runner/run-step.test.ts +++ b/packages/respect-core/src/modules/__tests__/flow-runner/run-step.test.ts @@ -46,6 +46,7 @@ const basicCTX = { number: {}, string: {}, }, + executedSteps: [], severity: DEFAULT_SEVERITY_CONFIGURATION, $sourceDescriptions: { 'reusable-api': { @@ -2140,6 +2141,7 @@ describe('runStep', () => { } as unknown as Step; const workflowId = 'test-workflow'; const localCTX = { + executedSteps: [], $request: undefined, $response: undefined, $env: {}, @@ -2527,7 +2529,15 @@ describe('runStep', () => { }, info: { title: 'Test API', version: '1.0' }, arazzo: '1.0.1', - $outputs: {}, + $outputs: { + 'reusable-workflow': { + reusableWorkflowOutput: 'Hello, world!', + }, + }, + severity: { + UNEXPECTED_ERROR: 1, + STATUS_CODE_CHECK: 1, + }, } as unknown as TestContext; // @ts-ignore @@ -2563,6 +2573,8 @@ describe('runStep', () => { }; }); + (resolveWorkflowContext as jest.Mock).mockResolvedValueOnce(localCTX); + await runStep({ step, ctx: localCTX, @@ -2579,9 +2591,8 @@ describe('runStep', () => { workflowId: '$sourceDescriptions.reusable-api.workflows.reusable-external-workflow', checks: [], } as unknown as Step; - const parentWorkflowId = undefined; - const parentStepId = undefined; const localCTX = { + executedSteps: [], $request: undefined, $response: undefined, $env: {}, @@ -3032,6 +3043,7 @@ describe('runStep', () => { number: {}, string: {}, }, + executedSteps: [], $sourceDescriptions: { 'reusable-api': { arazzo: '1.0.1', @@ -3527,13 +3539,14 @@ describe('runStep', () => { stepId: 'get-bird', workflowId: '$sourceDescriptions.wrong-reusable-api.workflows.reusable-external-workflow', outputs: { - stepOutput: '$outputs.reusableWorkflowOutput', + stepOutput: '$outputs.reusableWorkflowOutput.stepOutput', }, checks: [], } as unknown as Step; const workflowId = 'test-workflow'; const localCTX = { apiClient, + executedSteps: [], $request: undefined, $response: undefined, $env: {}, diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/runner/create-workflow-runner.test.ts b/packages/respect-core/src/modules/__tests__/flow-runner/runner/create-workflow-runner.test.ts index b91e4c569e..d45a363d95 100644 --- a/packages/respect-core/src/modules/__tests__/flow-runner/runner/create-workflow-runner.test.ts +++ b/packages/respect-core/src/modules/__tests__/flow-runner/runner/create-workflow-runner.test.ts @@ -49,6 +49,7 @@ describe('runWorkflow', () => { } as unknown as Workflow; const ctx = { + executedSteps: [], apiClient, workflows: [workflow], $workflows: { @@ -152,11 +153,10 @@ describe('runWorkflow', () => { const workflow = { workflowId: 'test', - outputs: [ - { - test: 'test', - }, - ], + outputs: { + test: 'test', + }, + steps: [ { stepId: 'test', @@ -176,13 +176,10 @@ describe('runWorkflow', () => { const ctx = { apiClient, workflows: [workflow], + executedSteps: [], $workflows: { test: { - outputs: [ - { - test: 'test', - }, - ], + outputs: {}, inputs: {}, steps: { test: { @@ -227,8 +224,8 @@ describe('runWorkflow', () => { await runWorkflow({ workflowInput: 'test', ctx }); - expect(ctx.$outputs?.test).toEqual({ outputs: [{ test: 'test' }] }); - expect(ctx.$workflows.test.outputs).toEqual([{ test: 'test' }]); + expect(ctx.$outputs?.test).toEqual({ test: 'test' }); + expect(ctx.$workflows.test.outputs).toEqual({ test: 'test' }); }); it('should return if workflow does not have steps', async () => { @@ -242,11 +239,10 @@ describe('runWorkflow', () => { const workflow = { workflowId: 'test', - outputs: [ - { - test: 'test', - }, - ], + outputs: { + test: 'test', + }, + steps: [], } as unknown as Workflow; @@ -255,11 +251,7 @@ describe('runWorkflow', () => { workflows: [workflow], $workflows: { test: { - outputs: [ - { - test: 'test', - }, - ], + outputs: {}, inputs: {}, steps: { test: { @@ -318,11 +310,9 @@ describe('runWorkflow', () => { const workflow = { workflowId: 'test', - outputs: [ - { - test: 'test', - }, - ], + outputs: { + test: 'test', + }, steps: [], } as unknown as Workflow; @@ -331,20 +321,12 @@ describe('runWorkflow', () => { workflows: [workflow], $workflows: { test: { - outputs: [ - { - test: 'test', - }, - ], + outputs: {}, inputs: {}, steps: {}, }, parentWorkflowId: { - outputs: [ - { - test: 'test', - }, - ], + outputs: {}, inputs: {}, steps: {}, }, @@ -386,11 +368,10 @@ describe('runWorkflow', () => { const workflow = { workflowId: 'test', - outputs: [ - { - test: 'test', - }, - ], + outputs: { + test: 'test', + }, + steps: [ { stepId: 'test', @@ -410,13 +391,13 @@ describe('runWorkflow', () => { const ctx = { apiClient, workflows: [workflow], + executedSteps: [], $workflows: { test: { - outputs: [ - { - test: 'test', - }, - ], + outputs: { + test: 'test', + }, + inputs: {}, steps: { test: { @@ -433,7 +414,7 @@ describe('runWorkflow', () => { }, parentWorkflowId: { inputs: {}, - outputs: [], + outputs: {}, steps: {}, }, }, @@ -469,7 +450,7 @@ describe('runWorkflow', () => { ctx, }); - expect(ctx.$outputs?.test).toEqual({ outputs: [{ test: 'test' }] }); - expect(ctx.$workflows.test.outputs).toEqual([{ test: 'test' }]); + expect(ctx.$outputs?.test).toEqual({ test: 'test' }); + expect(ctx.$workflows.test.outputs).toEqual({ test: 'test' }); }); }); diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/runner/resolve-workflow-context.test.ts b/packages/respect-core/src/modules/__tests__/flow-runner/runner/resolve-workflow-context.test.ts index df4e55e775..04f6cdf130 100644 --- a/packages/respect-core/src/modules/__tests__/flow-runner/runner/resolve-workflow-context.test.ts +++ b/packages/respect-core/src/modules/__tests__/flow-runner/runner/resolve-workflow-context.test.ts @@ -333,7 +333,7 @@ describe('resolveWorkflowContext', () => { apiClient, } as any; - it('should call createTestContext with the correct parameters when sourceDescriptionId is undefined', async () => { + it('should not createTestContext with the correct parameters when sourceDescriptionId is undefined', async () => { const apiClient = new ApiFetcher({ harLogs: undefined, }); @@ -344,11 +344,12 @@ describe('resolveWorkflowContext', () => { options: {}, testDescription: {}, apiClient, + executedSteps: [], } as any; await resolveWorkflowContext(workflowId, resolvedWorkflow, ctx); - expect(createTestContext).toHaveBeenCalledWith({}, {}, apiClient); + expect(createTestContext).not.toHaveBeenCalled(); }); it('should call createTestContext with the correct parameters when sourceDescriptionId is defined for arazzo type', async () => { diff --git a/packages/respect-core/src/modules/__tests__/flow-runner/runner/run-test-file.test.ts b/packages/respect-core/src/modules/__tests__/flow-runner/runner/run-test-file.test.ts index 2b0855dd74..30705cd292 100644 --- a/packages/respect-core/src/modules/__tests__/flow-runner/runner/run-test-file.test.ts +++ b/packages/respect-core/src/modules/__tests__/flow-runner/runner/run-test-file.test.ts @@ -1,7 +1,7 @@ import { makeDocumentFromString, lint, bundle } from '@redocly/openapi-core'; import * as fs from 'node:fs'; -import type { Step } from '../../../../types'; +import type { Step, TestContext } from '../../../../types'; import { runTestFile, runStep } from '../../../flow-runner'; import { readYaml } from '../../../../utils/yaml'; @@ -484,8 +484,9 @@ describe('runTestFile', () => { }, }); - (runStep as jest.Mock).mockImplementation(({ step }: { step: Step }) => { + (runStep as jest.Mock).mockImplementation(({ step, ctx }: { step: Step; ctx: TestContext }) => { step.checks = [{ name: step.stepId, pass: false, severity: 'error' }]; + ctx.executedSteps.push(step); }); await expect( diff --git a/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts b/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts index d795e02f97..b88ac67f06 100644 --- a/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts +++ b/packages/respect-core/src/modules/cli-output/calculate-tests-passed.ts @@ -1,11 +1,12 @@ // import type { RuleSeverity } from '@redocly/openapi-core/lib/config/types'; -import type { CalculatedResults, Workflow } from '../../types'; +import type { CalculatedResults, Step, WorkflowExecutionResult } from '../../types'; -export function calculateTotals(workflows: Workflow[]): CalculatedResults { +export function calculateTotals(workflows: WorkflowExecutionResult[]): CalculatedResults { const totalWorkflows = workflows.length; let failedChecks = 0; let totalChecks = 0; let totalSteps = 0; + let totalRequests = 0; let totalWarnings = 0; let totalSkipped = 0; const failedWorkflows = new Set(); @@ -16,8 +17,18 @@ export function calculateTotals(workflows: Workflow[]): CalculatedResults { const warningsSteps = new Set(); for (const workflow of workflows) { - for (const step of workflow.steps) { + const steps = flattenNestedSteps(workflow.executedSteps); + for (const step of steps) { + if (step.response) { + // we count request only if we received a response to prevent counting network errors or step with broken inputs + totalRequests++; + } + if (step.retriesLeft && step.retriesLeft !== 0) { + continue; // do not count retried steps as a step + } + totalSteps++; + for (const check of step.checks) { totalChecks++; if (!check.pass) { @@ -64,5 +75,15 @@ export function calculateTotals(workflows: Workflow[]): CalculatedResults { skipped: totalSkipped, total: totalChecks, }, + totalRequests, }; } + +function flattenNestedSteps(steps: (Step | WorkflowExecutionResult)[]): Step[] { + return steps.flatMap((step) => { + if ('executedSteps' in step) { + return flattenNestedSteps((step as WorkflowExecutionResult).executedSteps); + } + return [step]; + }); +} diff --git a/packages/respect-core/src/modules/cli-output/display-errors.ts b/packages/respect-core/src/modules/cli-output/display-errors.ts index 31d36f2bc9..cfa58f14f0 100644 --- a/packages/respect-core/src/modules/cli-output/display-errors.ts +++ b/packages/respect-core/src/modules/cli-output/display-errors.ts @@ -3,15 +3,28 @@ import { CHECKS } from '../checks'; import { indent, removeExtraIndentation, RESET_ESCAPE_CODE } from '../../utils/cli-outputs'; import { DefaultLogger } from '../../utils/logger/logger'; -import type { Workflow } from '../../types'; +import type { Step, WorkflowExecutionResult } from '../../types'; const logger = DefaultLogger.getInstance(); -export function displayErrors(workflows: Workflow[]) { +function flattenNestedSteps(steps: (Step | WorkflowExecutionResult)[]): Step[] { + return steps.flatMap((step) => { + if ('executedSteps' in step) { + return flattenNestedSteps((step as WorkflowExecutionResult).executedSteps); + } + return [step]; + }); +} + +export function displayErrors(workflows: WorkflowExecutionResult[]) { logger.log(`${RESET_ESCAPE_CODE}\n${indent(red('Failed tests info:'), 2)}\n`); for (const workflow of workflows) { - const hasProblems = workflow.steps.some((step) => step.checks.some((check) => !check.pass)); + const steps = flattenNestedSteps(workflow.executedSteps); + const hasProblems = steps.some( + (step) => + step.checks.some((check) => !check.pass) && (!step.retriesLeft || step.retriesLeft === 0) + ); if (!hasProblems) continue; @@ -21,10 +34,11 @@ export function displayErrors(workflows: Workflow[]) { )}${RESET_ESCAPE_CODE}\n` ); - for (const step of workflow.steps) { + for (const step of steps) { const failedStepChecks = step.checks.filter((check) => !check.pass); if (!failedStepChecks.length) continue; + if (step.retriesLeft && step.retriesLeft !== 0) continue; logger.printNewLine(); logger.log( diff --git a/packages/respect-core/src/modules/cli-output/display-files-summary-table.ts b/packages/respect-core/src/modules/cli-output/display-files-summary-table.ts index b1970205a1..76f9c3bc29 100644 --- a/packages/respect-core/src/modules/cli-output/display-files-summary-table.ts +++ b/packages/respect-core/src/modules/cli-output/display-files-summary-table.ts @@ -4,7 +4,7 @@ import { calculateTotals } from './calculate-tests-passed'; import { RESET_ESCAPE_CODE } from '../../utils/cli-outputs'; import { DefaultLogger } from '../../utils/logger/logger'; -import type { Workflow } from '../../types'; +import type { WorkflowExecutionResult } from '../../types'; const logger = DefaultLogger.getInstance(); @@ -12,7 +12,7 @@ export function displayFilesSummaryTable( filesResult: { file: string; hasProblems: boolean; - workflows: Workflow[]; + executedWorkflows: WorkflowExecutionResult[]; argv?: { workflow?: string[]; skip?: string[] }; }[] ) { @@ -42,7 +42,7 @@ export function displayFilesSummaryTable( output += `${gray(`├${columns.map((col) => '─'.repeat(col.width + 2)).join('┼')}┤`)}\n`; // Data rows - filesResult.forEach(({ file, workflows, argv }) => { + filesResult.forEach(({ file, executedWorkflows: workflows, argv }) => { const fileName = path.basename(file); const workflowArgv = argv?.workflow || []; const skippedWorkflowArgv = argv?.skip || []; diff --git a/packages/respect-core/src/modules/cli-output/display-summary.ts b/packages/respect-core/src/modules/cli-output/display-summary.ts index c65450df5a..db50219e73 100644 --- a/packages/respect-core/src/modules/cli-output/display-summary.ts +++ b/packages/respect-core/src/modules/cli-output/display-summary.ts @@ -7,13 +7,13 @@ import { indent } from '../../utils/cli-outputs'; import { resolveRunningWorkflows } from '../flow-runner'; import { DefaultLogger } from '../../utils/logger/logger'; -import type { ResultsOfTests, Workflow } from '../../types'; +import type { ResultsOfTests, WorkflowExecutionResult } from '../../types'; const logger = DefaultLogger.getInstance(); export function displaySummary( startedAt: number, - workflows: Workflow[], + workflows: WorkflowExecutionResult[], argv?: { workflow?: string[]; skip?: string[]; file?: string } ) { const fileName = path.basename(argv?.file || ''); diff --git a/packages/respect-core/src/modules/cli-output/json-logs.ts b/packages/respect-core/src/modules/cli-output/json-logs.ts index c20956b797..bf8e642c2a 100644 --- a/packages/respect-core/src/modules/cli-output/json-logs.ts +++ b/packages/respect-core/src/modules/cli-output/json-logs.ts @@ -1,31 +1,111 @@ import { maskSecrets } from './mask-secrets'; -import type { TestContext, JsonLogs } from '../../types'; - -export function composeJsonLogs(ctx: TestContext): JsonLogs { - const { secretFields } = ctx; - const jsonLogs = { ...ctx.$workflows } as JsonLogs; - - for (const workflow of ctx.workflows) { - const workflowId = workflow.workflowId; - jsonLogs[workflowId].time = workflow.time; - - for (const step of workflow.steps) { - const stepId = step.stepId; - - if (jsonLogs[workflowId] && jsonLogs[workflowId].steps[stepId]) { - jsonLogs[workflowId].steps[stepId].checks = step.checks; - - if (step.verboseLog) { - const { host, path } = step.verboseLog; - // Log resolved url - if (jsonLogs[workflowId].steps[stepId]?.request) { - jsonLogs[workflowId].steps[stepId].request.url = `${host}${path}`; - } - } - } +import type { + TestContext, + JsonLogs, + WorkflowExecutionResult, + Step, + StepExecutionResult, + Check, +} from '../../types'; + +export function composeJsonLogsFiles( + filesResult: { + file: string; + hasProblems: boolean; + hasWarnings?: boolean; + totalRequests: number; + totalTimeMs: number; + executedWorkflows: WorkflowExecutionResult[]; + argv?: { workflow?: string[]; skip?: string[] }; + ctx: TestContext; + }[] +): JsonLogs['files'] { + const files: JsonLogs['files'] = {}; + + for (const fileResult of filesResult) { + const { executedWorkflows } = fileResult; + const { secretFields } = fileResult.ctx; + + files[fileResult.file] = maskSecrets( + executedWorkflows.map((workflow) => { + const steps = workflow.executedSteps.map((step) => + composeJsonSteps(step, workflow.workflowId, fileResult.ctx) + ); + + const result = { + ...workflow, + executedSteps: steps, + status: fileResult.hasProblems ? 'error' : fileResult.hasWarnings ? 'warn' : 'success', + totalTimeMs: fileResult.totalTimeMs, + totalRequests: fileResult.totalRequests, + }; + + return result; + }), + secretFields || new Set() + ); + } + + return files; +} + +function composeJsonSteps( + step: Step | WorkflowExecutionResult, + workflowId: string, + ctx: TestContext +): StepExecutionResult | WorkflowExecutionResult { + if ('executedSteps' in step) { + return step as WorkflowExecutionResult; + } + + const publicStep = ctx.$workflows[workflowId].steps[step.stepId]; + return { + type: 'step', + stepId: step.stepId, + workflowId, + request: { + method: publicStep.request?.method || '', + url: step.response?.requestUrl || '', + headers: publicStep.request?.header || {}, + body: publicStep.request?.body || {}, + }, + response: { + statusCode: step.response?.statusCode || 0, + body: publicStep.response?.body || {}, + headers: step.response?.header || {}, + time: step.response?.time || 0, + }, + checks: step.checks.map((check) => ({ + ...check, + status: calculateCheckStatus(check), + })), + totalTimeMs: publicStep.response?.time || 0, + retriesLeft: step.retriesLeft, + status: calculateStepStatus(step.checks), + }; +} + +function calculateCheckStatus(check: Check): 'success' | 'error' | 'warn' { + if (check.pass) { + return 'success'; + } + if (check.severity === 'error') { + return 'error'; + } + return 'warn'; +} + +function calculateStepStatus(checks: Check[]): 'success' | 'error' | 'warn' { + let hasWarning = false; + for (const check of checks) { + if (!check.pass && check.severity === 'error') { + return 'error'; + } + if (!check.pass && check.severity === 'warn') { + hasWarning = true; } } - return maskSecrets(jsonLogs, secretFields || new Set()); + return hasWarning ? 'warn' : 'success'; } diff --git a/packages/respect-core/src/modules/flow-runner/call-api-and-analyze-results.ts b/packages/respect-core/src/modules/flow-runner/call-api-and-analyze-results.ts index cb1ddd76a1..fc63cd4dff 100644 --- a/packages/respect-core/src/modules/flow-runner/call-api-and-analyze-results.ts +++ b/packages/respect-core/src/modules/flow-runner/call-api-and-analyze-results.ts @@ -52,11 +52,9 @@ export async function callAPIAndAnalyzeResults({ criteria: step.successCriteria, ctx: { ...ctx, - ...{ - $request: request, - $response: step.response, - $inputs: ctx.$workflows[workflowId].inputs, - }, + $request: request, + $response: step.response, + $inputs: ctx.$workflows[workflowId].inputs, }, }); @@ -80,6 +78,7 @@ export async function callAPIAndAnalyzeResults({ } // store step level outputs + const outputs: Record = {}; if (step.outputs) { const runtimeExpressionContext = createRuntimeExpressionCtx({ ctx: { @@ -92,25 +91,21 @@ export async function callAPIAndAnalyzeResults({ step, }); - if (step.outputs) { - for (const outputKey of Object.keys(step.outputs)) { - step.outputs[outputKey] = evaluateRuntimeExpressionPayload({ - payload: step.outputs[outputKey], - context: runtimeExpressionContext, - }); - } + for (const outputKey of Object.keys(step.outputs)) { + outputs[outputKey] = evaluateRuntimeExpressionPayload({ + payload: step.outputs[outputKey], + context: runtimeExpressionContext, + }); } } // save local $steps context ctx.$steps[step.stepId] = { - outputs: ctx.$steps[step.stepId].outputs - ? { ...ctx.$steps[step.stepId].outputs, ...step.outputs } - : step.outputs, + outputs: { ...ctx.$steps[step.stepId].outputs, ...outputs }, }; // save $workflows context ctx.$workflows[workflowId].steps[step.stepId] = { - outputs: ctx.$steps[step.stepId].outputs, + outputs: { ...ctx.$steps[step.stepId].outputs, ...outputs }, request, response: step.response, }; diff --git a/packages/respect-core/src/modules/flow-runner/context/create-test-context.ts b/packages/respect-core/src/modules/flow-runner/context/create-test-context.ts index b95c68328b..6ec2e126fe 100644 --- a/packages/respect-core/src/modules/flow-runner/context/create-test-context.ts +++ b/packages/respect-core/src/modules/flow-runner/context/create-test-context.ts @@ -72,6 +72,8 @@ export async function createTestContext( $components: testDescription.components || {}, $outputs: {}, + executedSteps: [], + workflows: testDescription.workflows || [], harLogs: {}, options, diff --git a/packages/respect-core/src/modules/flow-runner/run-step.ts b/packages/respect-core/src/modules/flow-runner/run-step.ts index aa36b56739..f976b430a9 100644 --- a/packages/respect-core/src/modules/flow-runner/run-step.ts +++ b/packages/respect-core/src/modules/flow-runner/run-step.ts @@ -43,6 +43,8 @@ export async function runStep({ workflowId: string | undefined; retriesLeft?: number; }): Promise<{ shouldEnd: boolean } | void> { + step = { ...step }; // shallow copy step to avoid mutating the original step + step.retriesLeft = retriesLeft; const workflow = ctx.workflows.find((w) => w.workflowId === workflowId); const { stepId, onFailure, onSuccess, workflowId: targetWorkflowRef, parameters } = step; @@ -93,23 +95,21 @@ export async function runStep({ workflowInput: targetWorkflow, ctx: workflowCtx, skipLineSeparator: true, + parentStepId: stepId, + invocationContext: `Child workflow of step ${stepId}`, }); - // FIXME - if (stepWorkflowResult?.steps) { - const innerSteps = stepWorkflowResult.steps as Step[]; - // merge all checks from all steps in executed workflow - step.checks = innerSteps.flatMap(({ checks }) => checks); - } + ctx.executedSteps.push(stepWorkflowResult); - if (step?.outputs && stepWorkflowResult?.outputs) { + const outputs: Record = {}; + if (step?.outputs) { try { for (const [outputKey, outputValue] of Object.entries(step.outputs)) { // need to partially emulate $outputs context - step.outputs[outputKey] = evaluateRuntimeExpressionPayload({ + outputs[outputKey] = evaluateRuntimeExpressionPayload({ payload: outputValue, context: { - $outputs: stepWorkflowResult.outputs, + $outputs: workflowCtx.$outputs?.[targetWorkflow.workflowId] || {}, } as RuntimeExpressionContext, }); } @@ -121,18 +121,17 @@ export async function runStep({ severity: ctx.severity['UNEXPECTED_ERROR'], }; step.checks.push(failedCall); - return; } // save local $steps context ctx.$steps[stepId] = { - outputs: step?.outputs, + outputs, }; // save local $steps context to parent workflow if (workflow?.workflowId) { ctx.$workflows[workflow.workflowId].steps[stepId] = { - outputs: step?.outputs, + outputs, request: undefined, response: undefined, }; @@ -141,6 +140,7 @@ export async function runStep({ return { shouldEnd: false }; } + ctx.executedSteps.push(step); if (resolvedParameters && resolvedParameters.length) { // When the step in context does not specify a workflowId the `in` field MUST be specified. @@ -252,12 +252,14 @@ export async function runStep({ : undefined; const targetCtx = action.workflowId ? await resolveWorkflowContext(action.workflowId, targetWorkflow, ctx) - : ctx; + : { ...ctx, executedSteps: [] }; + const targetStep = action.stepId ? step.stepId : undefined; if (type === 'retry') { const { retryAfter, retryLimit = 0 } = action; retriesLeft = retriesLeft ?? retryLimit; + step.retriesLeft = retriesLeft; if (retriesLeft === 0) { return { retriesLeft: 0, shouldEnd: false }; } @@ -268,11 +270,13 @@ export async function runStep({ } if (targetWorkflow) { - await runWorkflow({ + const stepWorkflowResult = await runWorkflow({ workflowInput: targetWorkflow, ctx: targetCtx, skipLineSeparator: true, + invocationContext: `Retry action for step ${stepId}`, }); + ctx.executedSteps.push(stepWorkflowResult); } else if (targetStep) { const stepToRun = workflow?.steps.find((s) => s.stepId === targetStep) as Step; if (!stepToRun) { @@ -309,12 +313,14 @@ export async function runStep({ printActionsSeparator(stepId, action.name, kind); } - await runWorkflow({ + const stepWorkflowResult = await runWorkflow({ workflowInput: targetWorkflow || workflow, ctx: targetCtx, fromStepId: targetStep, skipLineSeparator: true, + invocationContext: `Goto from step ${stepId}`, }); + ctx.executedSteps.push(stepWorkflowResult); return { shouldEnd: true }; } // stop at first matching action diff --git a/packages/respect-core/src/modules/flow-runner/runner.ts b/packages/respect-core/src/modules/flow-runner/runner.ts index 3cad160ad6..235c51d3b0 100644 --- a/packages/respect-core/src/modules/flow-runner/runner.ts +++ b/packages/respect-core/src/modules/flow-runner/runner.ts @@ -7,16 +7,12 @@ import { createTestContext } from './context/create-test-context'; import { getValueFromContext } from '../config-parser'; import { getWorkflowsToRun } from './get-workflows-to-run'; import { runStep } from './run-step'; -import { - printWorkflowSeparator, - indent, - printRequiredWorkflowSeparator, -} from '../../utils/cli-outputs'; +import { printWorkflowSeparator, printRequiredWorkflowSeparator } from '../../utils/cli-outputs'; import { bundleArazzo } from './get-test-description-from-file'; import { CHECKS } from '../checks'; import { createRuntimeExpressionCtx } from './context'; import { evaluateRuntimeExpressionPayload } from '../runtime-expressions'; -import { calculateTotals, composeJsonLogs, maskSecrets } from '../cli-output'; +import { calculateTotals, maskSecrets } from '../cli-output'; import { resolveRunningWorkflows } from './resolve-running-workflows'; import { DefaultLogger } from '../../utils/logger/logger'; @@ -30,6 +26,7 @@ import type { SourceDescription, Check, RunWorkflowInput, + WorkflowExecutionResult, } from '../../types'; const logger = DefaultLogger.getInstance(); @@ -71,29 +68,18 @@ export async function runTestFile( const bundledTestDescription = await bundleArazzo(filePath); collectSpecData?.(bundledTestDescription); - const descriptionCopy = JSON.parse(JSON.stringify(bundledTestDescription)); - const { harLogs, jsonLogs, workflows, secretFields } = await runWorkflows( - bundledTestDescription, - options - ); + const result = await runWorkflows(bundledTestDescription, options); - if (output?.harFile && Object.keys(harLogs).length) { - const parsedHarLogs = maskSecrets(harLogs, secretFields || new Set()); + if (output?.harFile && Object.keys(result.harLogs).length) { + const parsedHarLogs = maskSecrets(result.harLogs, result.ctx.secretFields || new Set()); writeFileSync(output.harFile, JSON.stringify(parsedHarLogs, null, 2), 'utf-8'); logger.log(blue(`Har logs saved in ${green(output.harFile)}`)); logger.printNewLine(); logger.printNewLine(); } - if (output?.jsonFile && jsonLogs) { - writeFileSync(output.jsonFile, JSON.stringify(jsonLogs, null, 2), 'utf-8'); - logger.log(blue(indent(`JSON logs saved in ${green(output.jsonFile)}`, 2))); - logger.printNewLine(); - logger.printNewLine(); - } - - return { workflows, parsedYaml: descriptionCopy }; + return result; } async function runWorkflows(testDescription: TestDescription, options: AppOptions) { @@ -109,22 +95,24 @@ async function runWorkflows(testDescription: TestDescription, options: AppOption const workflows = getWorkflowsToRun(ctx.workflows, workflowsToRun, workflowsToSkip); + const executedWorkflows: WorkflowExecutionResult[] = []; + for (const workflow of workflows) { + ctx.executedSteps = []; // run dependencies workflows first if (workflow.dependsOn?.length) { await handleDependsOn({ workflow, ctx }); } - await runWorkflow({ + const workflowExecutionResult = await runWorkflow({ workflowInput: workflow.workflowId, ctx, }); - } - // json logs should be composed after all workflows are run - const jsonLogs = options.jsonOutput ? composeJsonLogs(ctx) : undefined; + executedWorkflows.push(workflowExecutionResult); + } - return { ...ctx, harLogs, jsonLogs }; + return { ctx, harLogs, executedWorkflows }; } export async function runWorkflow({ @@ -132,7 +120,9 @@ export async function runWorkflow({ ctx, fromStepId, skipLineSeparator, -}: RunWorkflowInput): Promise { + parentStepId, + invocationContext, +}: RunWorkflowInput): Promise { const workflowStartTime = performance.now(); const fileBaseName = basename(ctx.options.workflowPath); const workflow = @@ -158,13 +148,14 @@ export async function runWorkflow({ for (const step of workflowSteps) { try { - const stepResults = await runStep({ + const stepResult = await runStep({ step, ctx, workflowId, }); + // When `end` action is used, we should not continue with the next steps - if (stepResults?.shouldEnd) { + if (stepResult?.shouldEnd) { break; } } catch (err: any) { @@ -175,7 +166,6 @@ export async function runWorkflow({ severity: ctx.severity['UNEXPECTED_ERROR'], }; step.checks.push(failedCall); - return; } } @@ -199,26 +189,38 @@ export async function runWorkflow({ workflowId, }); + const outputs: Record = {}; for (const outputKey of Object.keys(workflow.outputs)) { try { - if (workflow.outputs) { - workflow.outputs[outputKey] = evaluateRuntimeExpressionPayload({ - payload: workflow.outputs[outputKey], - context: runtimeExpressionContext, - }); - } - ctx.$outputs[workflowId].outputs = workflow.outputs; - ctx.$workflows[workflowId].outputs = workflow.outputs; + outputs[outputKey] = evaluateRuntimeExpressionPayload({ + payload: workflow.outputs[outputKey], + context: runtimeExpressionContext, + }); } catch (error: any) { - throw new Error(`Failed to resolve outputs in workflow "${workflowId}": ${error.message}`); + throw new Error( + `Failed to resolve output "${outputKey}" in workflow "${workflowId}": ${error.message}` + ); } } + ctx.$outputs[workflowId] = outputs; + ctx.$workflows[workflowId].outputs = outputs; } workflow.time = Math.ceil(performance.now() - workflowStartTime); logger.printNewLine(); - return workflow; + const endTime = performance.now(); + + return { + type: 'workflow', + invocationContext, + workflowId, + stepId: parentStepId, + startTime: workflowStartTime, + endTime, + totalTimeMs: Math.ceil(endTime - workflowStartTime), + executedSteps: ctx.executedSteps, + }; } async function handleDependsOn({ workflow, ctx }: { workflow: Workflow; ctx: TestContext }) { @@ -238,11 +240,7 @@ async function handleDependsOn({ workflow, ctx }: { workflow: Workflow; ctx: Tes }) ); - if (dependenciesWorkflows.some((w) => !w)) { - throw new Error('Dependent workflows failed'); - } - - const totals = calculateTotals(dependenciesWorkflows as Workflow[]); + const totals = calculateTotals(dependenciesWorkflows); const hasProblems = totals.steps.failed > 0; if (hasProblems) { @@ -280,11 +278,10 @@ export async function resolveWorkflowContext( }, ctx.apiClient ) - : await createTestContext( - JSON.parse(JSON.stringify(ctx.testDescription)), - JSON.parse(JSON.stringify(ctx.options)), - ctx.apiClient - ); + : { + ...ctx, + executedSteps: [], + }; } function findSourceDescriptionUrl( diff --git a/packages/respect-core/src/types.ts b/packages/respect-core/src/types.ts index b11b269d1c..727345c6cb 100644 --- a/packages/respect-core/src/types.ts +++ b/packages/respect-core/src/types.ts @@ -27,9 +27,8 @@ export type ResponseContext = { statusCode: number; body: any; header: Record; - query?: Record; - path?: Record; contentType?: string; + time?: number; } & Record; export type SourceDescription = FromSchema; type ArazzoParameter = FromSchema; @@ -73,7 +72,9 @@ type AdditionalStepProps = { verboseLog?: VerboseLog; response: ResponseContext; checks: Check[]; + retriesLeft?: number; }; + export type Step = ArazzoStep & AdditionalStepProps; export type Workflow = Omit & { steps: Step[]; time?: number }; export type RunArgv = Omit & { @@ -189,10 +190,56 @@ export type RunWorkflowInput = { workflowInput: Workflow | string; ctx: TestContext; fromStepId?: string; + parentStepId?: string; skipLineSeparator?: boolean; + invocationContext?: string; }; +export type ArrazoItemExecutionResult = StepExecutionResult | WorkflowExecutionResult; + +export type ExecutionStatus = 'success' | 'error' | 'warn'; +export interface StepExecutionResult { + type: 'step'; + stepId: string; + workflowId: string; + totalTimeMs: number; + + retriesLeft?: number; // number of retries + status: ExecutionStatus; + + invocationContext?: string; + + request?: { + method: string; + url: string; + headers: Record; + body: any; + }; + response?: { + statusCode: number; + body: any; + headers: Record; + time: number; + }; + checks: (Check & { status: ExecutionStatus })[]; +} + +export interface WorkflowExecutionResult { + type: 'workflow'; + workflowId: string; + stepId?: string; // for child workflows + sourceDescriptionName?: string; // maybe drop for now + + startTime: number; + endTime: number; + totalTimeMs: number; + + executedSteps: (Step | WorkflowExecutionResult)[]; + invocationContext?: string; +} + export type TestContext = RuntimeExpressionContext & { + executedSteps: (Step | WorkflowExecutionResult)[]; arazzo: string; info: InfoObject & Record; sourceDescriptions?: SourceDescription[]; @@ -238,15 +285,20 @@ export type CalculatedResults = { workflows: ResultsOfTests; steps: ResultsOfTests; checks: ResultsOfTests; + totalRequests: number; }; export type StepCallContext = { $response?: ResponseContext; $request?: RequestContext; $inputs?: Record; }; -type StepWithChecks = PublicStep & { checks?: Check[] }; -type WorkflowWithChecks = PublicWorkflow & { steps: Record; time?: number }; -export type JsonLogs = Record; + +export type JsonLogs = { + files: Record; + status: string; + totalTime: number; +}; + export type DescriptionChecks = { checks: Check[]; descriptionOperation: OperationDetails; diff --git a/packages/respect-core/src/utils/api-fetcher.ts b/packages/respect-core/src/utils/api-fetcher.ts index 24a8c730c9..254f1609d7 100644 --- a/packages/respect-core/src/utils/api-fetcher.ts +++ b/packages/respect-core/src/utils/api-fetcher.ts @@ -245,7 +245,7 @@ export class ApiFetcher implements IFetcher { ...(headers['content-type'] === 'application/octet-stream' && { duplex: 'half', }), - dispatcher: ctx.mtlsCerts ? createMtlsClient(resolvedPath, ctx.mtlsCerts) : undefined, + dispatcher: ctx.mtlsCerts ? createMtlsClient(urlToFetch, ctx.mtlsCerts) : undefined, }); const responseTime = Math.ceil(performance.now() - startTime); const res = await result.text(); @@ -287,6 +287,7 @@ export class ApiFetcher implements IFetcher { time: responseTime, header: Object.fromEntries(result.headers?.entries() || []), contentType: responseContentType, + requestUrl: urlToFetch, }; }; }