From 21d6c78fc42db16e3eeb2ba4e6422bb0747ac1a0 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:51:42 +0200 Subject: [PATCH 01/41] feat: implement new command scorecard-classic --- .../scorecard-classic/auth/login-handler.ts | 27 ++++ .../formatters/stylish-formatter.ts | 93 +++++++++++++ .../src/commands/scorecard-classic/index.ts | 53 ++++++++ .../remote/fetch-scorecard.ts | 123 ++++++++++++++++++ .../src/commands/scorecard-classic/types.ts | 30 +++++ .../validation/plugin-evaluator.ts | 32 +++++ .../validation/validate-scorecard.ts | 44 +++++++ packages/cli/src/index.ts | 23 +++- 8 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/commands/scorecard-classic/auth/login-handler.ts create mode 100644 packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts create mode 100644 packages/cli/src/commands/scorecard-classic/index.ts create mode 100644 packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts create mode 100644 packages/cli/src/commands/scorecard-classic/types.ts create mode 100644 packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts create mode 100644 packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts new file mode 100644 index 0000000000..0cb81bd412 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -0,0 +1,27 @@ +import { logger } from '@redocly/openapi-core'; +import { blue } from 'colorette'; +import { RedoclyOAuthClient } from '../../../auth/oauth-client.js'; +import { getReuniteUrl } from '../../../reunite/api/index.js'; +import { exitWithError } from '../../../utils/error.js'; + +import type { Config } from '@redocly/openapi-core'; + +export async function handleLoginAndFetchToken(config: Config): Promise { + const reuniteUrl = getReuniteUrl(config, config.resolvedConfig?.residency); + const oauthClient = new RedoclyOAuthClient(); + const isAuthorized = await oauthClient.isAuthorized(reuniteUrl); + + if (!isAuthorized) { + logger.info(`\n${blue('Authentication required to fetch remote scorecard configuration.')}\n`); + logger.info(`Please login to continue:\n`); + + try { + await oauthClient.login(reuniteUrl); + } catch (error) { + exitWithError(`Login failed. Please try again or check your connection to ${reuniteUrl}.`); + } + } + + const token = await oauthClient.getAccessToken(reuniteUrl); + return token === null ? undefined : token; +} diff --git a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts new file mode 100644 index 0000000000..efa620a38e --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts @@ -0,0 +1,93 @@ +import { logger, getLineColLocation } from '@redocly/openapi-core'; +import { gray, green, yellow, red, cyan, bold, white } from 'colorette'; + +import type { ScorecardProblem } from '../types.js'; + +function formatStylishProblem( + problem: ScorecardProblem, + locationPad: number, + ruleIdPad: number +): string { + const severityColor = + problem.severity === 'error' ? red : problem.severity === 'warn' ? yellow : gray; + + const loc = problem.location?.[0]; + let line = 0; + let column = 0; + + if (loc) { + const lineColLoc = getLineColLocation(loc); + line = lineColLoc.start.line; + column = lineColLoc.start.col; + } + + const location = `${line}:${column}`.padEnd(locationPad); + const severity = severityColor(problem.severity.padEnd(7)); + const ruleId = problem.ruleId.padEnd(ruleIdPad); + const level = cyan(`[${problem.scorecardLevel || 'Unknown'}]`); + + return ` ${location} ${severity} ${level} ${ruleId} ${problem.message}`; +} + +export function printScorecardResults(problems: ScorecardProblem[], apiPath: string): void { + logger.info(`\n${bold('Scorecard Classic results for')} ${cyan(apiPath)}:\n`); + + if (problems.length === 0) { + logger.info(green('āœ… No issues found! Your API meets all scorecard requirements.\n')); + return; + } + + const problemsByLevel = problems.reduce((acc, problem) => { + const level = problem.scorecardLevel || 'Unknown'; + if (!acc[level]) { + acc[level] = []; + } + acc[level].push(problem); + return acc; + }, {} as Record); + + const totalErrors = problems.filter((p) => p.severity === 'error').length; + const totalWarnings = problems.filter((p) => p.severity === 'warn').length; + const levelCount = Object.keys(problemsByLevel).length; + + logger.info( + white( + `Found ${bold(red(totalErrors.toString()))} error(s) and ${bold( + yellow(totalWarnings.toString()) + )} warning(s) across ${bold(cyan(levelCount.toString()))} level(s)\n` + ) + ); + + for (const [level, levelProblems] of Object.entries(problemsByLevel)) { + const severityCounts = levelProblems.reduce((acc, p) => { + acc[p.severity] = (acc[p.severity] || 0) + 1; + return acc; + }, {} as Record); + + logger.info( + bold(cyan(`\n šŸ“‹ ${level}`)) + + gray(` (${severityCounts.error || 0} errors, ${severityCounts.warn || 0} warnings) \n`) + ); + + // Calculate padding for alignment + const locationPad = Math.max( + ...levelProblems.map((p) => { + const loc = p.location?.[0]; + if (loc) { + const lineColLoc = getLineColLocation(loc); + return `${lineColLoc.start.line}:${lineColLoc.start.col}`.length; + } + return 3; // "0:0".length + }), + 8 // minimum padding + ); + + const ruleIdPad = Math.max(...levelProblems.map((p) => p.ruleId.length)); + + for (const problem of levelProblems) { + logger.output(`${formatStylishProblem(problem, locationPad, ruleIdPad)}\n`); + } + + logger.info(''); + } +} diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts new file mode 100644 index 0000000000..e69987bb54 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -0,0 +1,53 @@ +import { getFallbackApisOrExit } from '../../utils/miscellaneous.js'; +import { BaseResolver, bundle, logger } from '@redocly/openapi-core'; +import { exitWithError } from '../../utils/error.js'; +import { handleLoginAndFetchToken } from './auth/login-handler.js'; +import { printScorecardResults } from './formatters/stylish-formatter.js'; +import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js'; +import { validateScorecard } from './validation/validate-scorecard.js'; + +import type { ScorecardClassicArgv } from './types.js'; +import type { CommandArgs } from '../../wrapper.js'; + +export async function handleScorecardClassic({ argv, config }: CommandArgs) { + const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); + const externalRefResolver = new BaseResolver(config.resolve); + const { bundle: document } = await bundle({ config, ref: path }); + const projectUrl = argv['project-url'] || config.resolvedConfig.scorecard?.fromProjectUrl; + + if (!projectUrl) { + exitWithError( + 'scorecard.fromProjectUrl is not configured. Please provide it via --project-url flag or configure it in redocly.yaml. Learn more: https://redocly.com/docs/realm/config/scorecard#fromprojecturl-example' + ); + } + + const accessToken = await handleLoginAndFetchToken(config); + + if (!accessToken) { + exitWithError('Failed to obtain access token.'); + } + + const remoteScorecardAndPlugins = await fetchRemoteScorecardAndPlugins(projectUrl, accessToken); + + const scorecard = + remoteScorecardAndPlugins?.scorecard || + config.resolvedConfig.scorecardClassic || + config.resolvedConfig.scorecard; + + if (!scorecard) { + exitWithError( + 'No scorecard configuration found. Please configure scorecard in your redocly.yaml or ensure remote scorecard is accessible.' + ); + } + + logger.info(`\nRunning Scorecard Classic...\n`); + const result = await validateScorecard( + document, + externalRefResolver, + scorecard, + config.configPath, + remoteScorecardAndPlugins?.plugins + ); + + printScorecardResults(result, path); +} diff --git a/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts new file mode 100644 index 0000000000..a4547fb6b9 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts @@ -0,0 +1,123 @@ +import { logger } from '@redocly/openapi-core'; + +import type { RemoteScorecardAndPlugins, Organization, Project, PaginatedList } from '../types.js'; + +export async function fetchRemoteScorecardAndPlugins( + projectUrl: string, + accessToken: string +): Promise { + const parsedProjectUrl = parseProjectUrl(projectUrl); + + if (!parsedProjectUrl) { + logger.warn(`Invalid project URL format: ${projectUrl}`); + return; + } + + const { residency, orgSlug, projectSlug } = parsedProjectUrl; + + const organization = await fetchOrganizationBySlug(residency, orgSlug, accessToken); + + if (!organization) { + logger.warn(`Organization not found: ${orgSlug}`); + return; + } + + const project = await fetchProjectBySlug(residency, organization.id, projectSlug, accessToken); + + if (!project) { + logger.warn(`Project not found: ${projectSlug}`); + return; + } + + const scorecard = project?.config.scorecard; + + if (!scorecard) { + logger.warn('No scorecard configuration found in the remote project.'); + return; + } + + const plugins = project.config.pluginsUrl + ? await fetchPlugins(project.config.pluginsUrl) + : undefined; + + return { + scorecard, + plugins, + }; +} + +function parseProjectUrl( + projectUrl: string +): { residency: string; orgSlug: string; projectSlug: string } | undefined { + const url = new URL(projectUrl); + const match = url.pathname.match(/\/org\/(?[^/]+)\/project\/(?[^/]+)/); + + if (!match?.groups) { + return; + } + + const { orgSlug, projectSlug } = match.groups; + + return { + residency: url.origin, + orgSlug, + projectSlug, + }; +} + +async function fetchOrganizationBySlug( + residency: string, + orgSlug: string, + accessToken: string +): Promise { + const orgsUrl = new URL(`${residency}/api/orgs`); + orgsUrl.searchParams.set('filter', `slug:${orgSlug}`); + orgsUrl.searchParams.set('limit', '1'); + + const authHeaders = createAuthHeaders(accessToken); + const organizationResponse = await fetch(orgsUrl, { headers: authHeaders }); + + if (organizationResponse.status !== 200) { + return; + } + + const organizations: PaginatedList = await organizationResponse.json(); + + return organizations.items[0]; +} + +async function fetchProjectBySlug( + residency: string, + orgId: string, + projectSlug: string, + accessToken: string +): Promise { + const projectsUrl = new URL(`${residency}/api/orgs/${orgId}/projects`); + projectsUrl.searchParams.set('filter', `slug:${projectSlug}`); + projectsUrl.searchParams.set('limit', '1'); + + const authHeaders = createAuthHeaders(accessToken); + const projectsResponse = await fetch(projectsUrl, { headers: authHeaders }); + + if (projectsResponse.status !== 200) { + return; + } + + const projects: PaginatedList = await projectsResponse.json(); + + return projects.items[0]; +} + +async function fetchPlugins(pluginsUrl: string): Promise { + const pluginsResponse = await fetch(pluginsUrl); + + if (pluginsResponse.status !== 200) { + return; + } + + return pluginsResponse.text(); +} + +function createAuthHeaders(accessToken: string) { + return { Cookie: `accessToken=${accessToken}` }; +} diff --git a/packages/cli/src/commands/scorecard-classic/types.ts b/packages/cli/src/commands/scorecard-classic/types.ts new file mode 100644 index 0000000000..5b31764376 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/types.ts @@ -0,0 +1,30 @@ +import type { VerifyConfigOptions } from '../../types.js'; +import type { NormalizedProblem, ResolvedConfig } from '@redocly/openapi-core'; + +export type ScorecardClassicArgv = { + api: string; + config: string; + 'project-url'?: string; +} & VerifyConfigOptions; + +export type ScorecardProblem = NormalizedProblem & { scorecardLevel?: string }; + +export type RemoteScorecardAndPlugins = { + scorecard: ResolvedConfig['scorecard']; + plugins: string | undefined; +}; + +export type Organization = { + id: `org_${string}`; + slug: string; +}; + +export type Project = { + id: `prj_${string}`; + slug: string; + config: ResolvedConfig & { pluginsUrl?: string; scorecardClassic?: ResolvedConfig['scorecard'] }; +}; + +export type PaginatedList = { + items: T[]; +}; diff --git a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts new file mode 100644 index 0000000000..1167595ec8 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts @@ -0,0 +1,32 @@ +import type { Plugin } from '@redocly/openapi-core'; + +type PluginFunction = () => Plugin; + +type PluginsModule = { + default: PluginFunction[]; +}; + +export async function evaluatePluginsFromCode(pluginsCode?: string): Promise { + if (!pluginsCode) { + return []; + } + + try { + // TODO: hotfix for Windows. Related: https://github.com/Redocly/redocly/pull/18127 + const normalizedDirname = + typeof __dirname === 'undefined' ? '' : __dirname.replaceAll(/\\/g, '/'); + // https://github.com/Redocly/redocly/pull/17602 + const pluginsCodeWithDirname = pluginsCode.replaceAll( + '__redocly_dirname', + `"${normalizedDirname}"` + ); + const base64 = btoa(pluginsCodeWithDirname); + const dataUri = `data:text/javascript;base64,${base64}`; + const module: PluginsModule = await import(dataUri); + const evaluatedPlugins = module.default.map((pluginFunction) => pluginFunction()); + + return evaluatedPlugins; + } catch (error) { + return []; + } +} diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts new file mode 100644 index 0000000000..34ee63beb8 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -0,0 +1,44 @@ +import { createConfig, lintDocument } from '@redocly/openapi-core'; +import { evaluatePluginsFromCode } from './plugin-evaluator.js'; + +import type { ScorecardConfig } from '@redocly/config'; +import type { Document, RawUniversalConfig, Plugin, BaseResolver } from '@redocly/openapi-core'; +import type { ScorecardProblem } from '../types.js'; + +export async function validateScorecard( + document: Document, + externalRefResolver: BaseResolver, + scorecardConfig: ScorecardConfig, + configPath?: string, + pluginsCodeOrPlugins?: string | Plugin[] +): Promise { + const problems: ScorecardProblem[] = []; + + for (const level of scorecardConfig?.levels || []) { + const plugins = + typeof pluginsCodeOrPlugins === 'string' + ? await evaluatePluginsFromCode(pluginsCodeOrPlugins) + : pluginsCodeOrPlugins; + + const config = await createConfig({ ...level, plugins } as RawUniversalConfig, { + configPath, + }); + + const levelProblems = await lintDocument({ + document, + externalRefResolver, + config, + }); + + problems.push( + ...levelProblems + .filter(({ ignored }) => !ignored) + .map((problem) => ({ + ...problem, + scorecardLevel: level.name, + })) + ); + } + + return problems; +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 7f03fba463..d8c33833d9 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -30,10 +30,12 @@ import { version } from './utils/package.js'; import { validatePositiveNumber } from './utils/validate-positive-number.js'; import { validateMountPath } from './utils/validate-mount-path.js'; import { validateMtlsCommandOption } from './commands/respect/mtls/validate-mtls-command-option.js'; +import { handleScorecardClassic } from './commands/scorecard-classic/index.js'; import type { Arguments } from 'yargs'; import type { OutputFormat, RuleSeverity } from '@redocly/openapi-core'; import type { BuildDocsArgv } from './commands/build-docs/types.js'; +import type { ScorecardClassicArgv } from './commands/scorecard-classic/types.js'; import type { EjectArgv } from './commands/eject.js'; dotenv.config({ path: path.resolve(process.cwd(), './.env') }); @@ -604,7 +606,7 @@ yargs(hideBin(process.argv)) }) .check((argv: any) => { if (argv.theme && !argv.theme?.openapi) - throw Error('Invalid option: theme.openapi not set.'); + throw new Error('Invalid option: theme.openapi not set.'); return true; }), async (argv) => { @@ -796,6 +798,25 @@ yargs(hideBin(process.argv)) commandWrapper(handleGenerateArazzo)(argv as Arguments); } ) + .command( + 'scorecard-classic [api]', + 'Run quality scorecards with multiple rule levels to validate and maintain API description standards.', + (yargs) => { + return yargs.positional('api', { type: 'string' }).option({ + config: { + describe: 'Path to the config file.', + type: 'string', + }, + 'project-url': { + describe: 'URL to the project scorecard configuration.', + type: 'string', + }, + }); + }, + async (argv) => { + commandWrapper(handleScorecardClassic)(argv as Arguments); + } + ) .completion('completion', 'Generate autocomplete script for `redocly` command.') .demandCommand(1) .middleware([notifyUpdateCliVersion]) From 978fc3df5c903b2302722deb1d3fd8b2c155dff1 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 1 Dec 2025 18:55:12 +0200 Subject: [PATCH 02/41] chore: remove comments --- .../commands/scorecard-classic/validation/plugin-evaluator.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts index 1167595ec8..9ee47f8733 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts @@ -12,10 +12,8 @@ export async function evaluatePluginsFromCode(pluginsCode?: string): Promise Date: Tue, 2 Dec 2025 12:28:03 +0200 Subject: [PATCH 03/41] chore: add tests to meet unit test threshold --- .../__tests__/fetch-scorecard.test.ts | 249 ++++++++++++++++++ .../__tests__/plugin-evaluator.test.ts | 47 ++++ .../__tests__/stylish-formatter.test.ts | 36 +++ .../__tests__/validate-scorecard.test.ts | 148 +++++++++++ 4 files changed, 480 insertions(+) create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts new file mode 100644 index 0000000000..809ed24f1b --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -0,0 +1,249 @@ +import { fetchRemoteScorecardAndPlugins } from '../remote/fetch-scorecard.js'; +import * as openapiCore from '@redocly/openapi-core'; + +describe('fetchRemoteScorecardAndPlugins', () => { + const mockFetch = vi.fn(); + const validProjectUrl = 'https://app.redocly.com/org/test-org/project/test-project'; + const testToken = 'test-token'; + + beforeEach(() => { + global.fetch = mockFetch; + mockFetch.mockClear(); + vi.spyOn(openapiCore.logger, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should handle invalid URL format', async () => { + await expect(fetchRemoteScorecardAndPlugins('not-a-valid-url', testToken)).rejects.toThrow(); + }); + + it('should return undefined when project URL pattern does not match', async () => { + const result = await fetchRemoteScorecardAndPlugins( + 'https://example.com/invalid/path', + testToken + ); + + expect(result).toBeUndefined(); + expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Invalid project URL format') + ); + }); + + it('should return undefined when organization is not found', async () => { + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [] }), + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toBeUndefined(); + expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Organization not found') + ); + }); + + it('should return undefined when organization fetch fails', async () => { + mockFetch.mockResolvedValueOnce({ + status: 404, + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toBeUndefined(); + expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Organization not found') + ); + }); + + it('should return undefined when project is not found', async () => { + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [] }), + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toBeUndefined(); + expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('Project not found') + ); + }); + + it('should return undefined when project has no scorecard config', async () => { + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + items: [ + { + id: 'project-123', + slug: 'test-project', + config: {}, + }, + ], + }), + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toBeUndefined(); + expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect.stringContaining('No scorecard configuration found') + ); + }); + + it('should return scorecard config without plugins when pluginsUrl is not set', async () => { + const mockScorecard = { + levels: [{ name: 'Gold', rules: {} }], + }; + + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + items: [ + { + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + }, + }, + ], + }), + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toEqual({ + scorecard: mockScorecard, + plugins: undefined, + }); + expect(openapiCore.logger.warn).not.toHaveBeenCalled(); + }); + + it('should return scorecard config with plugins when pluginsUrl is set', async () => { + const mockScorecard = { + levels: [{ name: 'Gold', rules: {} }], + }; + const mockPluginsCode = 'export default [() => ({ id: "test-plugin" })]'; + + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + items: [ + { + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + pluginsUrl: 'https://example.com/plugins.js', + }, + }, + ], + }), + }) + .mockResolvedValueOnce({ + status: 200, + text: async () => mockPluginsCode, + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toEqual({ + scorecard: mockScorecard, + plugins: mockPluginsCode, + }); + expect(mockFetch).toHaveBeenCalledTimes(3); + }); + + it('should return scorecard without plugins when plugin fetch fails', async () => { + const mockScorecard = { + levels: [{ name: 'Gold', rules: {} }], + }; + + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), + }) + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + items: [ + { + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + pluginsUrl: 'https://example.com/plugins.js', + }, + }, + ], + }), + }) + .mockResolvedValueOnce({ + status: 404, + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(result).toEqual({ + scorecard: mockScorecard, + plugins: undefined, + }); + }); + + it('should use correct auth headers when fetching organization', async () => { + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [] }), + }); + + await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(mockFetch).toHaveBeenCalledWith( + expect.any(URL), + expect.objectContaining({ + headers: { Cookie: `accessToken=${testToken}` }, + }) + ); + }); + + it('should parse project URL with different residency', async () => { + const customUrl = 'https://custom.redocly.com/org/my-org/project/my-project'; + + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ items: [] }), + }); + + await fetchRemoteScorecardAndPlugins(customUrl, testToken); + + const callUrl = mockFetch.mock.calls[0][0].toString(); + expect(callUrl).toContain('https://custom.redocly.com/api/orgs'); + expect(callUrl).toContain('filter=slug%3Amy-org'); + }); +}); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts new file mode 100644 index 0000000000..054e098207 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts @@ -0,0 +1,47 @@ +import { evaluatePluginsFromCode } from '../validation/plugin-evaluator.js'; + +describe('evaluatePluginsFromCode', () => { + it('should return empty array when no plugins code provided', async () => { + const result = await evaluatePluginsFromCode(undefined); + expect(result).toEqual([]); + }); + + it('should return empty array when empty string provided', async () => { + const result = await evaluatePluginsFromCode(''); + expect(result).toEqual([]); + }); + + it('should return empty array on invalid plugin code', async () => { + const result = await evaluatePluginsFromCode('invalid code'); + expect(result).toEqual([]); + }); + + it('should evaluate valid plugin code and return plugins', async () => { + const validPluginCode = ` + export default [ + () => ({ + id: 'test-plugin', + rules: { + oas3: { + 'test-rule': () => ({}) + } + } + }) + ]; + `; + + const result = await evaluatePluginsFromCode(validPluginCode); + expect(result).toHaveLength(1); + expect(result[0]).toHaveProperty('id', 'test-plugin'); + }); + + it('should handle __redocly_dirname replacement', async () => { + const pluginCodeWithDirname = ` + const dirname = __redocly_dirname; + export default [() => ({ id: 'test', dirname })]; + `; + + const result = await evaluatePluginsFromCode(pluginCodeWithDirname); + expect(result).toHaveLength(1); + }); +}); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts new file mode 100644 index 0000000000..1b8ab8fa23 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -0,0 +1,36 @@ +import { printScorecardResults } from '../formatters/stylish-formatter.js'; +import * as openapiCore from '@redocly/openapi-core'; + +describe('printScorecardResults', () => { + beforeEach(() => { + vi.spyOn(openapiCore.logger, 'info').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should print success message when no problems', () => { + printScorecardResults([], 'test.yaml'); + + expect(openapiCore.logger.info).toHaveBeenCalledWith( + expect.stringContaining('No issues found') + ); + }); + + it('should print results when problems exist', () => { + const problems = [ + { + message: 'Error 1', + ruleId: 'rule-1', + severity: 'error' as const, + location: [], + scorecardLevel: 'Gold', + }, + ]; + + printScorecardResults(problems as any, 'test.yaml'); + + expect(openapiCore.logger.info).toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts new file mode 100644 index 0000000000..22420aeed0 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -0,0 +1,148 @@ +import { validateScorecard } from '../validation/validate-scorecard.js'; +import * as openapiCore from '@redocly/openapi-core'; +import { evaluatePluginsFromCode } from '../validation/plugin-evaluator.js'; + +vi.mock('../validation/plugin-evaluator.js', () => ({ + evaluatePluginsFromCode: vi.fn(), +})); + +describe('validateScorecard', () => { + const mockDocument = { + parsed: { + openapi: '3.0.0', + info: { title: 'Test API', version: '1.0.0' }, + paths: {}, + }, + source: { + absoluteRef: 'test.yaml', + }, + } as any; + + const mockResolver = {} as any; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(openapiCore, 'createConfig').mockResolvedValue({} as any); + vi.spyOn(openapiCore, 'lintDocument').mockResolvedValue([]); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should return empty array when no scorecard levels defined', async () => { + const scorecardConfig = { levels: [] }; + + const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + + expect(result).toEqual([]); + expect(openapiCore.lintDocument).not.toHaveBeenCalled(); + }); + + it('should validate each scorecard level', async () => { + const scorecardConfig = { + levels: [ + { name: 'Baseline', rules: {} }, + { name: 'Gold', rules: {} }, + ], + }; + + await validateScorecard(mockDocument, mockResolver, scorecardConfig); + + expect(openapiCore.createConfig).toHaveBeenCalledTimes(2); + expect(openapiCore.lintDocument).toHaveBeenCalledTimes(2); + }); + + it('should attach scorecard level name to problems', async () => { + const scorecardConfig = { + levels: [{ name: 'Gold', rules: {} }], + }; + + const mockProblems = [ + { + message: 'Test error', + ruleId: 'test-rule', + severity: 'error' as const, + location: [], + ignored: false, + }, + ]; + + vi.mocked(openapiCore.lintDocument).mockResolvedValue(mockProblems as any); + + const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + + expect(result).toHaveLength(1); + expect(result[0].scorecardLevel).toBe('Gold'); + expect(result[0].message).toBe('Test error'); + }); + + it('should filter out ignored problems', async () => { + const scorecardConfig = { + levels: [{ name: 'Baseline', rules: {} }], + }; + + const mockProblems = [ + { + message: 'Error 1', + ruleId: 'rule-1', + severity: 'error' as const, + location: [], + ignored: false, + }, + { + message: 'Error 2', + ruleId: 'rule-2', + severity: 'error' as const, + location: [], + ignored: true, + }, + ]; + + vi.mocked(openapiCore.lintDocument).mockResolvedValue(mockProblems as any); + + const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + + expect(result).toHaveLength(1); + expect(result[0].message).toBe('Error 1'); + }); + + it('should evaluate plugins from code when string provided', async () => { + const scorecardConfig = { + levels: [{ name: 'Gold', rules: {} }], + }; + + const mockPlugins = [{ id: 'test-plugin' }]; + vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins as any); + + await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, 'plugin-code'); + + expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code'); + expect(openapiCore.createConfig).toHaveBeenCalledWith( + expect.objectContaining({ plugins: mockPlugins }), + expect.any(Object) + ); + }); + + it('should use plugins directly when array provided', async () => { + const scorecardConfig = { + levels: [{ name: 'Gold', rules: {} }], + }; + + const mockPlugins = [{ id: 'test-plugin' }]; + + await validateScorecard( + mockDocument, + mockResolver, + scorecardConfig, + undefined, + mockPlugins as any + ); + + expect(evaluatePluginsFromCode).not.toHaveBeenCalled(); + expect(openapiCore.createConfig).toHaveBeenCalledWith( + expect.objectContaining({ plugins: mockPlugins }), + expect.any(Object) + ); + }); +}); From 2eb091caf4248091480b76eefca5028c4196d860 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:16:25 +0200 Subject: [PATCH 04/41] feat: add functionality to cache token --- .../src/commands/scorecard-classic/auth/login-handler.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index 0cb81bd412..13c9184122 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -9,19 +9,19 @@ import type { Config } from '@redocly/openapi-core'; export async function handleLoginAndFetchToken(config: Config): Promise { const reuniteUrl = getReuniteUrl(config, config.resolvedConfig?.residency); const oauthClient = new RedoclyOAuthClient(); - const isAuthorized = await oauthClient.isAuthorized(reuniteUrl); + let accessToken = await oauthClient.getAccessToken(reuniteUrl); - if (!isAuthorized) { + if (!accessToken) { logger.info(`\n${blue('Authentication required to fetch remote scorecard configuration.')}\n`); logger.info(`Please login to continue:\n`); try { await oauthClient.login(reuniteUrl); + accessToken = await oauthClient.getAccessToken(reuniteUrl); } catch (error) { exitWithError(`Login failed. Please try again or check your connection to ${reuniteUrl}.`); } } - const token = await oauthClient.getAccessToken(reuniteUrl); - return token === null ? undefined : token; + return accessToken === null ? undefined : accessToken; } From 7579685b2135d2192c8962ee14d26cc32dc9ea77 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 14:44:30 +0200 Subject: [PATCH 05/41] feat: implement functionality to fetch scorecard using api key --- .../scorecard-classic/auth/login-handler.ts | 6 +- .../src/commands/scorecard-classic/index.ts | 17 ++- .../remote/fetch-scorecard.ts | 111 +++++++----------- .../src/commands/scorecard-classic/types.ts | 9 -- .../validation/plugin-evaluator.ts | 9 +- 5 files changed, 56 insertions(+), 96 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index 13c9184122..c9fabccfc2 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -1,5 +1,3 @@ -import { logger } from '@redocly/openapi-core'; -import { blue } from 'colorette'; import { RedoclyOAuthClient } from '../../../auth/oauth-client.js'; import { getReuniteUrl } from '../../../reunite/api/index.js'; import { exitWithError } from '../../../utils/error.js'; @@ -8,13 +6,11 @@ import type { Config } from '@redocly/openapi-core'; export async function handleLoginAndFetchToken(config: Config): Promise { const reuniteUrl = getReuniteUrl(config, config.resolvedConfig?.residency); + const oauthClient = new RedoclyOAuthClient(); let accessToken = await oauthClient.getAccessToken(reuniteUrl); if (!accessToken) { - logger.info(`\n${blue('Authentication required to fetch remote scorecard configuration.')}\n`); - logger.info(`Please login to continue:\n`); - try { await oauthClient.login(reuniteUrl); accessToken = await oauthClient.getAccessToken(reuniteUrl); diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index e69987bb54..87c298f6a1 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -14,26 +14,23 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs { const parsedProjectUrl = parseProjectUrl(projectUrl); if (!parsedProjectUrl) { - logger.warn(`Invalid project URL format: ${projectUrl}`); - return; + exitWithError(`Invalid project URL format: ${projectUrl}`); } const { residency, orgSlug, projectSlug } = parsedProjectUrl; - - const organization = await fetchOrganizationBySlug(residency, orgSlug, accessToken); - - if (!organization) { - logger.warn(`Organization not found: ${orgSlug}`); - return; - } - - const project = await fetchProjectBySlug(residency, organization.id, projectSlug, accessToken); - - if (!project) { - logger.warn(`Project not found: ${projectSlug}`); - return; - } - - const scorecard = project?.config.scorecard; - - if (!scorecard) { - logger.warn('No scorecard configuration found in the remote project.'); - return; + const apiKey = process.env.REDOCLY_AUTHORIZATION; + + try { + const project = await fetchProjectConfigBySlugs(residency, orgSlug, projectSlug, apiKey, auth); + const scorecard = project?.config.scorecard; + + if (!scorecard) { + throw new Error('No scorecard configuration found.'); + } + + const plugins = project.config.pluginsUrl + ? await fetchPlugins(project.config.pluginsUrl) + : undefined; + + return { + scorecard, + plugins, + }; + } catch (error) { + exitWithError(error.message); } - - const plugins = project.config.pluginsUrl - ? await fetchPlugins(project.config.pluginsUrl) - : undefined; - - return { - scorecard, - plugins, - }; } function parseProjectUrl( @@ -65,47 +55,29 @@ function parseProjectUrl( }; } -async function fetchOrganizationBySlug( +async function fetchProjectConfigBySlugs( residency: string, orgSlug: string, - accessToken: string -): Promise { - const orgsUrl = new URL(`${residency}/api/orgs`); - orgsUrl.searchParams.set('filter', `slug:${orgSlug}`); - orgsUrl.searchParams.set('limit', '1'); - - const authHeaders = createAuthHeaders(accessToken); - const organizationResponse = await fetch(orgsUrl, { headers: authHeaders }); - - if (organizationResponse.status !== 200) { - return; - } - - const organizations: PaginatedList = await organizationResponse.json(); - - return organizations.items[0]; -} - -async function fetchProjectBySlug( - residency: string, - orgId: string, projectSlug: string, + apiKey: string | undefined, accessToken: string ): Promise { - const projectsUrl = new URL(`${residency}/api/orgs/${orgId}/projects`); - projectsUrl.searchParams.set('filter', `slug:${projectSlug}`); - projectsUrl.searchParams.set('limit', '1'); + const authHeaders = createAuthHeaders(apiKey, accessToken); + const projectUrl = new URL(`${residency}/api/orgs/${orgSlug}/projects/${projectSlug}`); - const authHeaders = createAuthHeaders(accessToken); - const projectsResponse = await fetch(projectsUrl, { headers: authHeaders }); + const projectResponse = await fetch(projectUrl, { headers: authHeaders }); - if (projectsResponse.status !== 200) { - return; + if (projectResponse.status === 401 || projectResponse.status === 403) { + throw new Error( + `Unauthorized access to project: ${projectSlug}. Please check your credentials.` + ); } - const projects: PaginatedList = await projectsResponse.json(); + if (projectResponse.status !== 200) { + throw new Error(`Failed to fetch project: ${projectSlug}. Status: ${projectResponse.status}`); + } - return projects.items[0]; + return projectResponse.json(); } async function fetchPlugins(pluginsUrl: string): Promise { @@ -118,6 +90,13 @@ async function fetchPlugins(pluginsUrl: string): Promise { return pluginsResponse.text(); } -function createAuthHeaders(accessToken: string) { +function createAuthHeaders( + apiKey: string | undefined, + accessToken: string +): Record { + if (apiKey) { + return { Authorization: `Bearer ${apiKey}` }; + } + return { Cookie: `accessToken=${accessToken}` }; } diff --git a/packages/cli/src/commands/scorecard-classic/types.ts b/packages/cli/src/commands/scorecard-classic/types.ts index 5b31764376..d16abf0bd6 100644 --- a/packages/cli/src/commands/scorecard-classic/types.ts +++ b/packages/cli/src/commands/scorecard-classic/types.ts @@ -14,17 +14,8 @@ export type RemoteScorecardAndPlugins = { plugins: string | undefined; }; -export type Organization = { - id: `org_${string}`; - slug: string; -}; - export type Project = { id: `prj_${string}`; slug: string; config: ResolvedConfig & { pluginsUrl?: string; scorecardClassic?: ResolvedConfig['scorecard'] }; }; - -export type PaginatedList = { - items: T[]; -}; diff --git a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts index 9ee47f8733..ffcc7d4d58 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts @@ -12,12 +12,9 @@ export async function evaluatePluginsFromCode(pluginsCode?: string): Promise Date: Tue, 9 Dec 2025 14:48:02 +0200 Subject: [PATCH 06/41] fix: import in fetchScorecard --- .../src/commands/scorecard-classic/remote/fetch-scorecard.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts index 4d888258e2..edaedf1d3f 100644 --- a/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts @@ -1,4 +1,4 @@ -import { exitWithError } from 'cli/src/utils/error.js'; +import { exitWithError } from '../../../utils/error.js'; import type { RemoteScorecardAndPlugins, Project } from '../types.js'; From 3db9ed397f7cf2363eb28c65cc9fa0371d8eb1f0 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:48:33 +0200 Subject: [PATCH 07/41] feat: implement functionality to output the scorecard results to JSON --- .../formatters/json-formatter.ts | 119 ++++++++++++++++++ .../src/commands/scorecard-classic/index.ts | 12 +- .../src/commands/scorecard-classic/types.ts | 1 + packages/cli/src/index.ts | 5 + 4 files changed, 134 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts diff --git a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts new file mode 100644 index 0000000000..c04ba4517e --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts @@ -0,0 +1,119 @@ +import { writeFileSync } from 'node:fs'; +import { logger, getLineColLocation } from '@redocly/openapi-core'; +import { blue, green } from 'colorette'; +import { formatPath, getExecutionTime } from '../../../utils/miscellaneous.js'; + +import type { ScorecardProblem } from '../types.js'; + +type ScorecardLevel = { + summary: { + errors: number; + warnings: number; + }; + problems: Array<{ + ruleId: string; + ruleUrl?: string; + severity: string; + message: string; + location: { + file: string; + range: string; + pointer?: string; + }[]; + }>; +}; + +export type ScorecardJsonOutput = Record; + +function formatRange( + start: { line: number; col: number }, + end?: { line: number; col: number } +): string { + const startStr = `Line ${start.line}, Col ${start.col}`; + if (!end) { + return startStr; + } + const endStr = `Line ${end.line}, Col ${end.col}`; + return `${startStr} ā†’ ${endStr}`; +} + +function getRuleUrl(ruleId: string): string | undefined { + if (!ruleId.includes('/')) { + return `https://redocly.com/docs/cli/rules/oas/${ruleId}.md`; + } + return undefined; +} + +function stripAnsiCodes(text: string): string { + // eslint-disable-next-line no-control-regex + return text.replace(/\u001b\[\d+m/g, ''); +} + +export function exportScorecardResultsToJson( + path: string, + problems: ScorecardProblem[], + outputPath: string +): void { + const startedAt = performance.now(); + const groupedByLevel: Record = {}; + + for (const problem of problems) { + const level = problem.scorecardLevel || 'Unknown'; + if (!groupedByLevel[level]) { + groupedByLevel[level] = []; + } + groupedByLevel[level].push(problem); + } + + const output: ScorecardJsonOutput = {}; + + for (const [levelName, levelProblems] of Object.entries(groupedByLevel)) { + let errors = 0; + let warnings = 0; + + const formattedProblems = levelProblems.map((problem) => { + if (problem.severity === 'error') errors++; + if (problem.severity === 'warn') warnings++; + + return { + ruleId: problem.ruleId, + ruleUrl: getRuleUrl(problem.ruleId), + severity: problem.severity, + message: stripAnsiCodes(problem.message), + + location: problem.location.map((loc) => { + const lineCol = getLineColLocation(loc); + return { + file: loc.source.absoluteRef, + range: formatRange(lineCol.start, lineCol.end), + pointer: loc.pointer, + }; + }), + }; + }); + + output[levelName] = { + summary: { + errors, + warnings, + }, + problems: formattedProblems, + }; + } + + try { + writeFileSync(outputPath, JSON.stringify(output, null, 2), 'utf-8'); + const elapsed = getExecutionTime(startedAt); + logger.info( + `šŸ“Š Scorecard results for ${blue(formatPath(path))} at ${blue( + outputPath || 'stdout' + )} ${green(elapsed)}.\n` + ); + } catch (error) { + logger.info( + `āŒ Errors encountered while bundling ${blue( + formatPath(path) + )}: bundle not created (use --force to ignore errors).\n` + ); + } +} diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 87c298f6a1..af1e22071b 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -1,10 +1,12 @@ -import { getFallbackApisOrExit } from '../../utils/miscellaneous.js'; +import { formatPath, getFallbackApisOrExit } from '../../utils/miscellaneous.js'; import { BaseResolver, bundle, logger } from '@redocly/openapi-core'; import { exitWithError } from '../../utils/error.js'; import { handleLoginAndFetchToken } from './auth/login-handler.js'; import { printScorecardResults } from './formatters/stylish-formatter.js'; +import { exportScorecardResultsToJson } from './formatters/json-formatter.js'; import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js'; import { validateScorecard } from './validation/validate-scorecard.js'; +import { gray } from 'colorette'; import type { ScorecardClassicArgv } from './types.js'; import type { CommandArgs } from '../../wrapper.js'; @@ -37,7 +39,7 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs { From d5fb8938632122a1ba3cecdbac684a30b4c56270 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:58:43 +0200 Subject: [PATCH 08/41] fix: outdated tests --- .../__tests__/fetch-scorecard.test.ts | 211 +++++++++--------- 1 file changed, 100 insertions(+), 111 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index 809ed24f1b..7546553c7d 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -1,5 +1,5 @@ import { fetchRemoteScorecardAndPlugins } from '../remote/fetch-scorecard.js'; -import * as openapiCore from '@redocly/openapi-core'; +import * as errorUtils from '../../../utils/error.js'; describe('fetchRemoteScorecardAndPlugins', () => { const mockFetch = vi.fn(); @@ -9,7 +9,9 @@ describe('fetchRemoteScorecardAndPlugins', () => { beforeEach(() => { global.fetch = mockFetch; mockFetch.mockClear(); - vi.spyOn(openapiCore.logger, 'warn').mockImplementation(() => {}); + vi.spyOn(errorUtils, 'exitWithError').mockImplementation(() => { + throw new Error('exitWithError called'); + }); }); afterEach(() => { @@ -20,87 +22,65 @@ describe('fetchRemoteScorecardAndPlugins', () => { await expect(fetchRemoteScorecardAndPlugins('not-a-valid-url', testToken)).rejects.toThrow(); }); - it('should return undefined when project URL pattern does not match', async () => { - const result = await fetchRemoteScorecardAndPlugins( - 'https://example.com/invalid/path', - testToken - ); + it('should throw error when project URL pattern does not match', async () => { + await expect( + fetchRemoteScorecardAndPlugins('https://example.com/invalid/path', testToken) + ).rejects.toThrow(); - expect(result).toBeUndefined(); - expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('Invalid project URL format') ); }); - it('should return undefined when organization is not found', async () => { + it('should throw error when project is not found (404)', async () => { mockFetch.mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [] }), + status: 404, }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); - expect(result).toBeUndefined(); - expect(openapiCore.logger.warn).toHaveBeenCalledWith( - expect.stringContaining('Organization not found') + expect(errorUtils.exitWithError).toHaveBeenCalledWith( + expect.stringContaining('Failed to fetch project') ); }); - it('should return undefined when organization fetch fails', async () => { + it('should throw error when unauthorized (401)', async () => { mockFetch.mockResolvedValueOnce({ - status: 404, + status: 401, }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); - expect(result).toBeUndefined(); - expect(openapiCore.logger.warn).toHaveBeenCalledWith( - expect.stringContaining('Organization not found') + expect(errorUtils.exitWithError).toHaveBeenCalledWith( + expect.stringContaining('Unauthorized access to project') ); }); - it('should return undefined when project is not found', async () => { - mockFetch - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), - }) - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [] }), - }); + it('should throw error when forbidden (403)', async () => { + mockFetch.mockResolvedValueOnce({ + status: 403, + }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); - expect(result).toBeUndefined(); - expect(openapiCore.logger.warn).toHaveBeenCalledWith( - expect.stringContaining('Project not found') + expect(errorUtils.exitWithError).toHaveBeenCalledWith( + expect.stringContaining('Unauthorized access to project') ); }); - it('should return undefined when project has no scorecard config', async () => { - mockFetch - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), - }) - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - items: [ - { - id: 'project-123', - slug: 'test-project', - config: {}, - }, - ], - }), - }); + it('should throw error when project has no scorecard config', async () => { + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ + id: 'project-123', + slug: 'test-project', + config: {}, + }), + }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); - expect(result).toBeUndefined(); - expect(openapiCore.logger.warn).toHaveBeenCalledWith( + expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('No scorecard configuration found') ); }); @@ -110,25 +90,16 @@ describe('fetchRemoteScorecardAndPlugins', () => { levels: [{ name: 'Gold', rules: {} }], }; - mockFetch - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), - }) - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ - items: [ - { - id: 'project-123', - slug: 'test-project', - config: { - scorecard: mockScorecard, - }, - }, - ], - }), - }); + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + }, + }), + }); const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); @@ -136,7 +107,7 @@ describe('fetchRemoteScorecardAndPlugins', () => { scorecard: mockScorecard, plugins: undefined, }); - expect(openapiCore.logger.warn).not.toHaveBeenCalled(); + expect(errorUtils.exitWithError).not.toHaveBeenCalled(); }); it('should return scorecard config with plugins when pluginsUrl is set', async () => { @@ -146,23 +117,15 @@ describe('fetchRemoteScorecardAndPlugins', () => { const mockPluginsCode = 'export default [() => ({ id: "test-plugin" })]'; mockFetch - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), - }) .mockResolvedValueOnce({ status: 200, json: async () => ({ - items: [ - { - id: 'project-123', - slug: 'test-project', - config: { - scorecard: mockScorecard, - pluginsUrl: 'https://example.com/plugins.js', - }, - }, - ], + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + pluginsUrl: 'https://example.com/plugins.js', + }, }), }) .mockResolvedValueOnce({ @@ -176,7 +139,7 @@ describe('fetchRemoteScorecardAndPlugins', () => { scorecard: mockScorecard, plugins: mockPluginsCode, }); - expect(mockFetch).toHaveBeenCalledTimes(3); + expect(mockFetch).toHaveBeenCalledTimes(2); }); it('should return scorecard without plugins when plugin fetch fails', async () => { @@ -185,23 +148,15 @@ describe('fetchRemoteScorecardAndPlugins', () => { }; mockFetch - .mockResolvedValueOnce({ - status: 200, - json: async () => ({ items: [{ id: 'org-123', slug: 'test-org' }] }), - }) .mockResolvedValueOnce({ status: 200, json: async () => ({ - items: [ - { - id: 'project-123', - slug: 'test-project', - config: { - scorecard: mockScorecard, - pluginsUrl: 'https://example.com/plugins.js', - }, - }, - ], + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + pluginsUrl: 'https://example.com/plugins.js', + }, }), }) .mockResolvedValueOnce({ @@ -216,10 +171,13 @@ describe('fetchRemoteScorecardAndPlugins', () => { }); }); - it('should use correct auth headers when fetching organization', async () => { + it('should use correct auth headers with access token', async () => { mockFetch.mockResolvedValueOnce({ status: 200, - json: async () => ({ items: [] }), + json: async () => ({ + id: 'project-123', + config: { scorecard: { levels: [] } }, + }), }); await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); @@ -232,18 +190,49 @@ describe('fetchRemoteScorecardAndPlugins', () => { ); }); + it('should use correct auth headers with API key', async () => { + const originalApiKey = process.env.REDOCLY_AUTHORIZATION; + process.env.REDOCLY_AUTHORIZATION = 'test-api-key'; + + mockFetch.mockResolvedValueOnce({ + status: 200, + json: async () => ({ + id: 'project-123', + config: { scorecard: { levels: [] } }, + }), + }); + + await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + + expect(mockFetch).toHaveBeenCalledWith( + expect.any(URL), + expect.objectContaining({ + headers: { Authorization: 'Bearer test-api-key' }, + }) + ); + + // Restore original value + if (originalApiKey) { + process.env.REDOCLY_AUTHORIZATION = originalApiKey; + } else { + delete process.env.REDOCLY_AUTHORIZATION; + } + }); + it('should parse project URL with different residency', async () => { const customUrl = 'https://custom.redocly.com/org/my-org/project/my-project'; mockFetch.mockResolvedValueOnce({ status: 200, - json: async () => ({ items: [] }), + json: async () => ({ + id: 'project-123', + config: { scorecard: { levels: [] } }, + }), }); await fetchRemoteScorecardAndPlugins(customUrl, testToken); const callUrl = mockFetch.mock.calls[0][0].toString(); - expect(callUrl).toContain('https://custom.redocly.com/api/orgs'); - expect(callUrl).toContain('filter=slug%3Amy-org'); + expect(callUrl).toBe('https://custom.redocly.com/api/orgs/my-org/projects/my-project'); }); }); From 360234630e3f2bb35fa27e0494eba46b6cd62eeb Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:58:14 +0200 Subject: [PATCH 09/41] refactor: logger output --- .../formatters/json-formatter.ts | 22 +--------- .../formatters/stylish-formatter.ts | 16 ++------ .../src/commands/scorecard-classic/index.ts | 40 +++++++++++++++++-- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts index c04ba4517e..609e473850 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts @@ -1,7 +1,5 @@ import { writeFileSync } from 'node:fs'; -import { logger, getLineColLocation } from '@redocly/openapi-core'; -import { blue, green } from 'colorette'; -import { formatPath, getExecutionTime } from '../../../utils/miscellaneous.js'; +import { getLineColLocation } from '@redocly/openapi-core'; import type { ScorecardProblem } from '../types.js'; @@ -50,11 +48,9 @@ function stripAnsiCodes(text: string): string { } export function exportScorecardResultsToJson( - path: string, problems: ScorecardProblem[], outputPath: string ): void { - const startedAt = performance.now(); const groupedByLevel: Record = {}; for (const problem of problems) { @@ -101,19 +97,5 @@ export function exportScorecardResultsToJson( }; } - try { - writeFileSync(outputPath, JSON.stringify(output, null, 2), 'utf-8'); - const elapsed = getExecutionTime(startedAt); - logger.info( - `šŸ“Š Scorecard results for ${blue(formatPath(path))} at ${blue( - outputPath || 'stdout' - )} ${green(elapsed)}.\n` - ); - } catch (error) { - logger.info( - `āŒ Errors encountered while bundling ${blue( - formatPath(path) - )}: bundle not created (use --force to ignore errors).\n` - ); - } + writeFileSync(outputPath, JSON.stringify(output, null, 2), 'utf-8'); } diff --git a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts index efa620a38e..8a080fae02 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts @@ -1,5 +1,5 @@ import { logger, getLineColLocation } from '@redocly/openapi-core'; -import { gray, green, yellow, red, cyan, bold, white } from 'colorette'; +import { gray, yellow, red, cyan, bold, white } from 'colorette'; import type { ScorecardProblem } from '../types.js'; @@ -29,14 +29,7 @@ function formatStylishProblem( return ` ${location} ${severity} ${level} ${ruleId} ${problem.message}`; } -export function printScorecardResults(problems: ScorecardProblem[], apiPath: string): void { - logger.info(`\n${bold('Scorecard Classic results for')} ${cyan(apiPath)}:\n`); - - if (problems.length === 0) { - logger.info(green('āœ… No issues found! Your API meets all scorecard requirements.\n')); - return; - } - +export function printScorecardResults(problems: ScorecardProblem[]): void { const problemsByLevel = problems.reduce((acc, problem) => { const level = problem.scorecardLevel || 'Unknown'; if (!acc[level]) { @@ -69,7 +62,6 @@ export function printScorecardResults(problems: ScorecardProblem[], apiPath: str gray(` (${severityCounts.error || 0} errors, ${severityCounts.warn || 0} warnings) \n`) ); - // Calculate padding for alignment const locationPad = Math.max( ...levelProblems.map((p) => { const loc = p.location?.[0]; @@ -77,9 +69,9 @@ export function printScorecardResults(problems: ScorecardProblem[], apiPath: str const lineColLoc = getLineColLocation(loc); return `${lineColLoc.start.line}:${lineColLoc.start.col}`.length; } - return 3; // "0:0".length + return 3; }), - 8 // minimum padding + 8 ); const ruleIdPad = Math.max(...levelProblems.map((p) => p.ruleId.length)); diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index af1e22071b..080e548c87 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -1,4 +1,4 @@ -import { formatPath, getFallbackApisOrExit } from '../../utils/miscellaneous.js'; +import { formatPath, getExecutionTime, getFallbackApisOrExit } from '../../utils/miscellaneous.js'; import { BaseResolver, bundle, logger } from '@redocly/openapi-core'; import { exitWithError } from '../../utils/error.js'; import { handleLoginAndFetchToken } from './auth/login-handler.js'; @@ -6,12 +6,13 @@ import { printScorecardResults } from './formatters/stylish-formatter.js'; import { exportScorecardResultsToJson } from './formatters/json-formatter.js'; import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js'; import { validateScorecard } from './validation/validate-scorecard.js'; -import { gray } from 'colorette'; +import { blue, gray, green } from 'colorette'; import type { ScorecardClassicArgv } from './types.js'; import type { CommandArgs } from '../../wrapper.js'; export async function handleScorecardClassic({ argv, config }: CommandArgs) { + const startedAt = performance.now(); const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); const externalRefResolver = new BaseResolver(config.resolve); const { bundle: document } = await bundle({ config, ref: path }); @@ -48,9 +49,40 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs Date: Tue, 9 Dec 2025 16:59:01 +0200 Subject: [PATCH 10/41] feat: add tests --- .../__tests__/json-formatter.test.ts | 242 ++++++++++++++++++ .../__tests__/stylish-formatter.test.ts | 117 ++++++++- 2 files changed, 352 insertions(+), 7 deletions(-) create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts new file mode 100644 index 0000000000..9767c2f270 --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -0,0 +1,242 @@ +import { readFileSync, unlinkSync } from 'node:fs'; +import { exportScorecardResultsToJson } from '../formatters/json-formatter.js'; +import type { ScorecardProblem } from '../types.js'; +import type { ScorecardJsonOutput } from '../formatters/json-formatter.js'; + +const createMockSource = (absoluteRef: string) => ({ + absoluteRef, + getAst: () => ({}), + getRootAst: () => ({}), + getLineColLocation: () => ({ line: 1, col: 1 }), +}); + +describe('exportScorecardResultsToJson', () => { + const testOutputPath = './test-scorecard-output.json'; + + afterEach(() => { + try { + unlinkSync(testOutputPath); + } catch { + // File might not exist + } + }); + + it('should export empty results when no problems', () => { + exportScorecardResultsToJson([], testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output).toEqual({}); + }); + + it('should group problems by scorecard level', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error in Gold level', + ruleId: 'test-rule-1', + severity: 'error', + suggest: [], + location: [ + { + source: createMockSource('/test/file.yaml') as any, + pointer: '#/paths/~1test/get', + reportOnKey: false, + }, + ], + scorecardLevel: 'Gold', + }, + { + message: 'Warning in Gold level', + ruleId: 'test-rule-2', + severity: 'warn', + suggest: [], + location: [ + { + source: createMockSource('/test/file.yaml') as any, + pointer: '#/info', + reportOnKey: false, + }, + ], + scorecardLevel: 'Gold', + }, + { + message: 'Error in Silver level', + ruleId: 'test-rule-3', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Silver', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(Object.keys(output)).toEqual(['Gold', 'Silver']); + expect(output.Gold.summary).toEqual({ errors: 1, warnings: 1 }); + expect(output.Gold.problems).toHaveLength(2); + expect(output.Silver.summary).toEqual({ errors: 1, warnings: 0 }); + expect(output.Silver.problems).toHaveLength(1); + }); + + it('should include rule URLs for non-namespaced rules', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Test error', + ruleId: 'operation-summary', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output.Gold.problems[0].ruleUrl).toBe( + 'https://redocly.com/docs/cli/rules/oas/operation-summary.md' + ); + }); + + it('should not include rule URLs for namespaced rules', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Test error', + ruleId: 'custom/my-rule', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output.Gold.problems[0].ruleUrl).toBeUndefined(); + }); + + it('should format location with file path and range', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Test error', + ruleId: 'test-rule', + severity: 'error', + suggest: [], + location: [ + { + source: createMockSource('/test/file.yaml') as any, + pointer: '#/paths/~1test/get', + reportOnKey: false, + }, + ], + scorecardLevel: 'Gold', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output.Gold.problems[0].location).toHaveLength(1); + expect(output.Gold.problems[0].location[0].file).toBe('/test/file.yaml'); + expect(output.Gold.problems[0].location[0].pointer).toBe('#/paths/~1test/get'); + expect(output.Gold.problems[0].location[0].range).toContain('Line'); + }); + + it('should handle problems with Unknown level', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error without level', + ruleId: 'test-rule', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: undefined, + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(Object.keys(output)).toEqual(['Unknown']); + expect(output.Unknown.problems).toHaveLength(1); + }); + + it('should strip ANSI codes from messages', () => { + const problems: ScorecardProblem[] = [ + { + message: '\u001b[31mError message with color\u001b[0m', + ruleId: 'test-rule', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output.Gold.problems[0].message).toBe('Error message with color'); + expect(output.Gold.problems[0].message).not.toContain('\u001b'); + }); + + it('should count errors and warnings correctly', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error 1', + ruleId: 'rule-1', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + { + message: 'Error 2', + ruleId: 'rule-2', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + { + message: 'Warning 1', + ruleId: 'rule-3', + severity: 'warn', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + { + message: 'Warning 2', + ruleId: 'rule-4', + severity: 'warn', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + { + message: 'Warning 3', + ruleId: 'rule-5', + severity: 'warn', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + exportScorecardResultsToJson(problems, testOutputPath); + + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + + expect(output.Gold.summary.errors).toBe(2); + expect(output.Gold.summary.warnings).toBe(3); + }); +}); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts index 1b8ab8fa23..f43272f769 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -1,9 +1,18 @@ import { printScorecardResults } from '../formatters/stylish-formatter.js'; import * as openapiCore from '@redocly/openapi-core'; +import type { ScorecardProblem } from '../types.js'; + +const createMockSource = (absoluteRef: string) => ({ + absoluteRef, + getAst: () => ({}), + getRootAst: () => ({}), + getLineColLocation: () => ({ line: 1, col: 1 }), +}); describe('printScorecardResults', () => { beforeEach(() => { vi.spyOn(openapiCore.logger, 'info').mockImplementation(() => {}); + vi.spyOn(openapiCore.logger, 'output').mockImplementation(() => {}); }); afterEach(() => { @@ -11,26 +20,120 @@ describe('printScorecardResults', () => { }); it('should print success message when no problems', () => { - printScorecardResults([], 'test.yaml'); + printScorecardResults([]); + + expect(openapiCore.logger.info).toHaveBeenCalledWith( + expect.stringMatching(/Found.*0.*error.*0.*warning.*0.*level/) + ); + }); + + it('should handle problems without location', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error without location', + ruleId: 'test-rule', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + printScorecardResults(problems); + expect(openapiCore.logger.output).toHaveBeenCalled(); expect(openapiCore.logger.info).toHaveBeenCalledWith( - expect.stringContaining('No issues found') + expect.stringMatching(/Found.*1.*error.*0.*warning.*1.*level/) ); + expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); }); - it('should print results when problems exist', () => { - const problems = [ + it('should handle problems with Unknown level', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error without level', + ruleId: 'test-rule', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: undefined, + }, + ]; + + printScorecardResults(problems); + + expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('Unknown')); + }); + + it('should show correct severity counts per level', () => { + const problems: ScorecardProblem[] = [ { message: 'Error 1', ruleId: 'rule-1', - severity: 'error' as const, + severity: 'error', + suggest: [], location: [], scorecardLevel: 'Gold', }, + { + message: 'Error 2', + ruleId: 'rule-2', + severity: 'error', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + { + message: 'Warning 1', + ruleId: 'rule-3', + severity: 'warn', + suggest: [], + location: [], + scorecardLevel: 'Gold', + }, + ]; + + printScorecardResults(problems); + + expect(openapiCore.logger.info).toHaveBeenCalledWith( + expect.stringContaining('2 errors, 1 warnings') + ); + }); + + it('should calculate correct padding for alignment', () => { + const problems: ScorecardProblem[] = [ + { + message: 'Error 1', + ruleId: 'short', + severity: 'error', + suggest: [], + location: [ + { + source: createMockSource('/test/file.yaml') as any, + pointer: '#/paths/~1test/get', + reportOnKey: false, + }, + ], + scorecardLevel: 'Gold', + }, + { + message: 'Error 2', + ruleId: 'very-long-rule-id-name', + severity: 'error', + suggest: [], + location: [ + { + source: createMockSource('/test/file.yaml') as any, + pointer: '#/info', + reportOnKey: false, + }, + ], + scorecardLevel: 'Gold', + }, ]; - printScorecardResults(problems as any, 'test.yaml'); + printScorecardResults(problems); - expect(openapiCore.logger.info).toHaveBeenCalled(); + expect(openapiCore.logger.output).toHaveBeenCalledTimes(2); }); }); From eefd80053f7d17b81633b88ee876ddd9ff519220 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:03:41 +0200 Subject: [PATCH 11/41] chore: change url in tests --- .../scorecard-classic/__tests__/fetch-scorecard.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index 7546553c7d..725ba6c56b 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -3,7 +3,7 @@ import * as errorUtils from '../../../utils/error.js'; describe('fetchRemoteScorecardAndPlugins', () => { const mockFetch = vi.fn(); - const validProjectUrl = 'https://app.redocly.com/org/test-org/project/test-project'; + const validProjectUrl = 'https://app.valid-url.com/org/test-org/project/test-project'; const testToken = 'test-token'; beforeEach(() => { From 05da2d3f6ea49070fe03ca02241c52f771a89477 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:43:26 +0200 Subject: [PATCH 12/41] refactor: tests for scorecards --- .../__tests__/fetch-scorecard.test.ts | 26 ++----------------- .../__tests__/json-formatter.test.ts | 2 +- .../__tests__/stylish-formatter.test.ts | 8 ------ .../__tests__/validate-scorecard.test.ts | 16 ++++-------- 4 files changed, 8 insertions(+), 44 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index 725ba6c56b..3715b43b62 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -15,6 +15,8 @@ describe('fetchRemoteScorecardAndPlugins', () => { }); afterEach(() => { + vi.unstubAllEnvs(); + delete process.env.REDOCLY_AUTHORIZATION; vi.restoreAllMocks(); }); @@ -210,29 +212,5 @@ describe('fetchRemoteScorecardAndPlugins', () => { headers: { Authorization: 'Bearer test-api-key' }, }) ); - - // Restore original value - if (originalApiKey) { - process.env.REDOCLY_AUTHORIZATION = originalApiKey; - } else { - delete process.env.REDOCLY_AUTHORIZATION; - } - }); - - it('should parse project URL with different residency', async () => { - const customUrl = 'https://custom.redocly.com/org/my-org/project/my-project'; - - mockFetch.mockResolvedValueOnce({ - status: 200, - json: async () => ({ - id: 'project-123', - config: { scorecard: { levels: [] } }, - }), - }); - - await fetchRemoteScorecardAndPlugins(customUrl, testToken); - - const callUrl = mockFetch.mock.calls[0][0].toString(); - expect(callUrl).toBe('https://custom.redocly.com/api/orgs/my-org/projects/my-project'); }); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts index 9767c2f270..a5be3bafd9 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -24,7 +24,7 @@ describe('exportScorecardResultsToJson', () => { it('should export empty results when no problems', () => { exportScorecardResultsToJson([], testOutputPath); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')); expect(output).toEqual({}); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts index f43272f769..e64c23c05f 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -19,14 +19,6 @@ describe('printScorecardResults', () => { vi.restoreAllMocks(); }); - it('should print success message when no problems', () => { - printScorecardResults([]); - - expect(openapiCore.logger.info).toHaveBeenCalledWith( - expect.stringMatching(/Found.*0.*error.*0.*warning.*0.*level/) - ); - }); - it('should handle problems without location', () => { const problems: ScorecardProblem[] = [ { diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index 22420aeed0..5ce9f73d08 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -62,7 +62,7 @@ describe('validateScorecard', () => { { message: 'Test error', ruleId: 'test-rule', - severity: 'error' as const, + severity: 'error', location: [], ignored: false, }, @@ -86,14 +86,14 @@ describe('validateScorecard', () => { { message: 'Error 1', ruleId: 'rule-1', - severity: 'error' as const, + severity: 'error', location: [], ignored: false, }, { message: 'Error 2', ruleId: 'rule-2', - severity: 'error' as const, + severity: 'error', location: [], ignored: true, }, @@ -113,7 +113,7 @@ describe('validateScorecard', () => { }; const mockPlugins = [{ id: 'test-plugin' }]; - vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins as any); + vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins); await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, 'plugin-code'); @@ -131,13 +131,7 @@ describe('validateScorecard', () => { const mockPlugins = [{ id: 'test-plugin' }]; - await validateScorecard( - mockDocument, - mockResolver, - scorecardConfig, - undefined, - mockPlugins as any - ); + await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, mockPlugins); expect(evaluatePluginsFromCode).not.toHaveBeenCalled(); expect(openapiCore.createConfig).toHaveBeenCalledWith( From ad0af6a0de3d1f3b5d27705c769dd64345aa2031 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Tue, 9 Dec 2025 18:11:53 +0200 Subject: [PATCH 13/41] fix: change output argument to format --- .../__tests__/json-formatter.test.ts | 60 ++++++++++--------- .../formatters/json-formatter.ts | 11 ++-- .../src/commands/scorecard-classic/index.ts | 33 ++++------ .../src/commands/scorecard-classic/types.ts | 4 +- packages/cli/src/index.ts | 8 +-- 5 files changed, 51 insertions(+), 65 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts index a5be3bafd9..bb68f5377e 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -1,7 +1,6 @@ -import { readFileSync, unlinkSync } from 'node:fs'; -import { exportScorecardResultsToJson } from '../formatters/json-formatter.js'; +import { printScorecardResultsAsJson } from '../formatters/json-formatter.js'; +import * as openapiCore from '@redocly/openapi-core'; import type { ScorecardProblem } from '../types.js'; -import type { ScorecardJsonOutput } from '../formatters/json-formatter.js'; const createMockSource = (absoluteRef: string) => ({ absoluteRef, @@ -10,23 +9,19 @@ const createMockSource = (absoluteRef: string) => ({ getLineColLocation: () => ({ line: 1, col: 1 }), }); -describe('exportScorecardResultsToJson', () => { - const testOutputPath = './test-scorecard-output.json'; +describe('printScorecardResultsAsJson', () => { + beforeEach(() => { + vi.spyOn(openapiCore.logger, 'output').mockImplementation(() => {}); + }); afterEach(() => { - try { - unlinkSync(testOutputPath); - } catch { - // File might not exist - } + vi.restoreAllMocks(); }); - it('should export empty results when no problems', () => { - exportScorecardResultsToJson([], testOutputPath); - - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')); + it('should print empty results when no problems', () => { + printScorecardResultsAsJson([]); - expect(output).toEqual({}); + expect(openapiCore.logger.output).toHaveBeenCalledWith('{}'); }); it('should group problems by scorecard level', () => { @@ -69,9 +64,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(Object.keys(output)).toEqual(['Gold', 'Silver']); expect(output.Gold.summary).toEqual({ errors: 1, warnings: 1 }); @@ -92,9 +88,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(output.Gold.problems[0].ruleUrl).toBe( 'https://redocly.com/docs/cli/rules/oas/operation-summary.md' @@ -113,9 +110,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(output.Gold.problems[0].ruleUrl).toBeUndefined(); }); @@ -138,9 +136,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(output.Gold.problems[0].location).toHaveLength(1); expect(output.Gold.problems[0].location[0].file).toBe('/test/file.yaml'); @@ -160,9 +159,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(Object.keys(output)).toEqual(['Unknown']); expect(output.Unknown.problems).toHaveLength(1); @@ -180,9 +180,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(output.Gold.problems[0].message).toBe('Error message with color'); expect(output.Gold.problems[0].message).not.toContain('\u001b'); @@ -232,9 +233,10 @@ describe('exportScorecardResultsToJson', () => { }, ]; - exportScorecardResultsToJson(problems, testOutputPath); + printScorecardResultsAsJson(problems); - const output = JSON.parse(readFileSync(testOutputPath, 'utf-8')) as ScorecardJsonOutput; + const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; + const output = JSON.parse(outputCall); expect(output.Gold.summary.errors).toBe(2); expect(output.Gold.summary.warnings).toBe(3); diff --git a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts index 609e473850..7453e12acf 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts @@ -1,5 +1,4 @@ -import { writeFileSync } from 'node:fs'; -import { getLineColLocation } from '@redocly/openapi-core'; +import { logger, getLineColLocation } from '@redocly/openapi-core'; import type { ScorecardProblem } from '../types.js'; @@ -47,10 +46,7 @@ function stripAnsiCodes(text: string): string { return text.replace(/\u001b\[\d+m/g, ''); } -export function exportScorecardResultsToJson( - problems: ScorecardProblem[], - outputPath: string -): void { +export function printScorecardResultsAsJson(problems: ScorecardProblem[]): void { const groupedByLevel: Record = {}; for (const problem of problems) { @@ -97,5 +93,6 @@ export function exportScorecardResultsToJson( }; } - writeFileSync(outputPath, JSON.stringify(output, null, 2), 'utf-8'); + logger.output(JSON.stringify(output, null, 2)); + logger.info('\n'); } diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 080e548c87..0a8bf99751 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -3,7 +3,7 @@ import { BaseResolver, bundle, logger } from '@redocly/openapi-core'; import { exitWithError } from '../../utils/error.js'; import { handleLoginAndFetchToken } from './auth/login-handler.js'; import { printScorecardResults } from './formatters/stylish-formatter.js'; -import { exportScorecardResultsToJson } from './formatters/json-formatter.js'; +import { printScorecardResultsAsJson } from './formatters/json-formatter.js'; import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js'; import { validateScorecard } from './validation/validate-scorecard.js'; import { blue, gray, green } from 'colorette'; @@ -60,29 +60,16 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs Date: Wed, 10 Dec 2025 14:20:06 +0200 Subject: [PATCH 14/41] refactor: remove unused errors in catch and add warning log --- .../scorecard-classic/auth/login-handler.ts | 2 +- .../formatters/json-formatter.ts | 2 +- .../src/commands/scorecard-classic/index.ts | 15 ++++----- .../remote/fetch-scorecard.ts | 33 ++++++++++--------- .../validation/plugin-evaluator.ts | 5 ++- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index c9fabccfc2..0c689ae5c0 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -14,7 +14,7 @@ export async function handleLoginAndFetchToken(config: Config): Promise { + auth: string, + isApiKey = false +): Promise { const parsedProjectUrl = parseProjectUrl(projectUrl); if (!parsedProjectUrl) { @@ -13,10 +14,15 @@ export async function fetchRemoteScorecardAndPlugins( } const { residency, orgSlug, projectSlug } = parsedProjectUrl; - const apiKey = process.env.REDOCLY_AUTHORIZATION; try { - const project = await fetchProjectConfigBySlugs(residency, orgSlug, projectSlug, apiKey, auth); + const project = await fetchProjectConfigBySlugs( + residency, + orgSlug, + projectSlug, + auth, + isApiKey + ); const scorecard = project?.config.scorecard; if (!scorecard) { @@ -28,7 +34,7 @@ export async function fetchRemoteScorecardAndPlugins( : undefined; return { - scorecard, + scorecard: scorecard!, plugins, }; } catch (error) { @@ -59,10 +65,10 @@ async function fetchProjectConfigBySlugs( residency: string, orgSlug: string, projectSlug: string, - apiKey: string | undefined, - accessToken: string + auth: string, + isApiKey: boolean ): Promise { - const authHeaders = createAuthHeaders(apiKey, accessToken); + const authHeaders = createAuthHeaders(auth, isApiKey); const projectUrl = new URL(`${residency}/api/orgs/${orgSlug}/projects/${projectSlug}`); const projectResponse = await fetch(projectUrl, { headers: authHeaders }); @@ -90,13 +96,10 @@ async function fetchPlugins(pluginsUrl: string): Promise { return pluginsResponse.text(); } -function createAuthHeaders( - apiKey: string | undefined, - accessToken: string -): Record { - if (apiKey) { - return { Authorization: `Bearer ${apiKey}` }; +function createAuthHeaders(auth: string, isApiKey: boolean): Record { + if (isApiKey) { + return { Authorization: `Bearer ${auth}` }; } - return { Cookie: `accessToken=${accessToken}` }; + return { Cookie: `accessToken=${auth}` }; } diff --git a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts index ffcc7d4d58..29302b9b35 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts @@ -1,3 +1,5 @@ +import { logger } from '@redocly/openapi-core'; + import type { Plugin } from '@redocly/openapi-core'; type PluginFunction = () => Plugin; @@ -21,7 +23,8 @@ export async function evaluatePluginsFromCode(pluginsCode?: string): Promise pluginFunction()); return evaluatedPlugins; - } catch (error) { + } catch { + logger.warn(`Something went wrong during plugins evaluation.`); return []; } } From 107ce9142a2c9a303ce074d71120d9c4f50e971a Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Wed, 10 Dec 2025 15:06:50 +0200 Subject: [PATCH 15/41] refactor: return statement in login handler --- .../cli/src/commands/scorecard-classic/auth/login-handler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index 0c689ae5c0..d57a80e8e8 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -4,7 +4,7 @@ import { exitWithError } from '../../../utils/error.js'; import type { Config } from '@redocly/openapi-core'; -export async function handleLoginAndFetchToken(config: Config): Promise { +export async function handleLoginAndFetchToken(config: Config): Promise { const reuniteUrl = getReuniteUrl(config, config.resolvedConfig?.residency); const oauthClient = new RedoclyOAuthClient(); @@ -19,5 +19,5 @@ export async function handleLoginAndFetchToken(config: Config): Promise Date: Wed, 10 Dec 2025 15:33:33 +0200 Subject: [PATCH 16/41] fix: remove bundling before linting, change to resolveDocument --- packages/cli/src/commands/scorecard-classic/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index ae79762b25..1e6d518422 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -1,5 +1,5 @@ import { formatPath, getExecutionTime, getFallbackApisOrExit } from '../../utils/miscellaneous.js'; -import { BaseResolver, bundle, logger } from '@redocly/openapi-core'; +import { BaseResolver, logger } from '@redocly/openapi-core'; import { exitWithError } from '../../utils/error.js'; import { handleLoginAndFetchToken } from './auth/login-handler.js'; import { printScorecardResults } from './formatters/stylish-formatter.js'; @@ -10,12 +10,14 @@ import { blue, gray, green } from 'colorette'; import type { ScorecardClassicArgv } from './types.js'; import type { CommandArgs } from '../../wrapper.js'; +import type { Document } from '@redocly/openapi-core'; export async function handleScorecardClassic({ argv, config }: CommandArgs) { const startedAt = performance.now(); const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); const externalRefResolver = new BaseResolver(config.resolve); - const { bundle: document } = await bundle({ config, ref: path }); + const document = (await externalRefResolver.resolveDocument(null, path, true)) as Document; + const projectUrl = argv['project-url'] || config.resolvedConfig.scorecard?.fromProjectUrl; const apiKey = process.env.REDOCLY_AUTHORIZATION; From e24663f4d3b0148b2842bcb0c114c4e57a6b5c05 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Wed, 10 Dec 2025 19:09:26 +0200 Subject: [PATCH 17/41] feat: add verbose flag and detailed logs, add pluralize to logger messages --- .../scorecard-classic/auth/login-handler.ts | 29 +++++-- .../formatters/stylish-formatter.ts | 14 +++- .../src/commands/scorecard-classic/index.ts | 12 ++- .../remote/fetch-scorecard.ts | 78 ++++++++++++++++--- .../src/commands/scorecard-classic/types.ts | 1 + .../validation/plugin-evaluator.ts | 44 ++++++++++- .../validation/validate-scorecard.ts | 29 ++++++- packages/cli/src/index.ts | 5 ++ 8 files changed, 183 insertions(+), 29 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index d57a80e8e8..3e8341e599 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -1,22 +1,37 @@ +import { logger } from '@redocly/openapi-core'; import { RedoclyOAuthClient } from '../../../auth/oauth-client.js'; import { getReuniteUrl } from '../../../reunite/api/index.js'; import { exitWithError } from '../../../utils/error.js'; import type { Config } from '@redocly/openapi-core'; -export async function handleLoginAndFetchToken(config: Config): Promise { +export async function handleLoginAndFetchToken( + config: Config, + verbose = false +): Promise { const reuniteUrl = getReuniteUrl(config, config.resolvedConfig?.residency); const oauthClient = new RedoclyOAuthClient(); let accessToken = await oauthClient.getAccessToken(reuniteUrl); - if (!accessToken) { - try { - await oauthClient.login(reuniteUrl); - accessToken = await oauthClient.getAccessToken(reuniteUrl); - } catch { - exitWithError(`Login failed. Please try again or check your connection to ${reuniteUrl}.`); + if (accessToken) { + logger.info(`āœ… Found existing access token.\n`); + return accessToken; + } + + if (verbose) { + logger.warn(`No access token found. Attempting login...\n`); + } + + try { + await oauthClient.login(reuniteUrl); + accessToken = await oauthClient.getAccessToken(reuniteUrl); + } catch (error) { + if (verbose) { + logger.error(`āŒ Login failed.\n`); + logger.error(`Error details: ${error.message}\n`); } + exitWithError(`Login failed. Please try again or check your connection to ${reuniteUrl}.`); } return accessToken; diff --git a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts index 8a080fae02..e6f072e0fb 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts @@ -1,4 +1,4 @@ -import { logger, getLineColLocation } from '@redocly/openapi-core'; +import { logger, getLineColLocation, pluralize } from '@redocly/openapi-core'; import { gray, yellow, red, cyan, bold, white } from 'colorette'; import type { ScorecardProblem } from '../types.js'; @@ -45,9 +45,11 @@ export function printScorecardResults(problems: ScorecardProblem[]): void { logger.info( white( - `Found ${bold(red(totalErrors.toString()))} error(s) and ${bold( + `Found ${bold(red(totalErrors.toString()))} ${pluralize('error', totalErrors)} and ${bold( yellow(totalWarnings.toString()) - )} warning(s) across ${bold(cyan(levelCount.toString()))} level(s)\n` + )} ${pluralize('warning', totalWarnings)} across ${bold( + cyan(levelCount.toString()) + )} ${pluralize('level', levelCount)}\n` ) ); @@ -59,7 +61,11 @@ export function printScorecardResults(problems: ScorecardProblem[]): void { logger.info( bold(cyan(`\n šŸ“‹ ${level}`)) + - gray(` (${severityCounts.error || 0} errors, ${severityCounts.warn || 0} warnings) \n`) + gray( + ` (${severityCounts.error || 0} ${pluralize('error', severityCounts.error || 0)}, ${ + severityCounts.warn || 0 + } ${pluralize('warning', severityCounts.warn || 0)}) \n` + ) ); const locationPad = Math.max( diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 1e6d518422..ed1e4af66c 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -21,13 +21,17 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs { + if (verbose) { + logger.info(`Starting fetch for remote scorecard configuration...\n`); + } + const parsedProjectUrl = parseProjectUrl(projectUrl); if (!parsedProjectUrl) { @@ -21,7 +27,8 @@ export async function fetchRemoteScorecardAndPlugins( orgSlug, projectSlug, auth, - isApiKey + isApiKey, + verbose ); const scorecard = project?.config.scorecard; @@ -29,15 +36,37 @@ export async function fetchRemoteScorecardAndPlugins( throw new Error('No scorecard configuration found.'); } + if (verbose) { + logger.info(`Successfully fetched scorecard configuration.\n`); + logger.info(`Scorecard levels found: ${scorecard.levels?.length || 0}\n`); + } + const plugins = project.config.pluginsUrl - ? await fetchPlugins(project.config.pluginsUrl) + ? await fetchPlugins(project.config.pluginsUrl, verbose) : undefined; + if (verbose) { + if (plugins) { + logger.info(`Successfully fetched plugins from ${project.config.pluginsUrl}\n`); + } else if (project.config.pluginsUrl) { + logger.info(`No plugins were loaded from ${project.config.pluginsUrl}\n`); + } else { + logger.info(`No custom plugins configured for this scorecard.\n`); + } + } + return { scorecard: scorecard!, plugins, }; } catch (error) { + if (verbose) { + logger.error(`āŒ Failed to fetch remote scorecard configuration.\n`); + logger.error(`Error details: ${error.message}\n`); + if (error.stack) { + logger.error(`Stack trace:\n${error.stack}\n`); + } + } exitWithError(error.message); } } @@ -66,14 +95,23 @@ async function fetchProjectConfigBySlugs( orgSlug: string, projectSlug: string, auth: string, - isApiKey: boolean + isApiKey: boolean, + verbose = false ): Promise { const authHeaders = createAuthHeaders(auth, isApiKey); const projectUrl = new URL(`${residency}/api/orgs/${orgSlug}/projects/${projectSlug}`); const projectResponse = await fetch(projectUrl, { headers: authHeaders }); + if (verbose) { + logger.info(`Project fetch response status: ${projectResponse.status}\n`); + } + if (projectResponse.status === 401 || projectResponse.status === 403) { + if (verbose) { + logger.error(`Authentication failed with status ${projectResponse.status}.\n`); + logger.error(`Check that your credentials are valid and have the necessary permissions.\n`); + } throw new Error( `Unauthorized access to project: ${projectSlug}. Please check your credentials.` ); @@ -83,17 +121,39 @@ async function fetchProjectConfigBySlugs( throw new Error(`Failed to fetch project: ${projectSlug}. Status: ${projectResponse.status}`); } + if (verbose) { + logger.info(`Successfully received project configuration.\n`); + } + return projectResponse.json(); } -async function fetchPlugins(pluginsUrl: string): Promise { - const pluginsResponse = await fetch(pluginsUrl); +async function fetchPlugins(pluginsUrl: string, verbose = false): Promise { + if (verbose) { + logger.info(`Fetching plugins from: ${pluginsUrl}\n`); + } + + try { + const pluginsResponse = await fetch(pluginsUrl); + + if (verbose) { + logger.info(`Plugins fetch response status: ${pluginsResponse.status}\n`); + } - if (pluginsResponse.status !== 200) { + if (pluginsResponse.status !== 200) { + if (verbose) { + logger.error(`Failed to fetch plugins\n`); + } + return; + } + + return pluginsResponse.text(); + } catch (error) { + if (verbose) { + logger.error(`Error fetching plugins: ${error.message}\n`); + } return; } - - return pluginsResponse.text(); } function createAuthHeaders(auth: string, isApiKey: boolean): Record { diff --git a/packages/cli/src/commands/scorecard-classic/types.ts b/packages/cli/src/commands/scorecard-classic/types.ts index a8095ba2d9..285ca418e0 100644 --- a/packages/cli/src/commands/scorecard-classic/types.ts +++ b/packages/cli/src/commands/scorecard-classic/types.ts @@ -6,6 +6,7 @@ export type ScorecardClassicArgv = { config: string; 'project-url'?: string; format: OutputFormat; + verbose?: boolean; } & VerifyConfigOptions; export type ScorecardProblem = NormalizedProblem & { scorecardLevel?: string }; diff --git a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts index 29302b9b35..744e83e8e7 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/plugin-evaluator.ts @@ -1,4 +1,4 @@ -import { logger } from '@redocly/openapi-core'; +import { logger, pluralize } from '@redocly/openapi-core'; import type { Plugin } from '@redocly/openapi-core'; @@ -8,22 +8,60 @@ type PluginsModule = { default: PluginFunction[]; }; -export async function evaluatePluginsFromCode(pluginsCode?: string): Promise { +export async function evaluatePluginsFromCode( + pluginsCode?: string, + verbose = false +): Promise { if (!pluginsCode) { + if (verbose) { + logger.info(`No plugins code provided to evaluate.\n`); + } return []; } + if (verbose) { + logger.info(`Starting plugin evaluation...\n`); + } + try { const dirname = import.meta.url; const pluginsCodeWithDirname = pluginsCode.replaceAll('__redocly_dirname', `"${dirname}"`); + if (verbose) { + logger.info(`Encoding plugins code to base64 data URI...\n`); + } + const base64 = btoa(pluginsCodeWithDirname); const dataUri = `data:text/javascript;base64,${base64}`; + + if (verbose) { + logger.info(`Importing plugins module dynamically...\n`); + } + const module: PluginsModule = await import(dataUri); const evaluatedPlugins = module.default.map((pluginFunction) => pluginFunction()); + if (verbose) { + logger.info( + `Successfully evaluated ${evaluatedPlugins.length} ${pluralize( + 'plugin', + evaluatedPlugins.length + )}.\n` + ); + evaluatedPlugins.forEach((plugin, index) => { + logger.info(` Plugin ${index + 1}: ${plugin.id || 'unnamed'}\n`); + }); + } + return evaluatedPlugins; - } catch { + } catch (error) { + if (verbose) { + logger.error(`āŒ Failed to evaluate plugins.\n`); + logger.error(`Error details: ${error.message}\n`); + if (error.stack) { + logger.error(`Stack trace:\n${error.stack}\n`); + } + } logger.warn(`Something went wrong during plugins evaluation.`); return []; } diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts index 34ee63beb8..89577b9603 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -1,4 +1,4 @@ -import { createConfig, lintDocument } from '@redocly/openapi-core'; +import { logger, createConfig, lintDocument, pluralize } from '@redocly/openapi-core'; import { evaluatePluginsFromCode } from './plugin-evaluator.js'; import type { ScorecardConfig } from '@redocly/config'; @@ -10,26 +10,49 @@ export async function validateScorecard( externalRefResolver: BaseResolver, scorecardConfig: ScorecardConfig, configPath?: string, - pluginsCodeOrPlugins?: string | Plugin[] + pluginsCodeOrPlugins?: string | Plugin[], + verbose = false ): Promise { const problems: ScorecardProblem[] = []; for (const level of scorecardConfig?.levels || []) { + if (verbose) { + logger.info(`\nValidating level: "${level.name}"\n`); + } + const plugins = typeof pluginsCodeOrPlugins === 'string' - ? await evaluatePluginsFromCode(pluginsCodeOrPlugins) + ? await evaluatePluginsFromCode(pluginsCodeOrPlugins, verbose) : pluginsCodeOrPlugins; + if (verbose && plugins && plugins.length > 0) { + logger.info( + `Using ${plugins.length} ${pluralize('plugin', plugins.length)} for this level.\n` + ); + } + const config = await createConfig({ ...level, plugins } as RawUniversalConfig, { configPath, }); + if (verbose) { + logger.info(`Linting document against level rules...\n`); + } + const levelProblems = await lintDocument({ document, externalRefResolver, config, }); + if (verbose) { + logger.info( + `Found ${levelProblems.length} ${pluralize('problem', levelProblems.length)} for level "${ + level.name + }".\n` + ); + } + problems.push( ...levelProblems .filter(({ ignored }) => !ignored) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 9c8af54e3a..85b176c562 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -816,6 +816,11 @@ yargs(hideBin(process.argv)) choices: ['stylish', 'json'], default: 'stylish', }, + verbose: { + alias: 'v', + describe: 'Apply verbose mode.', + type: 'boolean', + }, }); }, async (argv) => { From b7cb6a1218a4798c60b17b0fcb69ec26397615ba Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Wed, 10 Dec 2025 20:00:31 +0200 Subject: [PATCH 18/41] feat: change tests due to changes and add tests for verbose flag --- .../__tests__/fetch-scorecard.test.ts | 42 +++++++++++++++++-- .../__tests__/json-formatter.test.ts | 2 +- .../__tests__/plugin-evaluator.test.ts | 19 +++++++++ .../__tests__/stylish-formatter.test.ts | 3 +- .../__tests__/validate-scorecard.test.ts | 26 +++++++++++- 5 files changed, 85 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index 3715b43b62..7571b84126 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -7,6 +7,8 @@ describe('fetchRemoteScorecardAndPlugins', () => { const testToken = 'test-token'; beforeEach(() => { + vi.unstubAllEnvs(); + delete process.env.REDOCLY_AUTHORIZATION; global.fetch = mockFetch; mockFetch.mockClear(); vi.spyOn(errorUtils, 'exitWithError').mockImplementation(() => { @@ -193,8 +195,8 @@ describe('fetchRemoteScorecardAndPlugins', () => { }); it('should use correct auth headers with API key', async () => { - const originalApiKey = process.env.REDOCLY_AUTHORIZATION; - process.env.REDOCLY_AUTHORIZATION = 'test-api-key'; + const apiKey = 'test-api-key'; + process.env.REDOCLY_AUTHORIZATION = apiKey; mockFetch.mockResolvedValueOnce({ status: 200, @@ -204,13 +206,45 @@ describe('fetchRemoteScorecardAndPlugins', () => { }), }); - await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await fetchRemoteScorecardAndPlugins(validProjectUrl, apiKey, true); expect(mockFetch).toHaveBeenCalledWith( expect.any(URL), expect.objectContaining({ - headers: { Authorization: 'Bearer test-api-key' }, + headers: { Authorization: `Bearer ${apiKey}` }, }) ); }); + + it('should handle verbose flag and fetch plugins successfully', async () => { + const mockScorecard = { + levels: [{ name: 'Gold', rules: {} }], + }; + const mockPluginsCode = 'export default [() => ({ id: "test-plugin" })]'; + + mockFetch + .mockResolvedValueOnce({ + status: 200, + json: async () => ({ + id: 'project-123', + slug: 'test-project', + config: { + scorecard: mockScorecard, + pluginsUrl: 'https://example.com/plugins.js', + }, + }), + }) + .mockResolvedValueOnce({ + status: 200, + text: async () => mockPluginsCode, + }); + + const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken, false, true); + + expect(result).toEqual({ + scorecard: mockScorecard, + plugins: mockPluginsCode, + }); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts index bb68f5377e..f56d41e675 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -94,7 +94,7 @@ describe('printScorecardResultsAsJson', () => { const output = JSON.parse(outputCall); expect(output.Gold.problems[0].ruleUrl).toBe( - 'https://redocly.com/docs/cli/rules/oas/operation-summary.md' + 'https://redocly.com/docs/cli/rules/oas/operation-summary' ); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts index 054e098207..37eb81de3b 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/plugin-evaluator.test.ts @@ -44,4 +44,23 @@ describe('evaluatePluginsFromCode', () => { const result = await evaluatePluginsFromCode(pluginCodeWithDirname); expect(result).toHaveLength(1); }); + + it('should handle verbose flag', async () => { + const validPluginCode = ` + export default [ + () => ({ + id: 'verbose-test-plugin', + rules: { + oas3: { + 'test-rule': () => ({}) + } + } + }) + ]; + `; + + const result = await evaluatePluginsFromCode(validPluginCode, true); + expect(result).toHaveLength(1); + expect(result[0]).toHaveProperty('id', 'verbose-test-plugin'); + }); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts index e64c23c05f..e886c3dc34 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -87,8 +87,9 @@ describe('printScorecardResults', () => { printScorecardResults(problems); + expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); expect(openapiCore.logger.info).toHaveBeenCalledWith( - expect.stringContaining('2 errors, 1 warnings') + expect.stringMatching(/2.*error.*1.*warning/) ); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index 5ce9f73d08..b6042fe561 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -117,7 +117,7 @@ describe('validateScorecard', () => { await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, 'plugin-code'); - expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code'); + expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code', false); expect(openapiCore.createConfig).toHaveBeenCalledWith( expect.objectContaining({ plugins: mockPlugins }), expect.any(Object) @@ -139,4 +139,28 @@ describe('validateScorecard', () => { expect.any(Object) ); }); + + it('should handle verbose flag', async () => { + const scorecardConfig = { + levels: [{ name: 'Gold', rules: {} }], + }; + + const mockPlugins = [{ id: 'test-plugin' }]; + vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins); + + await validateScorecard( + mockDocument, + mockResolver, + scorecardConfig, + undefined, + 'plugin-code', + true + ); + + expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code', true); + expect(openapiCore.createConfig).toHaveBeenCalledWith( + expect.objectContaining({ plugins: mockPlugins }), + expect.any(Object) + ); + }); }); From 3ca9d3b0c72023a92a7a99774a489ec76794a86e Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 10:48:54 +0200 Subject: [PATCH 19/41] feat: add docs for new command --- docs/@v2/commands/scorecard-classic.md | 154 +++++++++++++++++++++++++ docs/@v2/v2.sidebars.yaml | 2 + 2 files changed, 156 insertions(+) create mode 100644 docs/@v2/commands/scorecard-classic.md diff --git a/docs/@v2/commands/scorecard-classic.md b/docs/@v2/commands/scorecard-classic.md new file mode 100644 index 0000000000..6f9a784268 --- /dev/null +++ b/docs/@v2/commands/scorecard-classic.md @@ -0,0 +1,154 @@ +# `scorecard-classic` + +## Introduction + +The `scorecard-classic` command evaluates your API descriptions against quality standards defined in your Redocly project's scorecard configuration. +Use this command to validate API quality and track compliance with organizational governance standards across multiple severity levels. + +{% admonition type="info" name="Note" %} +The `scorecard-classic` command requires a scorecard configuration in your Redocly project. You can configure this in your project settings or by providing a `--project-url` flag. +{% /admonition %} + +## Usage + +```bash +redocly scorecard-classic --project-url= +redocly scorecard-classic --config= +redocly scorecard-classic --format=json +redocly scorecard-classic --verbose +``` + +## Options + +| Option | Type | Description | +| ------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| api | string | Path to the API description filename or alias that you want to evaluate. See [the API section](#specify-api) for more details. | +| --config | string | Specify path to the [configuration file](#use-alternative-configuration-file). | +| --format | string | Format for the output.
**Possible values:** `stylish`, `json`. Default value is `stylish`. | +| --help | boolean | Show help. | +| --project-url | string | URL to the project scorecard configuration. Required if not configured in the Redocly configuration file. Example: `https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic`. | +| --verbose, -v | boolean | Run the command in verbose mode to display additional information during execution. | + +## Examples + +### Specify API + +You can use the `scorecard-classic` command with an OpenAPI description file path or an API alias defined in your Redocly configuration file. + +```bash +redocly scorecard-classic openapi/openapi.yaml --project-url=https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic +``` + +In this example, `scorecard-classic` evaluates the specified API description against the scorecard rules defined in the provided project URL. + +### Use alternative configuration file + +By default, the CLI tool looks for the [Redocly configuration file](../configuration/index.md) in the current working directory. +Use the optional `--config` argument to provide an alternative path to a configuration file. + +```bash +redocly scorecard-classic openapi/openapi.yaml --config=./another/directory/redocly.yaml +``` + +### Configure scorecard in redocly.yaml + +You can configure the scorecard project URL in your Redocly configuration file to avoid passing it as a command-line argument: + +```yaml +scorecard: + fromProjectUrl: https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic + +apis: + core@v1: + root: ./openapi/api-description.json +``` + +With this configuration, you can run the command without the `--project-url` flag: + +```bash +redocly scorecard-classic core@v1 +``` + +### Use JSON output format + +To generate machine-readable output suitable for CI/CD pipelines or further processing, use the JSON format: + +```bash +redocly scorecard-classic openapi/openapi.yaml --format=json +``` + +The JSON output is grouped by scorecard level and includes: + +- Summary of errors and warnings for each level +- Rule ID and documentation link (for built-in rules) +- Severity level (error or warning) +- Location information (file path, line/column range, and JSON pointer) +- Descriptive message about the violation + +### Run in verbose mode + +For troubleshooting or detailed insights into the scorecard evaluation process, enable verbose mode: + +```bash +redocly scorecard-classic openapi/openapi.yaml --verbose +``` + +Verbose mode displays additional information such as: + +- Project URL being used +- Authentication status +- Detailed logging of the evaluation process + +## Authentication + +The `scorecard-classic` command requires authentication to access your project's scorecard configuration. +You can authenticate in one of two ways: + +### Using API key (recommended for CI/CD) + +Set the `REDOCLY_AUTHORIZATION` environment variable with your API key: + +```bash +export REDOCLY_AUTHORIZATION=your-api-key-here +redocly scorecard-classic openapi/openapi.yaml +``` + +### Interactive login + +If no API key is provided, the command prompts you to log in interactively: + +```bash +redocly scorecard-classic openapi/openapi.yaml +``` + +The CLI opens a browser window for you to authenticate with your Redocly account. + +## Understanding scorecard results + +The scorecard evaluation categorizes issues into multiple levels based on your project's configuration. +Each issue is associated with a specific scorecard level, allowing you to prioritize improvements. + +When all checks pass, you'll see a success message: + +``` +āœ… No issues found for openapi/openapi.yaml. Your API meets all scorecard requirements. +``` + +When issues are found, the output shows: + +- The rule that was violated +- The scorecard level of the rule +- The location in the API description where the issue occurs +- A descriptive message explaining the violation + +## Related commands + +- [`lint`](./lint.md) - Standard linting for API descriptions with pass/fail results +- [`bundle`](./bundle.md) - Bundle multi-file API descriptions into a single file +- [`stats`](./stats.md) - Display statistics about your API description structure + +## Resources + +- [API governance documentation](../api-standards.md) +- [Redocly configuration guide](../configuration/index.md) +- [Custom rules and plugins](../custom-plugins/index.md) diff --git a/docs/@v2/v2.sidebars.yaml b/docs/@v2/v2.sidebars.yaml index 9ddd428a84..c330b8611e 100644 --- a/docs/@v2/v2.sidebars.yaml +++ b/docs/@v2/v2.sidebars.yaml @@ -34,6 +34,8 @@ page: commands/push-status.md - label: respect page: commands/respect.md + - label: scorecard-classic + page: commands/scorecard-classic.md - label: split page: commands/split.md - label: stats From 9b09ead92ed5008c89c48eaec6225ad8137301af Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 10:49:22 +0200 Subject: [PATCH 20/41] feat: add functionality to exit with code 1 if problems exist in openapi --- packages/cli/src/commands/scorecard-classic/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index ed1e4af66c..491458f2ef 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -1,6 +1,6 @@ import { formatPath, getExecutionTime, getFallbackApisOrExit } from '../../utils/miscellaneous.js'; import { BaseResolver, logger } from '@redocly/openapi-core'; -import { exitWithError } from '../../utils/error.js'; +import { AbortFlowError, exitWithError } from '../../utils/error.js'; import { handleLoginAndFetchToken } from './auth/login-handler.js'; import { printScorecardResults } from './formatters/stylish-formatter.js'; import { printScorecardResultsAsJson } from './formatters/json-formatter.js'; @@ -77,4 +77,6 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs Date: Thu, 11 Dec 2025 10:52:41 +0200 Subject: [PATCH 21/41] docs: add link how to configure scorecard --- docs/@v2/commands/scorecard-classic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/@v2/commands/scorecard-classic.md b/docs/@v2/commands/scorecard-classic.md index 6f9a784268..d0ace181bb 100644 --- a/docs/@v2/commands/scorecard-classic.md +++ b/docs/@v2/commands/scorecard-classic.md @@ -6,7 +6,7 @@ The `scorecard-classic` command evaluates your API descriptions against quality Use this command to validate API quality and track compliance with organizational governance standards across multiple severity levels. {% admonition type="info" name="Note" %} -The `scorecard-classic` command requires a scorecard configuration in your Redocly project. You can configure this in your project settings or by providing a `--project-url` flag. +The `scorecard-classic` command requires a scorecard configuration in your Redocly project. You can configure this in your project settings or by providing a `--project-url` flag. Learn more about [configuring scorecards](https://redocly.com/docs/realm/config/scorecard). {% /admonition %} ## Usage From 464d590cc31bdf35458585516681a2f473d58d65 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 10:57:40 +0200 Subject: [PATCH 22/41] docs: fix errors to pass tests --- docs/@v2/commands/scorecard-classic.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/@v2/commands/scorecard-classic.md b/docs/@v2/commands/scorecard-classic.md index d0ace181bb..b0c7592a6a 100644 --- a/docs/@v2/commands/scorecard-classic.md +++ b/docs/@v2/commands/scorecard-classic.md @@ -123,14 +123,14 @@ redocly scorecard-classic openapi/openapi.yaml The CLI opens a browser window for you to authenticate with your Redocly account. -## Understanding scorecard results +## Scorecard results The scorecard evaluation categorizes issues into multiple levels based on your project's configuration. Each issue is associated with a specific scorecard level, allowing you to prioritize improvements. -When all checks pass, you'll see a success message: +When all checks pass, the command displays a success message: -``` +```text āœ… No issues found for openapi/openapi.yaml. Your API meets all scorecard requirements. ``` From 2584eca67cf2c6b9bc7850f66580b408c6b1ca30 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:04:15 +0200 Subject: [PATCH 23/41] feat: add changeset --- .changeset/sixty-trams-show.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sixty-trams-show.md diff --git a/.changeset/sixty-trams-show.md b/.changeset/sixty-trams-show.md new file mode 100644 index 0000000000..87e4e8c3b1 --- /dev/null +++ b/.changeset/sixty-trams-show.md @@ -0,0 +1,5 @@ +--- +"@redocly/cli": patch +--- + +Added `scorecard-classic` command to evaluate API descriptions against project scorecard configurations. From d94fd0f515806d5a8b3f7a9e1eae1b9ec8a35a3d Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:13:08 +0200 Subject: [PATCH 24/41] chore: add log under verbose flag --- .../cli/src/commands/scorecard-classic/auth/login-handler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index 3e8341e599..eb3b429b42 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -15,7 +15,9 @@ export async function handleLoginAndFetchToken( let accessToken = await oauthClient.getAccessToken(reuniteUrl); if (accessToken) { - logger.info(`āœ… Found existing access token.\n`); + if (verbose) { + logger.info(`Using existing access token.\n`); + } return accessToken; } From dfa5a0166fbcd2e96e0b230c2058e0f42e64ed37 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:25:55 +0200 Subject: [PATCH 25/41] feat: add tests for login handler --- .../__tests__/login-handler.test.ts | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts new file mode 100644 index 0000000000..c239122c1a --- /dev/null +++ b/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { handleLoginAndFetchToken } from '../auth/login-handler.js'; +import * as errorUtils from '../../../utils/error.js'; +import { RedoclyOAuthClient } from '../../../auth/oauth-client.js'; +import { logger } from '@redocly/openapi-core'; + +vi.mock('../../../auth/oauth-client.js'); +vi.mock('../../../reunite/api/index.js', () => ({ + getReuniteUrl: vi.fn(() => 'https://www.test.com'), +})); + +describe('handleLoginAndFetchToken', () => { + const mockConfig = { + resolvedConfig: { + residency: 'us', + }, + } as any; + + let mockOAuthClient: any; + + beforeEach(() => { + mockOAuthClient = { + getAccessToken: vi.fn(), + login: vi.fn(), + }; + vi.mocked(RedoclyOAuthClient).mockImplementation(() => mockOAuthClient); + vi.spyOn(logger, 'info').mockImplementation(() => {}); + vi.spyOn(logger, 'warn').mockImplementation(() => {}); + vi.spyOn(logger, 'error').mockImplementation(() => {}); + vi.spyOn(errorUtils, 'exitWithError').mockImplementation(() => { + throw new Error('exitWithError called'); + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should return existing access token when available', async () => { + const testToken = 'existing-token'; + mockOAuthClient.getAccessToken.mockResolvedValue(testToken); + + const result = await handleLoginAndFetchToken(mockConfig, false); + + expect(result).toBe(testToken); + expect(mockOAuthClient.getAccessToken).toHaveBeenCalledTimes(1); + expect(mockOAuthClient.login).not.toHaveBeenCalled(); + }); + + it('should log info when verbose is enabled and token exists', async () => { + const testToken = 'existing-token'; + mockOAuthClient.getAccessToken.mockResolvedValue(testToken); + + await handleLoginAndFetchToken(mockConfig, true); + + expect(logger.info).toHaveBeenCalledWith('Using existing access token.\n'); + }); + + it('should attempt login when no access token is found', async () => { + const newToken = 'new-token'; + mockOAuthClient.getAccessToken.mockResolvedValueOnce(null).mockResolvedValueOnce(newToken); + mockOAuthClient.login.mockResolvedValue(undefined); + + const result = await handleLoginAndFetchToken(mockConfig, false); + + expect(result).toBe(newToken); + expect(mockOAuthClient.login).toHaveBeenCalled(); + expect(mockOAuthClient.getAccessToken).toHaveBeenCalledTimes(2); + }); + + it('should log warning when verbose is enabled and no token found', async () => { + const newToken = 'new-token'; + mockOAuthClient.getAccessToken.mockResolvedValueOnce(null).mockResolvedValueOnce(newToken); + mockOAuthClient.login.mockResolvedValue(undefined); + + await handleLoginAndFetchToken(mockConfig, true); + + expect(logger.warn).toHaveBeenCalledWith('No access token found. Attempting login...\n'); + }); + + it('should handle login failure and exit with error', async () => { + const loginError = new Error('Login failed'); + mockOAuthClient.getAccessToken.mockResolvedValue(null); + mockOAuthClient.login.mockRejectedValue(loginError); + + await expect(handleLoginAndFetchToken(mockConfig, false)).rejects.toThrow( + 'exitWithError called' + ); + + expect(errorUtils.exitWithError).toHaveBeenCalledWith( + expect.stringContaining('Login failed. Please try again or check your connection') + ); + }); + + it('should log error details when verbose is enabled and login fails', async () => { + const loginError = new Error('Network error'); + mockOAuthClient.getAccessToken.mockResolvedValue(null); + mockOAuthClient.login.mockRejectedValue(loginError); + + await expect(handleLoginAndFetchToken(mockConfig, true)).rejects.toThrow(); + + expect(logger.error).toHaveBeenCalledWith('āŒ Login failed.\n'); + expect(logger.error).toHaveBeenCalledWith('Error details: Network error\n'); + }); +}); From 0f9726fa6d101858a910c62797f60906da43db88 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 14:05:50 +0200 Subject: [PATCH 26/41] feat: add fallback to old scorecard config --- packages/cli/src/commands/scorecard-classic/index.ts | 5 ++++- .../src/commands/scorecard-classic/remote/fetch-scorecard.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 491458f2ef..7ad5b477a6 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -18,7 +18,10 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs Date: Thu, 11 Dec 2025 14:24:26 +0200 Subject: [PATCH 27/41] fix: json formatter to match structure --- .../__tests__/json-formatter.test.ts | 56 +++++++++++++------ .../formatters/json-formatter.ts | 27 ++++++--- .../src/commands/scorecard-classic/index.ts | 8 ++- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts index f56d41e675..7823109f50 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -21,7 +21,16 @@ describe('printScorecardResultsAsJson', () => { it('should print empty results when no problems', () => { printScorecardResultsAsJson([]); - expect(openapiCore.logger.output).toHaveBeenCalledWith('{}'); + expect(openapiCore.logger.output).toHaveBeenCalledWith( + JSON.stringify( + { + version: '1.0', + levels: [], + }, + null, + 2 + ) + ); }); it('should group problems by scorecard level', () => { @@ -69,11 +78,16 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(Object.keys(output)).toEqual(['Gold', 'Silver']); - expect(output.Gold.summary).toEqual({ errors: 1, warnings: 1 }); - expect(output.Gold.problems).toHaveLength(2); - expect(output.Silver.summary).toEqual({ errors: 1, warnings: 0 }); - expect(output.Silver.problems).toHaveLength(1); + expect(output.version).toBe('1.0'); + expect(output.levels).toHaveLength(2); + + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + const silverLevel = output.levels.find((l: any) => l.name === 'Silver'); + + expect(goldLevel.total).toEqual({ errors: 1, warnings: 1 }); + expect(goldLevel.problems).toHaveLength(2); + expect(silverLevel.total).toEqual({ errors: 1, warnings: 0 }); + expect(silverLevel.problems).toHaveLength(1); }); it('should include rule URLs for non-namespaced rules', () => { @@ -93,7 +107,8 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(output.Gold.problems[0].ruleUrl).toBe( + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + expect(goldLevel.problems[0].ruleUrl).toBe( 'https://redocly.com/docs/cli/rules/oas/operation-summary' ); }); @@ -115,7 +130,8 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(output.Gold.problems[0].ruleUrl).toBeUndefined(); + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + expect(goldLevel.problems[0].ruleUrl).toBeUndefined(); }); it('should format location with file path and range', () => { @@ -141,10 +157,11 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(output.Gold.problems[0].location).toHaveLength(1); - expect(output.Gold.problems[0].location[0].file).toBe('/test/file.yaml'); - expect(output.Gold.problems[0].location[0].pointer).toBe('#/paths/~1test/get'); - expect(output.Gold.problems[0].location[0].range).toContain('Line'); + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + expect(goldLevel.problems[0].location).toHaveLength(1); + expect(goldLevel.problems[0].location[0].file).toBe('/test/file.yaml'); + expect(goldLevel.problems[0].location[0].pointer).toBe('#/paths/~1test/get'); + expect(goldLevel.problems[0].location[0].range).toContain('Line'); }); it('should handle problems with Unknown level', () => { @@ -164,8 +181,9 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(Object.keys(output)).toEqual(['Unknown']); - expect(output.Unknown.problems).toHaveLength(1); + const unknownLevel = output.levels.find((l: any) => l.name === 'Unknown'); + expect(unknownLevel).toBeDefined(); + expect(unknownLevel.problems).toHaveLength(1); }); it('should strip ANSI codes from messages', () => { @@ -185,8 +203,9 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(output.Gold.problems[0].message).toBe('Error message with color'); - expect(output.Gold.problems[0].message).not.toContain('\u001b'); + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + expect(goldLevel.problems[0].message).toBe('Error message with color'); + expect(goldLevel.problems[0].message).not.toContain('\u001b'); }); it('should count errors and warnings correctly', () => { @@ -238,7 +257,8 @@ describe('printScorecardResultsAsJson', () => { const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); - expect(output.Gold.summary.errors).toBe(2); - expect(output.Gold.summary.warnings).toBe(3); + const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); + expect(goldLevel.total.errors).toBe(2); + expect(goldLevel.total.warnings).toBe(3); }); }); diff --git a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts index 55f4360406..c63068f0c8 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts @@ -3,7 +3,8 @@ import { logger, getLineColLocation } from '@redocly/openapi-core'; import type { ScorecardProblem } from '../types.js'; type ScorecardLevel = { - summary: { + name: string; + total: { errors: number; warnings: number; }; @@ -20,7 +21,10 @@ type ScorecardLevel = { }>; }; -export type ScorecardJsonOutput = Record; +export type ScorecardJsonOutput = { + version: string; + levels: ScorecardLevel[]; +}; function formatRange( start: { line: number; col: number }, @@ -46,7 +50,10 @@ function stripAnsiCodes(text: string): string { return text.replace(/\u001b\[\d+m/g, ''); } -export function printScorecardResultsAsJson(problems: ScorecardProblem[]): void { +export function printScorecardResultsAsJson( + problems: ScorecardProblem[], + version: string = '1.0' +): void { const groupedByLevel: Record = {}; for (const problem of problems) { @@ -57,7 +64,7 @@ export function printScorecardResultsAsJson(problems: ScorecardProblem[]): void groupedByLevel[level].push(problem); } - const output: ScorecardJsonOutput = {}; + const levels: ScorecardLevel[] = []; for (const [levelName, levelProblems] of Object.entries(groupedByLevel)) { let errors = 0; @@ -84,15 +91,21 @@ export function printScorecardResultsAsJson(problems: ScorecardProblem[]): void }; }); - output[levelName] = { - summary: { + levels.push({ + name: levelName, + total: { errors, warnings, }, problems: formattedProblems, - }; + }); } + const output: ScorecardJsonOutput = { + version, + levels, + }; + logger.output(JSON.stringify(output, null, 2)); logger.info('\n'); } diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 7ad5b477a6..e541c2076e 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -12,7 +12,11 @@ import type { ScorecardClassicArgv } from './types.js'; import type { CommandArgs } from '../../wrapper.js'; import type { Document } from '@redocly/openapi-core'; -export async function handleScorecardClassic({ argv, config }: CommandArgs) { +export async function handleScorecardClassic({ + argv, + config, + version, +}: CommandArgs) { const startedAt = performance.now(); const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); const externalRefResolver = new BaseResolver(config.resolve); @@ -69,7 +73,7 @@ export async function handleScorecardClassic({ argv, config }: CommandArgs Date: Thu, 11 Dec 2025 14:29:17 +0200 Subject: [PATCH 28/41] fix: logger info in stylish formatter --- .../commands/scorecard-classic/formatters/stylish-formatter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts index e6f072e0fb..b4c5429ca9 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts @@ -59,7 +59,7 @@ export function printScorecardResults(problems: ScorecardProblem[]): void { return acc; }, {} as Record); - logger.info( + logger.output( bold(cyan(`\n šŸ“‹ ${level}`)) + gray( ` (${severityCounts.error || 0} ${pluralize('error', severityCounts.error || 0)}, ${ From 532c7ed0838bb838f43cb6527c41684a84c81cc8 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Thu, 11 Dec 2025 14:40:40 +0200 Subject: [PATCH 29/41] chore: add more obvious verbose message --- .../cli/src/commands/scorecard-classic/auth/login-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts index eb3b429b42..c816c26e0f 100644 --- a/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts +++ b/packages/cli/src/commands/scorecard-classic/auth/login-handler.ts @@ -22,7 +22,7 @@ export async function handleLoginAndFetchToken( } if (verbose) { - logger.warn(`No access token found. Attempting login...\n`); + logger.warn(`No valid access token found or refresh token expired. Attempting login...\n`); } try { From b70b8f65448d28338a26704d01ef4aa776c77020 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Fri, 12 Dec 2025 10:30:03 +0200 Subject: [PATCH 30/41] fix: tests after changes --- .../__tests__/login-handler.test.ts | 4 +++- .../__tests__/stylish-formatter.test.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts index c239122c1a..4326c6e0be 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/login-handler.test.ts @@ -75,7 +75,9 @@ describe('handleLoginAndFetchToken', () => { await handleLoginAndFetchToken(mockConfig, true); - expect(logger.warn).toHaveBeenCalledWith('No access token found. Attempting login...\n'); + expect(logger.warn).toHaveBeenCalledWith( + 'No valid access token found or refresh token expired. Attempting login...\n' + ); }); it('should handle login failure and exit with error', async () => { diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts index e886c3dc34..f959ed1605 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -33,11 +33,10 @@ describe('printScorecardResults', () => { printScorecardResults(problems); - expect(openapiCore.logger.output).toHaveBeenCalled(); expect(openapiCore.logger.info).toHaveBeenCalledWith( expect.stringMatching(/Found.*1.*error.*0.*warning.*1.*level/) ); - expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); + expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); }); it('should handle problems with Unknown level', () => { @@ -54,7 +53,7 @@ describe('printScorecardResults', () => { printScorecardResults(problems); - expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('Unknown')); + expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('Unknown')); }); it('should show correct severity counts per level', () => { @@ -87,8 +86,8 @@ describe('printScorecardResults', () => { printScorecardResults(problems); - expect(openapiCore.logger.info).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); - expect(openapiCore.logger.info).toHaveBeenCalledWith( + expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); + expect(openapiCore.logger.output).toHaveBeenCalledWith( expect.stringMatching(/2.*error.*1.*warning/) ); }); @@ -127,6 +126,7 @@ describe('printScorecardResults', () => { printScorecardResults(problems); - expect(openapiCore.logger.output).toHaveBeenCalledTimes(2); + // Should have 3 calls: 1 for level header + 2 for problems + expect(openapiCore.logger.output).toHaveBeenCalledTimes(3); }); }); From 0ffc51809d9abcff2daa0aac3ad8988c7fd162e5 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Fri, 12 Dec 2025 10:42:30 +0200 Subject: [PATCH 31/41] feat: add functionality to check if it is a CI env and throw an error --- .../src/commands/scorecard-classic/index.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index e541c2076e..e13d050e80 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -38,6 +38,12 @@ export async function handleScorecardClassic({ ); } + if (isNonInteractiveEnvironment() && !apiKey) { + exitWithError( + 'Please provide an API key using the REDOCLY_AUTHORIZATION environment variable.\n' + ); + } + const auth = apiKey || (await handleLoginAndFetchToken(config, argv.verbose)); if (!auth) { @@ -87,3 +93,15 @@ export async function handleScorecardClassic({ throw new AbortFlowError('Scorecard validation failed.'); } + +function isNonInteractiveEnvironment(): boolean { + if (process.env.CI) { + return true; + } + + if (!process.stdin.isTTY || !process.stdout.isTTY) { + return true; + } + + return false; +} From 3525480a7c62b1681352c26d90556c88c5d3b8cf Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Fri, 12 Dec 2025 11:30:43 +0200 Subject: [PATCH 32/41] fix: check for terminal interactivity --- packages/cli/src/commands/scorecard-classic/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index e13d050e80..a64550fa77 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -95,11 +95,7 @@ export async function handleScorecardClassic({ } function isNonInteractiveEnvironment(): boolean { - if (process.env.CI) { - return true; - } - - if (!process.stdin.isTTY || !process.stdout.isTTY) { + if (process.env.CI || !process.stdin.isTTY) { return true; } From 7eadbd48523fbf1f9d0a823f97e374617cbcbeca Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:46:58 +0200 Subject: [PATCH 33/41] feat: add functionality to check level of API --- docs/@v2/commands/scorecard-classic.md | 44 ++++++++-- .../__tests__/json-formatter.test.ts | 18 +++-- .../__tests__/stylish-formatter.test.ts | 15 ++-- .../__tests__/validate-scorecard.test.ts | 17 ++-- .../formatters/json-formatter.ts | 4 + .../formatters/stylish-formatter.ts | 9 ++- .../src/commands/scorecard-classic/index.ts | 28 ++++++- .../src/commands/scorecard-classic/types.ts | 1 + .../validation/validate-scorecard.ts | 81 ++++++++++++++++--- packages/cli/src/index.ts | 4 + 10 files changed, 175 insertions(+), 46 deletions(-) diff --git a/docs/@v2/commands/scorecard-classic.md b/docs/@v2/commands/scorecard-classic.md index b0c7592a6a..13c5270b9c 100644 --- a/docs/@v2/commands/scorecard-classic.md +++ b/docs/@v2/commands/scorecard-classic.md @@ -15,19 +15,21 @@ The `scorecard-classic` command requires a scorecard configuration in your Redoc redocly scorecard-classic --project-url= redocly scorecard-classic --config= redocly scorecard-classic --format=json +redocly scorecard-classic --target-level= redocly scorecard-classic --verbose ``` ## Options -| Option | Type | Description | -| ------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| api | string | Path to the API description filename or alias that you want to evaluate. See [the API section](#specify-api) for more details. | -| --config | string | Specify path to the [configuration file](#use-alternative-configuration-file). | -| --format | string | Format for the output.
**Possible values:** `stylish`, `json`. Default value is `stylish`. | -| --help | boolean | Show help. | -| --project-url | string | URL to the project scorecard configuration. Required if not configured in the Redocly configuration file. Example: `https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic`. | -| --verbose, -v | boolean | Run the command in verbose mode to display additional information during execution. | +| Option | Type | Description | +| -------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| api | string | Path to the API description filename or alias that you want to evaluate. See [the API section](#specify-api) for more details. | +| --config | string | Specify path to the [configuration file](#use-alternative-configuration-file). | +| --format | string | Format for the output.
**Possible values:** `stylish`, `json`. Default value is `stylish`. | +| --help | boolean | Show help. | +| --project-url | string | URL to the project scorecard configuration. Required if not configured in the Redocly configuration file. Example: `https://app.cloud.redocly.com/org/my-org/projects/my-project/scorecard-classic`. | +| --target-level | string | Target scorecard level to achieve. The command validates that the API meets this level and all preceding levels without errors. Exits with an error if the target level is not achieved. | +| --verbose, -v | boolean | Run the command in verbose mode to display additional information during execution. | ## Examples @@ -79,12 +81,30 @@ redocly scorecard-classic openapi/openapi.yaml --format=json The JSON output is grouped by scorecard level and includes: +- Version information +- Achieved scorecard level - Summary of errors and warnings for each level - Rule ID and documentation link (for built-in rules) - Severity level (error or warning) - Location information (file path, line/column range, and JSON pointer) - Descriptive message about the violation +### Validate against a target level + +Use the `--target-level` option to ensure your API meets a specific quality level. The command validates that your API satisfies the target level and all preceding levels without errors: + +```bash +redocly scorecard-classic openapi/openapi.yaml --target-level=Gold +``` + +If the API doesn't meet the target level, the command: + +- Displays which level was actually achieved +- Shows all validation issues preventing the target level from being met +- Exits with a non-zero exit code (useful for CI/CD pipelines) + +This is particularly useful in CI/CD pipelines to enforce minimum quality standards before deployment. + ### Run in verbose mode For troubleshooting or detailed insights into the scorecard evaluation process, enable verbose mode: @@ -128,19 +148,27 @@ The CLI opens a browser window for you to authenticate with your Redocly account The scorecard evaluation categorizes issues into multiple levels based on your project's configuration. Each issue is associated with a specific scorecard level, allowing you to prioritize improvements. +The command displays the achieved scorecard level, which is the highest level your API meets without errors. +The achieved level is shown in both stylish and JSON output formats. + When all checks pass, the command displays a success message: ```text + ā˜‘ļø Achieved Level: Gold + āœ… No issues found for openapi/openapi.yaml. Your API meets all scorecard requirements. ``` When issues are found, the output shows: +- The achieved scorecard level - The rule that was violated - The scorecard level of the rule - The location in the API description where the issue occurs - A descriptive message explaining the violation +If a `--target-level` is specified and not achieved, the command displays an error message and exits with a non-zero code. + ## Related commands - [`lint`](./lint.md) - Standard linting for API descriptions with pass/fail results diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts index 7823109f50..07e49a9786 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/json-formatter.test.ts @@ -19,12 +19,13 @@ describe('printScorecardResultsAsJson', () => { }); it('should print empty results when no problems', () => { - printScorecardResultsAsJson([]); + printScorecardResultsAsJson([], 'Gold', true); expect(openapiCore.logger.output).toHaveBeenCalledWith( JSON.stringify( { version: '1.0', + level: 'Gold', levels: [], }, null, @@ -73,12 +74,13 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Silver', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); expect(output.version).toBe('1.0'); + expect(output.level).toBe('Silver'); expect(output.levels).toHaveLength(2); const goldLevel = output.levels.find((l: any) => l.name === 'Gold'); @@ -102,7 +104,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Gold', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); @@ -125,7 +127,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Gold', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); @@ -152,7 +154,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Gold', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); @@ -176,7 +178,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Unknown', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); @@ -198,7 +200,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Gold', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); @@ -252,7 +254,7 @@ describe('printScorecardResultsAsJson', () => { }, ]; - printScorecardResultsAsJson(problems); + printScorecardResultsAsJson(problems, 'Gold', true); const outputCall = (openapiCore.logger.output as any).mock.calls[0][0]; const output = JSON.parse(outputCall); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts index f959ed1605..1ba3d59002 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/stylish-formatter.test.ts @@ -31,11 +31,14 @@ describe('printScorecardResults', () => { }, ]; - printScorecardResults(problems); + printScorecardResults(problems, 'Gold', true); expect(openapiCore.logger.info).toHaveBeenCalledWith( expect.stringMatching(/Found.*1.*error.*0.*warning.*1.*level/) ); + expect(openapiCore.logger.output).toHaveBeenCalledWith( + expect.stringContaining('Achieved Level: ') + ); expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); }); @@ -51,7 +54,7 @@ describe('printScorecardResults', () => { }, ]; - printScorecardResults(problems); + printScorecardResults(problems, 'Unknown', true); expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('Unknown')); }); @@ -84,7 +87,7 @@ describe('printScorecardResults', () => { }, ]; - printScorecardResults(problems); + printScorecardResults(problems, 'Gold', true); expect(openapiCore.logger.output).toHaveBeenCalledWith(expect.stringContaining('šŸ“‹ Gold')); expect(openapiCore.logger.output).toHaveBeenCalledWith( @@ -124,9 +127,9 @@ describe('printScorecardResults', () => { }, ]; - printScorecardResults(problems); + printScorecardResults(problems, 'Gold', true); - // Should have 3 calls: 1 for level header + 2 for problems - expect(openapiCore.logger.output).toHaveBeenCalledTimes(3); + // Should have 4 calls: 1 for level header + 2 for problems + 1 for achieved level + expect(openapiCore.logger.output).toHaveBeenCalledTimes(4); }); }); diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index b6042fe561..b8b76c8c33 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -35,7 +35,11 @@ describe('validateScorecard', () => { const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); - expect(result).toEqual([]); + expect(result).toEqual({ + achievedLevel: 'Non Conformant', + problems: [], + targetLevelAchieved: true, + }); expect(openapiCore.lintDocument).not.toHaveBeenCalled(); }); @@ -72,9 +76,9 @@ describe('validateScorecard', () => { const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); - expect(result).toHaveLength(1); - expect(result[0].scorecardLevel).toBe('Gold'); - expect(result[0].message).toBe('Test error'); + expect(result.problems).toHaveLength(1); + expect(result.problems[0].scorecardLevel).toBe('Gold'); + expect(result.problems[0].message).toBe('Test error'); }); it('should filter out ignored problems', async () => { @@ -103,8 +107,8 @@ describe('validateScorecard', () => { const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); - expect(result).toHaveLength(1); - expect(result[0].message).toBe('Error 1'); + expect(result.problems).toHaveLength(1); + expect(result.problems[0].message).toBe('Error 1'); }); it('should evaluate plugins from code when string provided', async () => { @@ -154,6 +158,7 @@ describe('validateScorecard', () => { scorecardConfig, undefined, 'plugin-code', + undefined, true ); diff --git a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts index c63068f0c8..12a81d5561 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/json-formatter.ts @@ -23,6 +23,7 @@ type ScorecardLevel = { export type ScorecardJsonOutput = { version: string; + level?: string; levels: ScorecardLevel[]; }; @@ -52,6 +53,8 @@ function stripAnsiCodes(text: string): string { export function printScorecardResultsAsJson( problems: ScorecardProblem[], + achievedLevel: string, + targetLevelAchieved: boolean, version: string = '1.0' ): void { const groupedByLevel: Record = {}; @@ -103,6 +106,7 @@ export function printScorecardResultsAsJson( const output: ScorecardJsonOutput = { version, + ...(targetLevelAchieved ? { level: achievedLevel } : {}), levels, }; diff --git a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts index b4c5429ca9..479814b8d8 100644 --- a/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts +++ b/packages/cli/src/commands/scorecard-classic/formatters/stylish-formatter.ts @@ -29,7 +29,11 @@ function formatStylishProblem( return ` ${location} ${severity} ${level} ${ruleId} ${problem.message}`; } -export function printScorecardResults(problems: ScorecardProblem[]): void { +export function printScorecardResults( + problems: ScorecardProblem[], + achievedLevel: string, + targetLevelAchieved: boolean +): void { const problemsByLevel = problems.reduce((acc, problem) => { const level = problem.scorecardLevel || 'Unknown'; if (!acc[level]) { @@ -53,6 +57,9 @@ export function printScorecardResults(problems: ScorecardProblem[]): void { ) ); + targetLevelAchieved && + logger.output(white(bold(`\n ā˜‘ļø Achieved Level: ${cyan(achievedLevel)}\n`))); + for (const [level, levelProblems] of Object.entries(problemsByLevel)) { const severityCounts = levelProblems.reduce((acc, p) => { acc[p.severity] = (acc[p.severity] || 0) + 1; diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index a64550fa77..1ff188415e 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -6,7 +6,7 @@ import { printScorecardResults } from './formatters/stylish-formatter.js'; import { printScorecardResultsAsJson } from './formatters/json-formatter.js'; import { fetchRemoteScorecardAndPlugins } from './remote/fetch-scorecard.js'; import { validateScorecard } from './validation/validate-scorecard.js'; -import { blue, gray, green } from 'colorette'; +import { blue, bold, cyan, gray, green, white } from 'colorette'; import type { ScorecardClassicArgv } from './types.js'; import type { CommandArgs } from '../../wrapper.js'; @@ -21,6 +21,7 @@ export async function handleScorecardClassic({ const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); const externalRefResolver = new BaseResolver(config.resolve); const document = (await externalRefResolver.resolveDocument(null, path, true)) as Document; + const targetLevel = argv['target-level']; const projectUrl = argv['project-url'] || @@ -58,16 +59,23 @@ export async function handleScorecardClassic({ ); logger.info(gray(`\nRunning scorecard for ${formatPath(path)}...\n`)); - const result = await validateScorecard( + const { + problems: result, + achievedLevel, + targetLevelAchieved, + } = await validateScorecard( document, externalRefResolver, remoteScorecardAndPlugins.scorecard!, config.configPath, remoteScorecardAndPlugins?.plugins, + targetLevel, argv.verbose ); if (result.length === 0) { + logger.output(white(bold(`\n ā˜‘ļø Achieved Level: ${cyan(achievedLevel)}\n`))); + logger.output( green( `āœ… No issues found for ${blue( @@ -78,10 +86,16 @@ export async function handleScorecardClassic({ return; } + if (targetLevel && !targetLevelAchieved) { + logger.error( + `\nāŒ Your API specification does not satisfy the target scorecard level "${targetLevel}".\n` + ); + } + if (argv.format === 'json') { - printScorecardResultsAsJson(result, version); + printScorecardResultsAsJson(result, achievedLevel, targetLevelAchieved, version); } else { - printScorecardResults(result); + printScorecardResults(result, achievedLevel, targetLevelAchieved); } const elapsed = getExecutionTime(startedAt); @@ -91,6 +105,12 @@ export async function handleScorecardClassic({ )}.\n` ); + if (targetLevel && !targetLevelAchieved) { + throw new AbortFlowError('Target scorecard level not achieved.'); + } else if (achievedLevel !== 'Non Conformant') { + return; + } + throw new AbortFlowError('Scorecard validation failed.'); } diff --git a/packages/cli/src/commands/scorecard-classic/types.ts b/packages/cli/src/commands/scorecard-classic/types.ts index 285ca418e0..8f21b5dc44 100644 --- a/packages/cli/src/commands/scorecard-classic/types.ts +++ b/packages/cli/src/commands/scorecard-classic/types.ts @@ -6,6 +6,7 @@ export type ScorecardClassicArgv = { config: string; 'project-url'?: string; format: OutputFormat; + 'target-level'?: string; verbose?: boolean; } & VerifyConfigOptions; diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts index 89577b9603..961a4dc56d 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -1,19 +1,34 @@ import { logger, createConfig, lintDocument, pluralize } from '@redocly/openapi-core'; import { evaluatePluginsFromCode } from './plugin-evaluator.js'; +import { exitWithError } from '../../../utils/error.js'; import type { ScorecardConfig } from '@redocly/config'; import type { Document, RawUniversalConfig, Plugin, BaseResolver } from '@redocly/openapi-core'; import type { ScorecardProblem } from '../types.js'; +export type ScorecardValidationResult = { + problems: ScorecardProblem[]; + achievedLevel: string; + targetLevelAchieved: boolean; +}; + export async function validateScorecard( document: Document, externalRefResolver: BaseResolver, scorecardConfig: ScorecardConfig, configPath?: string, pluginsCodeOrPlugins?: string | Plugin[], + targetLevel?: string, verbose = false -): Promise { +): Promise { const problems: ScorecardProblem[] = []; + const levelResults: Map = new Map(); + + if (targetLevel && !scorecardConfig.levels?.some((level) => level.name === targetLevel)) { + exitWithError( + `Target level "${targetLevel}" not found in the scorecard configuration levels.\n` + ); + } for (const level of scorecardConfig?.levels || []) { if (verbose) { @@ -45,23 +60,63 @@ export async function validateScorecard( config, }); + const filteredProblems = levelProblems + .filter(({ ignored }) => !ignored) + .map((problem) => ({ + ...problem, + scorecardLevel: level.name, + })); + + levelResults.set(level.name, filteredProblems); + if (verbose) { logger.info( - `Found ${levelProblems.length} ${pluralize('problem', levelProblems.length)} for level "${ - level.name - }".\n` + `Found ${filteredProblems.length} ${pluralize( + 'problem', + filteredProblems.length + )} for level "${level.name}".\n` ); } - problems.push( - ...levelProblems - .filter(({ ignored }) => !ignored) - .map((problem) => ({ - ...problem, - scorecardLevel: level.name, - })) - ); + problems.push(...filteredProblems); + } + + const achievedLevel = determineAchievedLevel( + levelResults, + scorecardConfig.levels || [], + targetLevel + ); + + const targetLevelAchieved = targetLevel ? achievedLevel === targetLevel : true; + + return { + problems, + achievedLevel, + targetLevelAchieved, + }; +} + +function determineAchievedLevel( + levelResults: Map, + levels: Array<{ name: string }>, + targetLevel?: string +): string { + let lastPassedLevel: string | null = null; + + for (const level of levels) { + const levelProblems = levelResults.get(level.name) || []; + const hasErrors = levelProblems.some((p) => p.severity === 'error'); + + if (hasErrors) { + return lastPassedLevel || 'Non Conformant'; + } + + lastPassedLevel = level.name; + + if (targetLevel && level.name === targetLevel) { + return level.name; + } } - return problems; + return lastPassedLevel || 'Non Conformant'; } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 85b176c562..5b5d65d19c 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -816,6 +816,10 @@ yargs(hideBin(process.argv)) choices: ['stylish', 'json'], default: 'stylish', }, + 'target-level': { + describe: 'Target level for the scorecard.', + type: 'string', + }, verbose: { alias: 'v', describe: 'Apply verbose mode.', From 795a684b78720165dbeb8783c4dbdb6cac4091b7 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:56:58 +0200 Subject: [PATCH 34/41] refactor: change params from positional to named --- .../__tests__/fetch-scorecard.test.ts | 58 +++++++++++++++---- .../__tests__/validate-scorecard.test.ts | 52 ++++++++++++----- .../src/commands/scorecard-classic/index.ts | 20 +++---- .../remote/fetch-scorecard.ts | 48 ++++++++++----- .../validation/validate-scorecard.ts | 28 ++++++--- 5 files changed, 145 insertions(+), 61 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index 7571b84126..b1f976848a 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -23,12 +23,17 @@ describe('fetchRemoteScorecardAndPlugins', () => { }); it('should handle invalid URL format', async () => { - await expect(fetchRemoteScorecardAndPlugins('not-a-valid-url', testToken)).rejects.toThrow(); + await expect( + fetchRemoteScorecardAndPlugins({ projectUrl: 'not-a-valid-url', auth: testToken }) + ).rejects.toThrow(); }); it('should throw error when project URL pattern does not match', async () => { await expect( - fetchRemoteScorecardAndPlugins('https://example.com/invalid/path', testToken) + fetchRemoteScorecardAndPlugins({ + projectUrl: 'https://example.com/invalid/path', + auth: testToken, + }) ).rejects.toThrow(); expect(errorUtils.exitWithError).toHaveBeenCalledWith( @@ -41,7 +46,9 @@ describe('fetchRemoteScorecardAndPlugins', () => { status: 404, }); - await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); + await expect( + fetchRemoteScorecardAndPlugins({ projectUrl: validProjectUrl, auth: testToken }) + ).rejects.toThrow(); expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('Failed to fetch project') @@ -53,7 +60,9 @@ describe('fetchRemoteScorecardAndPlugins', () => { status: 401, }); - await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); + await expect( + fetchRemoteScorecardAndPlugins({ projectUrl: validProjectUrl, auth: testToken }) + ).rejects.toThrow(); expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('Unauthorized access to project') @@ -65,7 +74,9 @@ describe('fetchRemoteScorecardAndPlugins', () => { status: 403, }); - await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); + await expect( + fetchRemoteScorecardAndPlugins({ projectUrl: validProjectUrl, auth: testToken }) + ).rejects.toThrow(); expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('Unauthorized access to project') @@ -82,7 +93,9 @@ describe('fetchRemoteScorecardAndPlugins', () => { }), }); - await expect(fetchRemoteScorecardAndPlugins(validProjectUrl, testToken)).rejects.toThrow(); + await expect( + fetchRemoteScorecardAndPlugins({ projectUrl: validProjectUrl, auth: testToken }) + ).rejects.toThrow(); expect(errorUtils.exitWithError).toHaveBeenCalledWith( expect.stringContaining('No scorecard configuration found') @@ -105,7 +118,10 @@ describe('fetchRemoteScorecardAndPlugins', () => { }), }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + const result = await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: testToken, + }); expect(result).toEqual({ scorecard: mockScorecard, @@ -137,7 +153,10 @@ describe('fetchRemoteScorecardAndPlugins', () => { text: async () => mockPluginsCode, }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + const result = await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: testToken, + }); expect(result).toEqual({ scorecard: mockScorecard, @@ -167,7 +186,10 @@ describe('fetchRemoteScorecardAndPlugins', () => { status: 404, }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + const result = await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: testToken, + }); expect(result).toEqual({ scorecard: mockScorecard, @@ -184,7 +206,10 @@ describe('fetchRemoteScorecardAndPlugins', () => { }), }); - await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken); + await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: testToken, + }); expect(mockFetch).toHaveBeenCalledWith( expect.any(URL), @@ -206,7 +231,11 @@ describe('fetchRemoteScorecardAndPlugins', () => { }), }); - await fetchRemoteScorecardAndPlugins(validProjectUrl, apiKey, true); + await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: apiKey, + isApiKey: true, + }); expect(mockFetch).toHaveBeenCalledWith( expect.any(URL), @@ -239,7 +268,12 @@ describe('fetchRemoteScorecardAndPlugins', () => { text: async () => mockPluginsCode, }); - const result = await fetchRemoteScorecardAndPlugins(validProjectUrl, testToken, false, true); + const result = await fetchRemoteScorecardAndPlugins({ + projectUrl: validProjectUrl, + auth: testToken, + isApiKey: false, + verbose: true, + }); expect(result).toEqual({ scorecard: mockScorecard, diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index b8b76c8c33..89e55a38d8 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -33,7 +33,11 @@ describe('validateScorecard', () => { it('should return empty array when no scorecard levels defined', async () => { const scorecardConfig = { levels: [] }; - const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); expect(result).toEqual({ achievedLevel: 'Non Conformant', @@ -51,7 +55,11 @@ describe('validateScorecard', () => { ], }; - await validateScorecard(mockDocument, mockResolver, scorecardConfig); + await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); expect(openapiCore.createConfig).toHaveBeenCalledTimes(2); expect(openapiCore.lintDocument).toHaveBeenCalledTimes(2); @@ -74,7 +82,11 @@ describe('validateScorecard', () => { vi.mocked(openapiCore.lintDocument).mockResolvedValue(mockProblems as any); - const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); expect(result.problems).toHaveLength(1); expect(result.problems[0].scorecardLevel).toBe('Gold'); @@ -105,7 +117,11 @@ describe('validateScorecard', () => { vi.mocked(openapiCore.lintDocument).mockResolvedValue(mockProblems as any); - const result = await validateScorecard(mockDocument, mockResolver, scorecardConfig); + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); expect(result.problems).toHaveLength(1); expect(result.problems[0].message).toBe('Error 1'); @@ -119,7 +135,12 @@ describe('validateScorecard', () => { const mockPlugins = [{ id: 'test-plugin' }]; vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins); - await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, 'plugin-code'); + await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + pluginsCodeOrPlugins: 'plugin-code', + }); expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code', false); expect(openapiCore.createConfig).toHaveBeenCalledWith( @@ -135,7 +156,12 @@ describe('validateScorecard', () => { const mockPlugins = [{ id: 'test-plugin' }]; - await validateScorecard(mockDocument, mockResolver, scorecardConfig, undefined, mockPlugins); + await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + pluginsCodeOrPlugins: mockPlugins, + }); expect(evaluatePluginsFromCode).not.toHaveBeenCalled(); expect(openapiCore.createConfig).toHaveBeenCalledWith( @@ -152,15 +178,13 @@ describe('validateScorecard', () => { const mockPlugins = [{ id: 'test-plugin' }]; vi.mocked(evaluatePluginsFromCode).mockResolvedValue(mockPlugins); - await validateScorecard( - mockDocument, - mockResolver, + await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, scorecardConfig, - undefined, - 'plugin-code', - undefined, - true - ); + pluginsCodeOrPlugins: 'plugin-code', + verbose: true, + }); expect(evaluatePluginsFromCode).toHaveBeenCalledWith('plugin-code', true); expect(openapiCore.createConfig).toHaveBeenCalledWith( diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index 1ff188415e..a48319012f 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -51,27 +51,27 @@ export async function handleScorecardClassic({ exitWithError('Failed to obtain access token or API key.'); } - const remoteScorecardAndPlugins = await fetchRemoteScorecardAndPlugins( + const remoteScorecardAndPlugins = await fetchRemoteScorecardAndPlugins({ projectUrl, auth, - !!apiKey, - argv.verbose - ); + isApiKey: !!apiKey, + verbose: argv.verbose, + }); logger.info(gray(`\nRunning scorecard for ${formatPath(path)}...\n`)); const { problems: result, achievedLevel, targetLevelAchieved, - } = await validateScorecard( + } = await validateScorecard({ document, externalRefResolver, - remoteScorecardAndPlugins.scorecard!, - config.configPath, - remoteScorecardAndPlugins?.plugins, + scorecardConfig: remoteScorecardAndPlugins.scorecard!, + configPath: config.configPath, + pluginsCodeOrPlugins: remoteScorecardAndPlugins?.plugins, targetLevel, - argv.verbose - ); + verbose: argv.verbose, + }); if (result.length === 0) { logger.output(white(bold(`\n ā˜‘ļø Achieved Level: ${cyan(achievedLevel)}\n`))); diff --git a/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts index c05840c1d1..3849ee7654 100644 --- a/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/remote/fetch-scorecard.ts @@ -3,12 +3,19 @@ import { exitWithError } from '../../../utils/error.js'; import type { RemoteScorecardAndPlugins, Project } from '../types.js'; -export async function fetchRemoteScorecardAndPlugins( - projectUrl: string, - auth: string, +export type FetchRemoteScorecardAndPluginsParams = { + projectUrl: string; + auth: string; + isApiKey?: boolean; + verbose?: boolean; +}; + +export async function fetchRemoteScorecardAndPlugins({ + projectUrl, + auth, isApiKey = false, - verbose = false -): Promise { + verbose = false, +}: FetchRemoteScorecardAndPluginsParams): Promise { if (verbose) { logger.info(`Starting fetch for remote scorecard configuration...\n`); } @@ -22,14 +29,14 @@ export async function fetchRemoteScorecardAndPlugins( const { residency, orgSlug, projectSlug } = parsedProjectUrl; try { - const project = await fetchProjectConfigBySlugs( + const project = await fetchProjectConfigBySlugs({ residency, orgSlug, projectSlug, auth, isApiKey, - verbose - ); + verbose, + }); const scorecard = project?.config.scorecardClassic || project?.config.scorecard; if (!scorecard) { @@ -90,14 +97,23 @@ function parseProjectUrl( }; } -async function fetchProjectConfigBySlugs( - residency: string, - orgSlug: string, - projectSlug: string, - auth: string, - isApiKey: boolean, - verbose = false -): Promise { +type FetchProjectConfigBySlugsParams = { + residency: string; + orgSlug: string; + projectSlug: string; + auth: string; + isApiKey: boolean; + verbose?: boolean; +}; + +async function fetchProjectConfigBySlugs({ + residency, + orgSlug, + projectSlug, + auth, + isApiKey, + verbose = false, +}: FetchProjectConfigBySlugsParams): Promise { const authHeaders = createAuthHeaders(auth, isApiKey); const projectUrl = new URL(`${residency}/api/orgs/${orgSlug}/projects/${projectSlug}`); diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts index 961a4dc56d..031d1e7a46 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -12,15 +12,25 @@ export type ScorecardValidationResult = { targetLevelAchieved: boolean; }; -export async function validateScorecard( - document: Document, - externalRefResolver: BaseResolver, - scorecardConfig: ScorecardConfig, - configPath?: string, - pluginsCodeOrPlugins?: string | Plugin[], - targetLevel?: string, - verbose = false -): Promise { +export type ValidateScorecardParams = { + document: Document; + externalRefResolver: BaseResolver; + scorecardConfig: ScorecardConfig; + configPath?: string; + pluginsCodeOrPlugins?: string | Plugin[]; + targetLevel?: string; + verbose?: boolean; +}; + +export async function validateScorecard({ + document, + externalRefResolver, + scorecardConfig, + configPath, + pluginsCodeOrPlugins, + targetLevel, + verbose = false, +}: ValidateScorecardParams): Promise { const problems: ScorecardProblem[] = []; const levelResults: Map = new Map(); From e45bbc3c3ca8fd71ee14e2e6a25f6f686ef00224 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 11:08:45 +0200 Subject: [PATCH 35/41] fix: change logic to check if level has any problems --- .../scorecard-classic/validation/validate-scorecard.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts index 031d1e7a46..6b1f1924f7 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -115,9 +115,9 @@ function determineAchievedLevel( for (const level of levels) { const levelProblems = levelResults.get(level.name) || []; - const hasErrors = levelProblems.some((p) => p.severity === 'error'); + const hasProblems = levelProblems.length > 0; - if (hasErrors) { + if (hasProblems) { return lastPassedLevel || 'Non Conformant'; } From 0f3795225048937dba407fd4319ebb0e524c13fa Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 11:09:04 +0200 Subject: [PATCH 36/41] feat: add tests for determineAchievedLevel function --- .../__tests__/validate-scorecard.test.ts | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index 89e55a38d8..804d0bcbcd 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -192,4 +192,168 @@ describe('validateScorecard', () => { expect.any(Object) ); }); + + describe('determineAchievedLevel', () => { + it('should return highest level when all levels pass without problems', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + { name: 'Gold', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument).mockResolvedValue([]); + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); + + expect(result.achievedLevel).toBe('Gold'); + expect(result.targetLevelAchieved).toBe(true); + }); + + it('should return previous level when current level has errors', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + { name: 'Gold', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument) + .mockResolvedValueOnce([]) // Bronze: no problems + .mockResolvedValueOnce([ + { + message: 'Silver level error', + ruleId: 'test-rule', + severity: 'error', + location: [], + ignored: false, + }, + ] as any); // Silver: has error + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); + + expect(result.achievedLevel).toBe('Bronze'); + expect(result.problems).toHaveLength(1); + }); + + it('should return previous level when current level has warnings', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument) + .mockResolvedValueOnce([]) // Bronze: no problems + .mockResolvedValueOnce([ + { + message: 'Silver level warning', + ruleId: 'test-rule', + severity: 'warn', + location: [], + ignored: false, + }, + ] as any); // Silver: has warning + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); + + expect(result.achievedLevel).toBe('Bronze'); + expect(result.problems).toHaveLength(1); + }); + + it('should return "Non Conformant" when first level has problems', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument).mockResolvedValue([ + { + message: 'Bronze level error', + ruleId: 'test-rule', + severity: 'error', + location: [], + ignored: false, + }, + ] as any); + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + }); + + expect(result.achievedLevel).toBe('Non Conformant'); + }); + + it('should return target level when specified and achieved', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + { name: 'Gold', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument).mockResolvedValue([]); + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + targetLevel: 'Silver', + }); + + expect(result.achievedLevel).toBe('Silver'); + expect(result.targetLevelAchieved).toBe(true); + }); + + it('should indicate target level not achieved when level has problems', async () => { + const scorecardConfig = { + levels: [ + { name: 'Bronze', rules: {} }, + { name: 'Silver', rules: {} }, + ], + }; + + vi.mocked(openapiCore.lintDocument) + .mockResolvedValueOnce([]) // Bronze: no problems + .mockResolvedValueOnce([ + { + message: 'Silver level error', + ruleId: 'test-rule', + severity: 'error', + location: [], + ignored: false, + }, + ] as any); // Silver: has error + + const result = await validateScorecard({ + document: mockDocument, + externalRefResolver: mockResolver, + scorecardConfig, + targetLevel: 'Silver', + }); + + expect(result.achievedLevel).toBe('Bronze'); + expect(result.targetLevelAchieved).toBe(false); + }); + }); }); From 641258685c9ad6019676462d4f0c31a2847fd7d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacek=20=C5=81=C4=99kawa?= <164185257+JLekawa@users.noreply.github.com> Date: Mon, 15 Dec 2025 11:12:45 +0100 Subject: [PATCH 37/41] Apply suggestions from code review --- docs/@v2/commands/scorecard-classic.md | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/@v2/commands/scorecard-classic.md b/docs/@v2/commands/scorecard-classic.md index 13c5270b9c..0505d473ed 100644 --- a/docs/@v2/commands/scorecard-classic.md +++ b/docs/@v2/commands/scorecard-classic.md @@ -81,13 +81,13 @@ redocly scorecard-classic openapi/openapi.yaml --format=json The JSON output is grouped by scorecard level and includes: -- Version information -- Achieved scorecard level -- Summary of errors and warnings for each level -- Rule ID and documentation link (for built-in rules) -- Severity level (error or warning) -- Location information (file path, line/column range, and JSON pointer) -- Descriptive message about the violation +- version information +- achieved scorecard level +- summary of errors and warnings for each level +- rule ID and documentation link (for built-in rules) +- severity level (error or warning) +- location information (file path, line/column range, and JSON pointer) +- descriptive message about the violation ### Validate against a target level @@ -99,9 +99,9 @@ redocly scorecard-classic openapi/openapi.yaml --target-level=Gold If the API doesn't meet the target level, the command: -- Displays which level was actually achieved -- Shows all validation issues preventing the target level from being met -- Exits with a non-zero exit code (useful for CI/CD pipelines) +- displays which level was actually achieved +- shows all validation issues preventing the target level from being met +- exits with a non-zero exit code (useful for CI/CD pipelines) This is particularly useful in CI/CD pipelines to enforce minimum quality standards before deployment. @@ -115,9 +115,9 @@ redocly scorecard-classic openapi/openapi.yaml --verbose Verbose mode displays additional information such as: -- Project URL being used -- Authentication status -- Detailed logging of the evaluation process +- project URL being used +- authentication status +- detailed logging of the evaluation process ## Authentication @@ -161,11 +161,11 @@ When all checks pass, the command displays a success message: When issues are found, the output shows: -- The achieved scorecard level -- The rule that was violated -- The scorecard level of the rule -- The location in the API description where the issue occurs -- A descriptive message explaining the violation +- the achieved scorecard level +- the rule that was violated +- the scorecard level of the rule +- the location in the API description where the issue occurs +- a descriptive message explaining the violation If a `--target-level` is specified and not achieved, the command displays an error message and exits with a non-zero code. From db0ba06b503f258bbf02b571723de47498877e6c Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 12:40:01 +0200 Subject: [PATCH 38/41] fix: remove dupication in tests --- .../scorecard-classic/__tests__/fetch-scorecard.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts index b1f976848a..88dafeb7e7 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/fetch-scorecard.test.ts @@ -7,8 +7,6 @@ describe('fetchRemoteScorecardAndPlugins', () => { const testToken = 'test-token'; beforeEach(() => { - vi.unstubAllEnvs(); - delete process.env.REDOCLY_AUTHORIZATION; global.fetch = mockFetch; mockFetch.mockClear(); vi.spyOn(errorUtils, 'exitWithError').mockImplementation(() => { From 0886f41b35d10d35b67674241b10e10f2d2028bc Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:01:14 +0200 Subject: [PATCH 39/41] fix: types in command args --- packages/cli/src/commands/scorecard-classic/types.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/types.ts b/packages/cli/src/commands/scorecard-classic/types.ts index 8f21b5dc44..721b1063a1 100644 --- a/packages/cli/src/commands/scorecard-classic/types.ts +++ b/packages/cli/src/commands/scorecard-classic/types.ts @@ -1,4 +1,3 @@ -import type { VerifyConfigOptions } from '../../types.js'; import type { NormalizedProblem, OutputFormat, ResolvedConfig } from '@redocly/openapi-core'; export type ScorecardClassicArgv = { @@ -8,7 +7,7 @@ export type ScorecardClassicArgv = { format: OutputFormat; 'target-level'?: string; verbose?: boolean; -} & VerifyConfigOptions; +}; export type ScorecardProblem = NormalizedProblem & { scorecardLevel?: string }; From e7484d3437c34f6a19f0413c3ec797ef09f0d5c2 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:01:58 +0200 Subject: [PATCH 40/41] feat: add collecting spec data and error handling --- packages/cli/src/commands/scorecard-classic/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/commands/scorecard-classic/index.ts b/packages/cli/src/commands/scorecard-classic/index.ts index a48319012f..0460230d59 100644 --- a/packages/cli/src/commands/scorecard-classic/index.ts +++ b/packages/cli/src/commands/scorecard-classic/index.ts @@ -16,12 +16,19 @@ export async function handleScorecardClassic({ argv, config, version, + collectSpecData, }: CommandArgs) { const startedAt = performance.now(); - const [{ path }] = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); + const apis = await getFallbackApisOrExit(argv.api ? [argv.api] : [], config); + if (!apis.length) { + exitWithError('No APIs were provided.'); + } + + const path = apis[0].path; const externalRefResolver = new BaseResolver(config.resolve); const document = (await externalRefResolver.resolveDocument(null, path, true)) as Document; const targetLevel = argv['target-level']; + collectSpecData?.(document.parsed); const projectUrl = argv['project-url'] || From c182102074a8bbdba022a9c1a29bba2363332827 Mon Sep 17 00:00:00 2001 From: Albina Blazhko <46962291+AlbinaBlazhko17@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:52:13 +0200 Subject: [PATCH 41/41] fix: pass level if level has only warnings --- .../scorecard-classic/__tests__/validate-scorecard.test.ts | 6 +++--- .../scorecard-classic/validation/validate-scorecard.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts index 804d0bcbcd..e91788f7f4 100644 --- a/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts +++ b/packages/cli/src/commands/scorecard-classic/__tests__/validate-scorecard.test.ts @@ -246,7 +246,7 @@ describe('validateScorecard', () => { expect(result.problems).toHaveLength(1); }); - it('should return previous level when current level has warnings', async () => { + it('should achieve level even with warnings (only errors prevent achievement)', async () => { const scorecardConfig = { levels: [ { name: 'Bronze', rules: {} }, @@ -264,7 +264,7 @@ describe('validateScorecard', () => { location: [], ignored: false, }, - ] as any); // Silver: has warning + ] as any); // Silver: has warning but no errors const result = await validateScorecard({ document: mockDocument, @@ -272,7 +272,7 @@ describe('validateScorecard', () => { scorecardConfig, }); - expect(result.achievedLevel).toBe('Bronze'); + expect(result.achievedLevel).toBe('Silver'); expect(result.problems).toHaveLength(1); }); diff --git a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts index 6b1f1924f7..031d1e7a46 100644 --- a/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts +++ b/packages/cli/src/commands/scorecard-classic/validation/validate-scorecard.ts @@ -115,9 +115,9 @@ function determineAchievedLevel( for (const level of levels) { const levelProblems = levelResults.get(level.name) || []; - const hasProblems = levelProblems.length > 0; + const hasErrors = levelProblems.some((p) => p.severity === 'error'); - if (hasProblems) { + if (hasErrors) { return lastPassedLevel || 'Non Conformant'; }