Skip to content

Commit 0d80439

Browse files
ravikiranvmravikiranvm
authored
Add Benchmarks endpoint (#1955)
Fixes OPS-3646. - Adds Benchmark module and wizard endpoint - Adds integration tests for the module - Adds utility functions and enum for the benchmark module
1 parent cfcf842 commit 0d80439

File tree

11 files changed

+318
-2
lines changed

11 files changed

+318
-2
lines changed

‎packages/server/api/src/app/app.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import { aiModule } from './ai/ai.module';
3636
import { appConnectionModule } from './app-connection/app-connection.module';
3737
import { appEventRoutingModule } from './app-event-routing/app-event-routing.module';
3838
import { authenticationModule } from './authentication/authentication.module';
39+
import { benchmarkModule } from './benchmark/benchmark-module';
3940
import { blockVariableModule } from './block-variable/block-variable-module';
4041
import { blockModule } from './blocks/base-block-module';
4142
import { blockSyncService } from './blocks/block-sync-service';
@@ -224,6 +225,7 @@ export const setupApp = async (
224225
await app.register(userSettingsModule);
225226
await app.register(aiModule);
226227
await app.register(blockVariableModule);
228+
await app.register(benchmarkModule);
227229

228230
app.get(
229231
'/redirect',

‎packages/server/api/src/app/benchmark/benchmark-feature-guard.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { AppSystemProp, logger, system } from '@openops/server-shared';
2+
import { throwFeatureDisabledError } from './errors';
3+
4+
export async function assertBenchmarkFeatureEnabled(
5+
provider: string,
6+
projectId: string,
7+
): Promise<void> {
8+
if (system.getBoolean(AppSystemProp.FINOPS_BENCHMARK_ENABLED) !== true) {
9+
logger.info(
10+
'Benchmark access denied: FINOPS_BENCHMARK_ENABLED flag is not enabled',
11+
{ provider, projectId },
12+
);
13+
throwFeatureDisabledError('Benchmark feature is not enabled');
14+
}
15+
}

‎packages/server/api/src/app/benchmark/benchmark-module.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { FastifyPluginAsyncTypebox } from '@fastify/type-provider-typebox';
2+
import { benchmarkController } from './benchmark.controller';
3+
4+
export const benchmarkModule: FastifyPluginAsyncTypebox = async (app) => {
5+
await app.register(benchmarkController, { prefix: '/v1/benchmarks' });
6+
};

‎packages/server/api/src/app/benchmark/benchmark.controller.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import {
2+
FastifyPluginAsyncTypebox,
3+
Type,
4+
} from '@fastify/type-provider-typebox';
5+
import {
6+
BenchmarkProviders,
7+
BenchmarkWizardRequest,
8+
BenchmarkWizardStepResponse,
9+
PrincipalType,
10+
} from '@openops/shared';
11+
import { StatusCodes } from 'http-status-codes';
12+
import { assertBenchmarkFeatureEnabled } from './benchmark-feature-guard';
13+
import { resolveWizardNavigation } from './wizard.service';
14+
15+
export const benchmarkController: FastifyPluginAsyncTypebox = async (app) => {
16+
app.post(
17+
'/:provider/wizard',
18+
WizardStepRequestOptions,
19+
async (request, reply) => {
20+
await assertBenchmarkFeatureEnabled(
21+
request.params.provider,
22+
request.principal.projectId,
23+
);
24+
25+
const step = await resolveWizardNavigation(
26+
request.params.provider,
27+
{
28+
currentStep: request.body.currentStep,
29+
benchmarkConfiguration: request.body.benchmarkConfiguration,
30+
},
31+
request.principal.projectId,
32+
);
33+
return reply.status(StatusCodes.OK).send(step);
34+
},
35+
);
36+
};
37+
38+
const WizardStepRequestOptions = {
39+
config: {
40+
allowedPrincipals: [PrincipalType.USER],
41+
},
42+
schema: {
43+
tags: ['benchmarks'],
44+
description:
45+
'Returns a step in the benchmark configuration wizard for the specified provider, including options and progress.',
46+
params: Type.Object({
47+
provider: Type.Enum(BenchmarkProviders),
48+
}),
49+
body: BenchmarkWizardRequest,
50+
response: {
51+
[StatusCodes.OK]: BenchmarkWizardStepResponse,
52+
},
53+
},
54+
};

‎packages/server/api/src/app/benchmark/errors.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,10 @@ export function throwValidationError(message: string): never {
66
message,
77
);
88
}
9+
10+
export function throwFeatureDisabledError(message: string): never {
11+
throw new ApplicationError(
12+
{ code: ErrorCode.FEATURE_DISABLED, params: { message } },
13+
message,
14+
);
15+
}

‎packages/server/api/src/app/benchmark/register-providers.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1+
import { BenchmarkProviders } from '@openops/shared';
12
import { registerProvider } from './provider-adapter';
23
import { awsProviderAdapter } from './providers/aws';
34

45
function registerProviders(): void {
5-
registerProvider('aws', awsProviderAdapter);
6+
registerProvider(BenchmarkProviders.AWS, awsProviderAdapter);
67
}
78

89
registerProviders();

‎packages/server/api/src/app/flags/flag.service.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ export const flagService = {
297297
},
298298
{
299299
id: FlagId.FINOPS_BENCHMARK_ENABLED,
300-
value: false,
300+
value: system.getBoolean(AppSystemProp.FINOPS_BENCHMARK_ENABLED),
301301
created,
302302
updated,
303303
},

‎packages/server/api/test/integration/ce/benchmark/benchmark-module.test.ts‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
const wizardServiceMock = {
2+
resolveWizardNavigation: jest.fn(),
3+
};
4+
jest.mock(
5+
'../../../../src/app/benchmark/wizard.service',
6+
() => wizardServiceMock,
7+
);
8+
9+
import {
10+
ApplicationError,
11+
BenchmarkProviders,
12+
type BenchmarkWizardStepResponse,
13+
ErrorCode,
14+
PrincipalType,
15+
Project,
16+
} from '@openops/shared';
17+
import { FastifyInstance, LightMyRequestResponse } from 'fastify';
18+
import { StatusCodes } from 'http-status-codes';
19+
import { databaseConnection } from '../../../../src/app/database/database-connection';
20+
import { setupServer } from '../../../../src/app/server';
21+
import { generateMockToken } from '../../../helpers/auth';
22+
import {
23+
createMockOrganization,
24+
createMockProject,
25+
createMockUser,
26+
} from '../../../helpers/mocks';
27+
28+
const mockWizardStep: BenchmarkWizardStepResponse = {
29+
currentStep: 'mock-current-step-id',
30+
title: 'Mock current step title',
31+
nextStep: 'mock-next-step-id',
32+
selectionType: 'single',
33+
options: [],
34+
stepIndex: 1,
35+
totalSteps: 3,
36+
};
37+
38+
const originalEnv = { ...process.env };
39+
let app: FastifyInstance | null = null;
40+
41+
beforeAll(async () => {
42+
wizardServiceMock.resolveWizardNavigation.mockResolvedValue(mockWizardStep);
43+
process.env.OPS_FINOPS_BENCHMARK_ENABLED = 'true';
44+
await databaseConnection().initialize();
45+
app = await setupServer();
46+
});
47+
48+
afterAll(async () => {
49+
await databaseConnection().destroy();
50+
await app?.close();
51+
process.env = originalEnv;
52+
});
53+
54+
describe('Benchmark wizard API', () => {
55+
const createAndInsertMocks = async (): Promise<{
56+
token: string;
57+
project: Project;
58+
}> => {
59+
const mockUser = createMockUser();
60+
await databaseConnection().getRepository('user').save([mockUser]);
61+
62+
const mockOrganization = createMockOrganization({ ownerId: mockUser.id });
63+
await databaseConnection()
64+
.getRepository('organization')
65+
.save(mockOrganization);
66+
67+
const mockProject = createMockProject({
68+
ownerId: mockUser.id,
69+
organizationId: mockOrganization.id,
70+
});
71+
await databaseConnection().getRepository('project').save([mockProject]);
72+
73+
const mockToken = await generateMockToken({
74+
id: mockUser.id,
75+
type: PrincipalType.USER,
76+
projectId: mockProject.id,
77+
organization: { id: mockOrganization.id },
78+
});
79+
80+
return { token: mockToken, project: mockProject };
81+
};
82+
83+
const postWizard = async ({
84+
provider,
85+
token,
86+
body = {},
87+
}: {
88+
provider: string;
89+
token?: string;
90+
body?: Record<string, unknown>;
91+
}): Promise<LightMyRequestResponse | undefined> =>
92+
app?.inject({
93+
method: 'POST',
94+
url: `/v1/benchmarks/${provider}/wizard`,
95+
headers: token ? { authorization: `Bearer ${token}` } : undefined,
96+
body,
97+
});
98+
99+
describe('POST /v1/benchmarks/:provider/wizard', () => {
100+
beforeEach(() => {
101+
wizardServiceMock.resolveWizardNavigation.mockResolvedValue(
102+
mockWizardStep,
103+
);
104+
process.env.OPS_FINOPS_BENCHMARK_ENABLED = 'true';
105+
});
106+
107+
it('calls resolveWizardNavigation with AWS provider, body, and projectId and returns mocked step', async () => {
108+
const { token, project } = await createAndInsertMocks();
109+
const body = {};
110+
111+
const response = await postWizard({
112+
provider: BenchmarkProviders.AWS,
113+
token,
114+
body,
115+
});
116+
117+
expect(response?.statusCode).toBe(StatusCodes.OK);
118+
expect(response?.json()).toEqual(mockWizardStep);
119+
expect(wizardServiceMock.resolveWizardNavigation).toHaveBeenCalledTimes(
120+
1,
121+
);
122+
expect(wizardServiceMock.resolveWizardNavigation).toHaveBeenCalledWith(
123+
BenchmarkProviders.AWS,
124+
{
125+
currentStep: undefined,
126+
benchmarkConfiguration: undefined,
127+
},
128+
project.id,
129+
);
130+
});
131+
132+
it('passes currentStep and benchmarkConfiguration to resolveWizardNavigation', async () => {
133+
const { token, project } = await createAndInsertMocks();
134+
const body = {
135+
currentStep: 'connection',
136+
benchmarkConfiguration: { connection: ['conn-1'] },
137+
};
138+
139+
const response = await postWizard({
140+
provider: BenchmarkProviders.AWS,
141+
token,
142+
body,
143+
});
144+
145+
expect(response?.statusCode).toBe(StatusCodes.OK);
146+
expect(response?.json()).toEqual(mockWizardStep);
147+
expect(wizardServiceMock.resolveWizardNavigation).toHaveBeenCalledWith(
148+
BenchmarkProviders.AWS,
149+
{
150+
currentStep: 'connection',
151+
benchmarkConfiguration: { connection: ['conn-1'] },
152+
},
153+
project.id,
154+
);
155+
});
156+
157+
it('returns 400 when provider is not in BenchmarkProviders enum', async () => {
158+
const { token } = await createAndInsertMocks();
159+
wizardServiceMock.resolveWizardNavigation.mockClear();
160+
161+
const response = await postWizard({
162+
provider: 'invalidprovider',
163+
token,
164+
body: {},
165+
});
166+
167+
expect(response?.statusCode).toBe(StatusCodes.BAD_REQUEST);
168+
expect(wizardServiceMock.resolveWizardNavigation).not.toHaveBeenCalled();
169+
});
170+
171+
it('returns 409 with VALIDATION code when resolveWizardNavigation throws', async () => {
172+
wizardServiceMock.resolveWizardNavigation.mockRejectedValue(
173+
new ApplicationError(
174+
{
175+
code: ErrorCode.VALIDATION,
176+
params: { message: 'Unknown provider' },
177+
},
178+
'Unknown provider',
179+
),
180+
);
181+
const { token } = await createAndInsertMocks();
182+
183+
const response = await postWizard({
184+
provider: BenchmarkProviders.AWS,
185+
token,
186+
body: {},
187+
});
188+
189+
expect(response?.statusCode).toBe(StatusCodes.CONFLICT);
190+
const data = response?.json();
191+
expect(data?.code).toBe('VALIDATION');
192+
expect(data?.params).toBeDefined();
193+
});
194+
195+
it('returns 402 when FINOPS_BENCHMARK_ENABLED flag is disabled', async () => {
196+
process.env.OPS_FINOPS_BENCHMARK_ENABLED = 'false';
197+
wizardServiceMock.resolveWizardNavigation.mockClear();
198+
199+
const { token } = await createAndInsertMocks();
200+
const response = await postWizard({
201+
provider: BenchmarkProviders.AWS,
202+
token,
203+
body: {},
204+
});
205+
206+
expect(response?.statusCode).toBe(StatusCodes.PAYMENT_REQUIRED);
207+
const data = response?.json();
208+
expect(data?.code).toBe('FEATURE_DISABLED');
209+
expect(data?.params).toBeDefined();
210+
expect(wizardServiceMock.resolveWizardNavigation).not.toHaveBeenCalled();
211+
});
212+
213+
it('returns 401 when not authenticated', async () => {
214+
wizardServiceMock.resolveWizardNavigation.mockClear();
215+
216+
const response = await postWizard({
217+
provider: BenchmarkProviders.AWS,
218+
body: {},
219+
});
220+
221+
expect(response?.statusCode).toBe(StatusCodes.UNAUTHORIZED);
222+
expect(wizardServiceMock.resolveWizardNavigation).not.toHaveBeenCalled();
223+
});
224+
});
225+
});

‎packages/server/shared/src/lib/system/system-prop.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ export enum AppSystemProp {
114114

115115
MAX_LLM_CALLS_WITHOUT_INTERACTION = 'MAX_LLM_CALLS_WITHOUT_INTERACTION',
116116
LLM_CHAT_EXPIRE_TIME_SECONDS = 'LLM_CHAT_EXPIRE_TIME_SECONDS',
117+
118+
FINOPS_BENCHMARK_ENABLED = 'FINOPS_BENCHMARK_ENABLED',
117119
}
118120

119121
export enum SharedSystemProp {

‎packages/shared/src/lib/benchmark/benchmark-providers.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
export enum BenchmarkProviders {
2+
AWS = 'aws',
3+
}

0 commit comments

Comments
 (0)