Skip to content

Commit a76a805

Browse files
aarsilvaarsilv
authored
Update automated tests to also check evaluation details (#272)
* combine test code for obsfucated and unobfuscated * handle looking at obfuscated rules for details * tests passing * bandit config uses updatedAt * throw if no details when expected * assignment value type
1 parent 8b5f765 commit a76a805

File tree

5 files changed

+137
-113
lines changed

5 files changed

+137
-113
lines changed

‎src/client/eppo-client-assignment-details.spec.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as fs from 'fs';
22

33
import {
4+
AssignmentVariationValue,
45
IAssignmentTestCase,
56
MOCK_UFC_RESPONSE_FILE,
67
readMockUFCResponse,
@@ -328,8 +329,8 @@ describe('EppoClient get*AssignmentDetails', () => {
328329
flagKey: string,
329330
subjectKey: string,
330331
subjectAttributes: Record<string, AttributeType>,
331-
defaultValue: boolean | string | number | object,
332-
) => IAssignmentDetails<boolean | string | number | object>;
332+
defaultValue: AssignmentVariationValue,
333+
) => IAssignmentDetails<AssignmentVariationValue>;
333334
if (!assignmentFn) {
334335
throw new Error(`Unknown variation type: ${variationType}`);
335336
}

‎src/client/eppo-client.spec.ts‎

Lines changed: 49 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as td from 'testdouble';
44

55
import {
66
ASSIGNMENT_TEST_DATA_DIR,
7+
AssignmentVariationValue,
78
getTestAssignments,
89
IAssignmentTestCase,
910
MOCK_UFC_RESPONSE_FILE,
@@ -28,7 +29,11 @@ import { Flag, ObfuscatedFlag, VariationType, FormatEnum, Variation } from '../i
2829
import { getMD5Hash } from '../obfuscation';
2930
import { AttributeType } from '../types';
3031

31-
import EppoClient, { checkTypeMatch, FlagConfigurationRequestParameters } from './eppo-client';
32+
import EppoClient, {
33+
checkTypeMatch,
34+
FlagConfigurationRequestParameters,
35+
IAssignmentDetails,
36+
} from './eppo-client';
3237
import { initConfiguration } from './test-utils';
3338

3439
// Use a known salt to produce deterministic hashes
@@ -317,50 +322,66 @@ describe('EppoClient E2E test', () => {
317322
});
318323
});
319324

320-
describe('UFC Shared Test Cases', () => {
325+
describe.each(['Not Obfuscated', 'Obfuscated'])('UFC Shared Test Cases %s', (obfuscationType) => {
321326
const testCases = testCasesByFileName<IAssignmentTestCase>(ASSIGNMENT_TEST_DATA_DIR);
327+
const isObfuscated = obfuscationType === 'Obfuscated';
322328

323-
describe('Not obfuscated', () => {
324-
beforeAll(async () => {
325-
global.fetch = jest.fn(() => {
326-
return Promise.resolve({
327-
ok: true,
328-
status: 200,
329-
json: () => Promise.resolve(readMockUFCResponse(MOCK_UFC_RESPONSE_FILE)),
330-
});
331-
}) as jest.Mock;
329+
beforeAll(async () => {
330+
global.fetch = jest.fn(() => {
331+
return Promise.resolve({
332+
ok: true,
333+
status: 200,
334+
json: () =>
335+
Promise.resolve(
336+
readMockUFCResponse(
337+
isObfuscated ? OBFUSCATED_MOCK_UFC_RESPONSE_FILE : MOCK_UFC_RESPONSE_FILE,
338+
),
339+
),
340+
});
341+
}) as jest.Mock;
332342

333-
await initConfiguration(storage);
334-
});
343+
await initConfiguration(storage);
344+
});
335345

336-
afterAll(() => {
337-
jest.restoreAllMocks();
338-
});
346+
afterAll(() => {
347+
jest.restoreAllMocks();
348+
});
349+
350+
describe.each(['Scalar', 'With Details'])('%s', (assignmentType) => {
351+
const assignmentWithDetails = assignmentType === 'With Details';
339352

340353
it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => {
341354
const { flag, variationType, defaultValue, subjects } = testCases[fileName];
342-
const client = new EppoClient({ flagConfigurationStore: storage });
355+
const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated });
343356
client.setIsGracefulFailureMode(false);
344357

345358
let assignments: {
346359
subject: SubjectTestCase;
347-
assignment: string | boolean | number | null | object;
360+
assignment: AssignmentVariationValue;
348361
}[] = [];
349362

350-
const typeAssignmentFunctions = {
351-
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
352-
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
353-
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
354-
[VariationType.STRING]: client.getStringAssignment.bind(client),
355-
[VariationType.JSON]: client.getJSONAssignment.bind(client),
356-
};
363+
const typeAssignmentFunctions = assignmentWithDetails
364+
? {
365+
[VariationType.BOOLEAN]: client.getBooleanAssignmentDetails.bind(client),
366+
[VariationType.NUMERIC]: client.getNumericAssignmentDetails.bind(client),
367+
[VariationType.INTEGER]: client.getIntegerAssignmentDetails.bind(client),
368+
[VariationType.STRING]: client.getStringAssignmentDetails.bind(client),
369+
[VariationType.JSON]: client.getJSONAssignmentDetails.bind(client),
370+
}
371+
: {
372+
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
373+
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
374+
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
375+
[VariationType.STRING]: client.getStringAssignment.bind(client),
376+
[VariationType.JSON]: client.getJSONAssignment.bind(client),
377+
};
357378

358379
const assignmentFn = typeAssignmentFunctions[variationType] as (
359380
flagKey: string,
360381
subjectKey: string,
361382
subjectAttributes: Record<string, AttributeType>,
362-
defaultValue: boolean | string | number | object,
363-
) => never;
383+
defaultValue: AssignmentVariationValue,
384+
) => AssignmentVariationValue | IAssignmentDetails<AssignmentVariationValue>;
364385
if (!assignmentFn) {
365386
throw new Error(`Unknown variation type: ${variationType}`);
366387
}
@@ -370,56 +391,7 @@ describe('EppoClient E2E test', () => {
370391
assignmentFn,
371392
);
372393

373-
validateTestAssignments(assignments, flag);
374-
});
375-
});
376-
377-
describe('Obfuscated', () => {
378-
beforeAll(async () => {
379-
global.fetch = jest.fn(() => {
380-
return Promise.resolve({
381-
ok: true,
382-
status: 200,
383-
json: () => Promise.resolve(readMockUFCResponse(OBFUSCATED_MOCK_UFC_RESPONSE_FILE)),
384-
});
385-
}) as jest.Mock;
386-
387-
await initConfiguration(storage);
388-
});
389-
390-
afterAll(() => {
391-
jest.restoreAllMocks();
392-
});
393-
394-
it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => {
395-
const { flag, variationType, defaultValue, subjects } = testCases[fileName];
396-
const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated: true });
397-
client.setIsGracefulFailureMode(false);
398-
399-
const typeAssignmentFunctions = {
400-
[VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client),
401-
[VariationType.NUMERIC]: client.getNumericAssignment.bind(client),
402-
[VariationType.INTEGER]: client.getIntegerAssignment.bind(client),
403-
[VariationType.STRING]: client.getStringAssignment.bind(client),
404-
[VariationType.JSON]: client.getJSONAssignment.bind(client),
405-
};
406-
407-
const assignmentFn = typeAssignmentFunctions[variationType] as (
408-
flagKey: string,
409-
subjectKey: string,
410-
subjectAttributes: Record<string, AttributeType>,
411-
defaultValue: boolean | string | number | object,
412-
) => never;
413-
if (!assignmentFn) {
414-
throw new Error(`Unknown variation type: ${variationType}`);
415-
}
416-
417-
const assignments = getTestAssignments(
418-
{ flag, variationType, defaultValue, subjects },
419-
assignmentFn,
420-
);
421-
422-
validateTestAssignments(assignments, flag);
394+
validateTestAssignments(assignments, flag, assignmentWithDetails, isObfuscated);
423395
});
424396
});
425397
});

‎src/evaluator.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ export class Evaluator {
144144
configDetails.configFormat,
145145
);
146146
} catch (err: any) {
147-
console.error('>>>>', err);
148147
const flagEvaluationDetails = flagEvaluationDetailsBuilder.gracefulBuild(
149148
'ASSIGNMENT_ERROR',
150149
`Assignment Error: ${err.message}`,

‎src/http-client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ export interface IUniversalFlagConfigResponse {
4343
}
4444

4545
export interface IBanditParametersResponse {
46+
updatedAt: string; // ISO formatted string
4647
bandits: Record<string, BanditParameters>;
4748
}
4849

‎test/testHelpers.ts‎

Lines changed: 84 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,13 @@ import * as fs from 'fs';
22

33
import { isEqual } from 'lodash';
44

5-
import { AttributeType, ContextAttributes, IAssignmentDetails, VariationType } from '../src';
5+
import {
6+
AttributeType,
7+
ContextAttributes,
8+
IAssignmentDetails,
9+
Variation,
10+
VariationType,
11+
} from '../src';
612
import { IFlagEvaluationDetails } from '../src/flag-evaluation-details-builder';
713
import { IBanditParametersResponse, IUniversalFlagConfigResponse } from '../src/http-client';
814

@@ -20,17 +26,19 @@ const MOCK_PRECOMPUTED_FILENAME = 'precomputed-v1';
2026
export const MOCK_PRECOMPUTED_WIRE_FILE = `${MOCK_PRECOMPUTED_FILENAME}.json`;
2127
export const MOCK_DEOBFUSCATED_PRECOMPUTED_RESPONSE_FILE = `${MOCK_PRECOMPUTED_FILENAME}-deobfuscated.json`;
2228

29+
export type AssignmentVariationValue = Variation['value'] | object;
30+
2331
export interface SubjectTestCase {
2432
subjectKey: string;
2533
subjectAttributes: Record<string, AttributeType>;
26-
assignment: string | number | boolean | object;
34+
assignment: AssignmentVariationValue;
2735
evaluationDetails: IFlagEvaluationDetails;
2836
}
2937

3038
export interface IAssignmentTestCase {
3139
flag: string;
3240
variationType: VariationType;
33-
defaultValue: string | number | boolean | object;
41+
defaultValue: AssignmentVariationValue;
3442
subjects: SubjectTestCase[];
3543
}
3644

@@ -85,12 +93,15 @@ export function getTestAssignments(
8593
flagKey: string,
8694
subjectKey: string,
8795
subjectAttributes: Record<string, AttributeType>,
88-
defaultValue: string | number | boolean | object,
89-
) => never,
90-
): { subject: SubjectTestCase; assignment: string | boolean | number | null | object }[] {
96+
defaultValue: AssignmentVariationValue,
97+
) => AssignmentVariationValue | IAssignmentDetails<AssignmentVariationValue>,
98+
): {
99+
subject: SubjectTestCase;
100+
assignment: AssignmentVariationValue | IAssignmentDetails<AssignmentVariationValue>;
101+
}[] {
91102
const assignments: {
92103
subject: SubjectTestCase;
93-
assignment: string | boolean | number | null | object;
104+
assignment: AssignmentVariationValue;
94105
}[] = [];
95106
for (const subject of testCase.subjects) {
96107
const assignment = assignmentFn(
@@ -104,48 +115,88 @@ export function getTestAssignments(
104115
return assignments;
105116
}
106117

107-
export function getTestAssignmentDetails(
108-
testCase: IAssignmentTestCase,
109-
assignmentDetailsFn: (
110-
flagKey: string,
111-
subjectKey: string,
112-
subjectAttributes: Record<string, AttributeType>,
113-
defaultValue: string | number | boolean | object,
114-
) => never,
115-
): {
116-
subject: SubjectTestCase;
117-
assignmentDetails: IAssignmentDetails<string | boolean | number | object>;
118-
}[] {
119-
return testCase.subjects.map((subject) => ({
120-
subject,
121-
assignmentDetails: assignmentDetailsFn(
122-
testCase.flag,
123-
subject.subjectKey,
124-
subject.subjectAttributes,
125-
testCase.defaultValue,
126-
),
127-
}));
128-
}
118+
const configCreatedAt = (
119+
readMockUFCResponse(MOCK_UFC_RESPONSE_FILE) as IUniversalFlagConfigResponse
120+
).createdAt;
121+
const testHelperInstantiationDate = new Date();
129122

130123
export function validateTestAssignments(
131124
assignments: {
132125
subject: SubjectTestCase;
133-
assignment: string | boolean | number | object | null;
126+
assignment: AssignmentVariationValue | IAssignmentDetails<AssignmentVariationValue>;
134127
}[],
135128
flag: string,
129+
withDetails: boolean,
130+
isObfuscated: boolean,
136131
) {
137132
for (const { subject, assignment } of assignments) {
138-
if (!isEqual(assignment, subject.assignment)) {
133+
let assignedVariation = assignment;
134+
let assignmentDetails: IFlagEvaluationDetails | null = null;
135+
if (
136+
withDetails === true &&
137+
typeof assignment === 'object' &&
138+
assignment !== null &&
139+
'variation' in assignment
140+
) {
141+
assignedVariation = assignment.variation;
142+
assignmentDetails = assignment.evaluationDetails;
143+
}
144+
145+
if (!isEqual(assignedVariation, subject.assignment)) {
139146
// More friendly error message
140147
console.error(
141148
`subject ${subject.subjectKey} was assigned ${JSON.stringify(
142-
assignment,
149+
assignedVariation,
143150
undefined,
144151
2,
145152
)} when expected ${JSON.stringify(subject.assignment, undefined, 2)} for flag ${flag}`,
146153
);
147154
}
148155

149-
expect(assignment).toEqual(subject.assignment);
156+
expect(assignedVariation).toEqual(subject.assignment);
157+
158+
if (withDetails) {
159+
if (!assignmentDetails) {
160+
throw new Error('Expected assignmentDetails to be populated');
161+
}
162+
expect(assignmentDetails.environmentName).toBe(subject.evaluationDetails.environmentName);
163+
expect(assignmentDetails.flagEvaluationCode).toBe(
164+
subject.evaluationDetails.flagEvaluationCode,
165+
);
166+
expect(assignmentDetails.flagEvaluationDescription).toBe(
167+
subject.evaluationDetails.flagEvaluationDescription,
168+
);
169+
expect(assignmentDetails.variationKey).toBe(subject.evaluationDetails.variationKey);
170+
// Use toString() to handle comparing JSON
171+
expect(assignmentDetails.variationValue?.toString()).toBe(
172+
subject.evaluationDetails.variationValue?.toString(),
173+
);
174+
expect(assignmentDetails.configPublishedAt).toBe(configCreatedAt);
175+
// cannot do an exact match for configFetchedAt because it will change based on fetch
176+
expect(new Date(assignmentDetails.configFetchedAt).getTime()).toBeGreaterThan(
177+
testHelperInstantiationDate.getTime(),
178+
);
179+
180+
if (!isObfuscated) {
181+
expect(assignmentDetails.matchedRule).toEqual(subject.evaluationDetails.matchedRule);
182+
} else {
183+
// When obfuscated, rules may be one-way hashed (e.g., for ONE_OF checks) so cannot be unobfuscated
184+
// Thus we'll just check that the number of conditions is equal and relay on the unobfuscated
185+
// tests for correctness
186+
expect(assignmentDetails.matchedRule?.conditions || []).toHaveLength(
187+
subject.evaluationDetails.matchedRule?.conditions.length || 0,
188+
);
189+
}
190+
191+
expect(assignmentDetails.matchedAllocation).toEqual(
192+
subject.evaluationDetails.matchedAllocation,
193+
);
194+
expect(assignmentDetails.unmatchedAllocations).toEqual(
195+
subject.evaluationDetails.unmatchedAllocations,
196+
);
197+
expect(assignmentDetails.unevaluatedAllocations).toEqual(
198+
subject.evaluationDetails.unevaluatedAllocations,
199+
);
200+
}
150201
}
151202
}

0 commit comments

Comments
 (0)