From 570984c3a58cb3c7d44493a2baeae8a1a0706593 Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Mon, 21 Apr 2025 09:48:31 -0600 Subject: [PATCH 1/6] combine test code for obsfucated and unobfuscated --- src/client/eppo-client.spec.ts | 113 ++++++++++++--------------------- test/testHelpers.ts | 68 ++++++++++++++++++-- 2 files changed, 102 insertions(+), 79 deletions(-) diff --git a/src/client/eppo-client.spec.ts b/src/client/eppo-client.spec.ts index 3cb4fde..f63a95d 100644 --- a/src/client/eppo-client.spec.ts +++ b/src/client/eppo-client.spec.ts @@ -317,29 +317,37 @@ describe('EppoClient E2E test', () => { }); }); - describe('UFC Shared Test Cases', () => { + describe.each(['Not Obfuscated', 'Obfuscated'])('UFC Shared Test Cases %s', (obfuscationType) => { const testCases = testCasesByFileName(ASSIGNMENT_TEST_DATA_DIR); + const isObfuscated = obfuscationType === 'Obfuscated'; - describe('Not obfuscated', () => { - beforeAll(async () => { - global.fetch = jest.fn(() => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(readMockUFCResponse(MOCK_UFC_RESPONSE_FILE)), - }); - }) as jest.Mock; + beforeAll(async () => { + global.fetch = jest.fn(() => { + return Promise.resolve({ + ok: true, + status: 200, + json: () => + Promise.resolve( + readMockUFCResponse( + isObfuscated ? OBFUSCATED_MOCK_UFC_RESPONSE_FILE : MOCK_UFC_RESPONSE_FILE, + ), + ), + }); + }) as jest.Mock; - await initConfiguration(storage); - }); + await initConfiguration(storage); + }); - afterAll(() => { - jest.restoreAllMocks(); - }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + describe.each(['Scalar', 'With Details'])('%s', (assignmentType) => { + const assignmentWithDetails = assignmentType === 'With Details'; it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => { const { flag, variationType, defaultValue, subjects } = testCases[fileName]; - const client = new EppoClient({ flagConfigurationStore: storage }); + const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated }); client.setIsGracefulFailureMode(false); let assignments: { @@ -347,13 +355,21 @@ describe('EppoClient E2E test', () => { assignment: string | boolean | number | null | object; }[] = []; - const typeAssignmentFunctions = { - [VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client), - [VariationType.NUMERIC]: client.getNumericAssignment.bind(client), - [VariationType.INTEGER]: client.getIntegerAssignment.bind(client), - [VariationType.STRING]: client.getStringAssignment.bind(client), - [VariationType.JSON]: client.getJSONAssignment.bind(client), - }; + const typeAssignmentFunctions = assignmentWithDetails + ? { + [VariationType.BOOLEAN]: client.getBooleanAssignmentDetails.bind(client), + [VariationType.NUMERIC]: client.getNumericAssignmentDetails.bind(client), + [VariationType.INTEGER]: client.getIntegerAssignmentDetails.bind(client), + [VariationType.STRING]: client.getStringAssignmentDetails.bind(client), + [VariationType.JSON]: client.getJSONAssignmentDetails.bind(client), + } + : { + [VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client), + [VariationType.NUMERIC]: client.getNumericAssignment.bind(client), + [VariationType.INTEGER]: client.getIntegerAssignment.bind(client), + [VariationType.STRING]: client.getStringAssignment.bind(client), + [VariationType.JSON]: client.getJSONAssignment.bind(client), + }; const assignmentFn = typeAssignmentFunctions[variationType] as ( flagKey: string, @@ -370,56 +386,7 @@ describe('EppoClient E2E test', () => { assignmentFn, ); - validateTestAssignments(assignments, flag); - }); - }); - - describe('Obfuscated', () => { - beforeAll(async () => { - global.fetch = jest.fn(() => { - return Promise.resolve({ - ok: true, - status: 200, - json: () => Promise.resolve(readMockUFCResponse(OBFUSCATED_MOCK_UFC_RESPONSE_FILE)), - }); - }) as jest.Mock; - - await initConfiguration(storage); - }); - - afterAll(() => { - jest.restoreAllMocks(); - }); - - it.each(Object.keys(testCases))('test variation assignment splits - %s', async (fileName) => { - const { flag, variationType, defaultValue, subjects } = testCases[fileName]; - const client = new EppoClient({ flagConfigurationStore: storage, isObfuscated: true }); - client.setIsGracefulFailureMode(false); - - const typeAssignmentFunctions = { - [VariationType.BOOLEAN]: client.getBooleanAssignment.bind(client), - [VariationType.NUMERIC]: client.getNumericAssignment.bind(client), - [VariationType.INTEGER]: client.getIntegerAssignment.bind(client), - [VariationType.STRING]: client.getStringAssignment.bind(client), - [VariationType.JSON]: client.getJSONAssignment.bind(client), - }; - - const assignmentFn = typeAssignmentFunctions[variationType] as ( - flagKey: string, - subjectKey: string, - subjectAttributes: Record, - defaultValue: boolean | string | number | object, - ) => never; - if (!assignmentFn) { - throw new Error(`Unknown variation type: ${variationType}`); - } - - const assignments = getTestAssignments( - { flag, variationType, defaultValue, subjects }, - assignmentFn, - ); - - validateTestAssignments(assignments, flag); + validateTestAssignments(assignments, flag, assignmentWithDetails); }); }); }); diff --git a/test/testHelpers.ts b/test/testHelpers.ts index 24b2c49..6320929 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import { isEqual } from 'lodash'; -import { AttributeType, ContextAttributes, IAssignmentDetails, VariationType } from '../src'; +import { AttributeType, ContextAttributes, IAssignmentDetails, Variation, VariationType } from '../src'; import { IFlagEvaluationDetails } from '../src/flag-evaluation-details-builder'; import { IBanditParametersResponse, IUniversalFlagConfigResponse } from '../src/http-client'; @@ -87,7 +87,16 @@ export function getTestAssignments( subjectAttributes: Record, defaultValue: string | number | boolean | object, ) => never, -): { subject: SubjectTestCase; assignment: string | boolean | number | null | object }[] { +): { + subject: SubjectTestCase; + assignment: + | string + | boolean + | number + | null + | object + | IAssignmentDetails; +}[] { const assignments: { subject: SubjectTestCase; assignment: string | boolean | number | null | object; @@ -130,22 +139,69 @@ export function getTestAssignmentDetails( export function validateTestAssignments( assignments: { subject: SubjectTestCase; - assignment: string | boolean | number | object | null; + assignment: + | string + | boolean + | number + | object + | null + | IAssignmentDetails; }[], flag: string, + withDetails: boolean, ) { for (const { subject, assignment } of assignments) { - if (!isEqual(assignment, subject.assignment)) { + let assignedVariation = assignment; + let assignmentDetails: IFlagEvaluationDetails | null = null; + if ( + withDetails === true && + typeof assignment === 'object' && + assignment !== null && + 'variation' in assignment + ) { + assignedVariation = assignment.variation; + assignmentDetails = assignment.evaluationDetails; + } + + if (!isEqual(assignedVariation, subject.assignment)) { // More friendly error message console.error( `subject ${subject.subjectKey} was assigned ${JSON.stringify( - assignment, + assignedVariation, undefined, 2, )} when expected ${JSON.stringify(subject.assignment, undefined, 2)} for flag ${flag}`, ); } - expect(assignment).toEqual(subject.assignment); + expect(assignedVariation).toEqual(subject.assignment); + + if (withDetails && assignmentDetails) { + expect(assignmentDetails.environmentName).toBe(subject.evaluationDetails.environmentName); + expect(assignmentDetails.flagEvaluationCode).toBe( + subject.evaluationDetails.flagEvaluationCode, + ); + expect(assignmentDetails.flagEvaluationDescription).toBe( + subject.evaluationDetails.flagEvaluationDescription, + ); + expect(assignmentDetails.variationKey).toBe(subject.evaluationDetails.variationKey); + // Use toString() to handle comparing JSON + expect(assignmentDetails.variationValue?.toString()).toBe( + subject.evaluationDetails.variationValue?.toString(), + ); + // TODO: below needs to be fixed + //expect(assignmentDetails.configFetchedAt).toBe(subject.evaluationDetails.configFetchedAt); + //expect(assignmentDetails.configPublishedAt).toBe(subject.evaluationDetails.configPublishedAt); + expect(assignmentDetails.matchedRule).toEqual(subject.evaluationDetails.matchedRule); + expect(assignmentDetails.matchedAllocation).toEqual( + subject.evaluationDetails.matchedAllocation, + ); + expect(assignmentDetails.unmatchedAllocations).toEqual( + subject.evaluationDetails.unmatchedAllocations, + ); + expect(assignmentDetails.unevaluatedAllocations).toEqual( + subject.evaluationDetails.unevaluatedAllocations, + ); + } } } From 19de7c35e85f7327d52cce70198e1ccc366be961 Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Mon, 21 Apr 2025 12:19:35 -0600 Subject: [PATCH 2/6] handle looking at obfuscated rules for details --- src/client/eppo-client.spec.ts | 2 +- src/evaluator.ts | 1 - test/testHelpers.ts | 37 ++++++++++++---------------------- 3 files changed, 14 insertions(+), 26 deletions(-) diff --git a/src/client/eppo-client.spec.ts b/src/client/eppo-client.spec.ts index f63a95d..ec4e8d0 100644 --- a/src/client/eppo-client.spec.ts +++ b/src/client/eppo-client.spec.ts @@ -386,7 +386,7 @@ describe('EppoClient E2E test', () => { assignmentFn, ); - validateTestAssignments(assignments, flag, assignmentWithDetails); + validateTestAssignments(assignments, flag, assignmentWithDetails, isObfuscated); }); }); }); diff --git a/src/evaluator.ts b/src/evaluator.ts index 98ab3fd..ce7e381 100644 --- a/src/evaluator.ts +++ b/src/evaluator.ts @@ -144,7 +144,6 @@ export class Evaluator { configDetails.configFormat, ); } catch (err: any) { - console.error('>>>>', err); const flagEvaluationDetails = flagEvaluationDetailsBuilder.gracefulBuild( 'ASSIGNMENT_ERROR', `Assignment Error: ${err.message}`, diff --git a/test/testHelpers.ts b/test/testHelpers.ts index 6320929..7917383 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -5,6 +5,7 @@ import { isEqual } from 'lodash'; import { AttributeType, ContextAttributes, IAssignmentDetails, Variation, VariationType } from '../src'; import { IFlagEvaluationDetails } from '../src/flag-evaluation-details-builder'; import { IBanditParametersResponse, IUniversalFlagConfigResponse } from '../src/http-client'; +import { getMD5Hash } from '../src/obfuscation'; export const TEST_DATA_DIR = './test/data/ufc/'; export const ASSIGNMENT_TEST_DATA_DIR = TEST_DATA_DIR + 'tests/'; @@ -113,29 +114,6 @@ export function getTestAssignments( return assignments; } -export function getTestAssignmentDetails( - testCase: IAssignmentTestCase, - assignmentDetailsFn: ( - flagKey: string, - subjectKey: string, - subjectAttributes: Record, - defaultValue: string | number | boolean | object, - ) => never, -): { - subject: SubjectTestCase; - assignmentDetails: IAssignmentDetails; -}[] { - return testCase.subjects.map((subject) => ({ - subject, - assignmentDetails: assignmentDetailsFn( - testCase.flag, - subject.subjectKey, - subject.subjectAttributes, - testCase.defaultValue, - ), - })); -} - export function validateTestAssignments( assignments: { subject: SubjectTestCase; @@ -149,6 +127,7 @@ export function validateTestAssignments( }[], flag: string, withDetails: boolean, + isObfuscated: boolean, ) { for (const { subject, assignment } of assignments) { let assignedVariation = assignment; @@ -192,7 +171,17 @@ export function validateTestAssignments( // TODO: below needs to be fixed //expect(assignmentDetails.configFetchedAt).toBe(subject.evaluationDetails.configFetchedAt); //expect(assignmentDetails.configPublishedAt).toBe(subject.evaluationDetails.configPublishedAt); - expect(assignmentDetails.matchedRule).toEqual(subject.evaluationDetails.matchedRule); + + if (!isObfuscated) { + expect(assignmentDetails.matchedRule).toEqual(subject.evaluationDetails.matchedRule); + } else { + // When obfuscated, rules may be one-way hashed (e.g., for ONE_OF checks) so cannot be unobfuscated + // Thus we'll just check that the number of conditions is equal + expect(assignmentDetails.matchedRule?.conditions || []).toHaveLength( + subject.evaluationDetails.matchedRule?.conditions.length || 0, + ); + } + expect(assignmentDetails.matchedAllocation).toEqual( subject.evaluationDetails.matchedAllocation, ); From 411b818988fe88fc4a7cf6a2057af58835313a2f Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Mon, 21 Apr 2025 12:57:17 -0600 Subject: [PATCH 3/6] tests passing --- src/http-client.ts | 1 + test/testHelpers.ts | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/http-client.ts b/src/http-client.ts index 063ecb3..eef44c4 100644 --- a/src/http-client.ts +++ b/src/http-client.ts @@ -43,6 +43,7 @@ export interface IUniversalFlagConfigResponse { } export interface IBanditParametersResponse { + createdAt: string; // ISO formatted string bandits: Record; } diff --git a/test/testHelpers.ts b/test/testHelpers.ts index 7917383..9af3669 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -2,10 +2,9 @@ import * as fs from 'fs'; import { isEqual } from 'lodash'; -import { AttributeType, ContextAttributes, IAssignmentDetails, Variation, VariationType } from '../src'; +import { AttributeType, ContextAttributes, IAssignmentDetails, VariationType } from '../src'; import { IFlagEvaluationDetails } from '../src/flag-evaluation-details-builder'; import { IBanditParametersResponse, IUniversalFlagConfigResponse } from '../src/http-client'; -import { getMD5Hash } from '../src/obfuscation'; export const TEST_DATA_DIR = './test/data/ufc/'; export const ASSIGNMENT_TEST_DATA_DIR = TEST_DATA_DIR + 'tests/'; @@ -114,6 +113,9 @@ export function getTestAssignments( return assignments; } +const configCreatedAt = readMockUFCResponse(MOCK_UFC_RESPONSE_FILE).createdAt; +const testHelperInstantiationDate = new Date(); + export function validateTestAssignments( assignments: { subject: SubjectTestCase; @@ -168,15 +170,18 @@ export function validateTestAssignments( expect(assignmentDetails.variationValue?.toString()).toBe( subject.evaluationDetails.variationValue?.toString(), ); - // TODO: below needs to be fixed - //expect(assignmentDetails.configFetchedAt).toBe(subject.evaluationDetails.configFetchedAt); - //expect(assignmentDetails.configPublishedAt).toBe(subject.evaluationDetails.configPublishedAt); + expect(assignmentDetails.configPublishedAt).toBe(configCreatedAt); + // cannot do an exact match for configFetchedAt because it will change based on fetch + expect(new Date(assignmentDetails.configFetchedAt).getTime()).toBeGreaterThan( + testHelperInstantiationDate.getTime(), + ); if (!isObfuscated) { expect(assignmentDetails.matchedRule).toEqual(subject.evaluationDetails.matchedRule); } else { // When obfuscated, rules may be one-way hashed (e.g., for ONE_OF checks) so cannot be unobfuscated - // Thus we'll just check that the number of conditions is equal + // Thus we'll just check that the number of conditions is equal and relay on the unobfuscated + // tests for correctness expect(assignmentDetails.matchedRule?.conditions || []).toHaveLength( subject.evaluationDetails.matchedRule?.conditions.length || 0, ); From 780c73bbc552c6bd0f2ed54c955302bcf81d6761 Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Mon, 21 Apr 2025 13:04:44 -0600 Subject: [PATCH 4/6] bandit config uses updatedAt --- src/http-client.ts | 2 +- test/testHelpers.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/http-client.ts b/src/http-client.ts index eef44c4..7d13b81 100644 --- a/src/http-client.ts +++ b/src/http-client.ts @@ -43,7 +43,7 @@ export interface IUniversalFlagConfigResponse { } export interface IBanditParametersResponse { - createdAt: string; // ISO formatted string + updatedAt: string; // ISO formatted string bandits: Record; } diff --git a/test/testHelpers.ts b/test/testHelpers.ts index 9af3669..d6129cf 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -113,7 +113,9 @@ export function getTestAssignments( return assignments; } -const configCreatedAt = readMockUFCResponse(MOCK_UFC_RESPONSE_FILE).createdAt; +const configCreatedAt = ( + readMockUFCResponse(MOCK_UFC_RESPONSE_FILE) as IUniversalFlagConfigResponse +).createdAt; const testHelperInstantiationDate = new Date(); export function validateTestAssignments( From d8ed352789adddd0b2b98101ab4919b956c083bd Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Mon, 21 Apr 2025 13:08:10 -0600 Subject: [PATCH 5/6] throw if no details when expected --- test/testHelpers.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/testHelpers.ts b/test/testHelpers.ts index d6129cf..2b547aa 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -159,7 +159,10 @@ export function validateTestAssignments( expect(assignedVariation).toEqual(subject.assignment); - if (withDetails && assignmentDetails) { + if (withDetails) { + if (!assignmentDetails) { + throw new Error('Expected assignmentDetails to be populated'); + } expect(assignmentDetails.environmentName).toBe(subject.evaluationDetails.environmentName); expect(assignmentDetails.flagEvaluationCode).toBe( subject.evaluationDetails.flagEvaluationCode, From e8e983e176a965a1116f56aac28d5eaba78d9132 Mon Sep 17 00:00:00 2001 From: Aaron Silverman Date: Tue, 22 Apr 2025 09:55:03 -0600 Subject: [PATCH 6/6] assignment value type --- .../eppo-client-assignment-details.spec.ts | 5 +-- src/client/eppo-client.spec.ts | 13 ++++--- test/testHelpers.ts | 36 +++++++++---------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/client/eppo-client-assignment-details.spec.ts b/src/client/eppo-client-assignment-details.spec.ts index b0eb09b..6dfecd3 100644 --- a/src/client/eppo-client-assignment-details.spec.ts +++ b/src/client/eppo-client-assignment-details.spec.ts @@ -1,6 +1,7 @@ import * as fs from 'fs'; import { + AssignmentVariationValue, IAssignmentTestCase, MOCK_UFC_RESPONSE_FILE, readMockUFCResponse, @@ -328,8 +329,8 @@ describe('EppoClient get*AssignmentDetails', () => { flagKey: string, subjectKey: string, subjectAttributes: Record, - defaultValue: boolean | string | number | object, - ) => IAssignmentDetails; + defaultValue: AssignmentVariationValue, + ) => IAssignmentDetails; if (!assignmentFn) { throw new Error(`Unknown variation type: ${variationType}`); } diff --git a/src/client/eppo-client.spec.ts b/src/client/eppo-client.spec.ts index ec4e8d0..d4e8266 100644 --- a/src/client/eppo-client.spec.ts +++ b/src/client/eppo-client.spec.ts @@ -4,6 +4,7 @@ import * as td from 'testdouble'; import { ASSIGNMENT_TEST_DATA_DIR, + AssignmentVariationValue, getTestAssignments, IAssignmentTestCase, MOCK_UFC_RESPONSE_FILE, @@ -28,7 +29,11 @@ import { Flag, ObfuscatedFlag, VariationType, FormatEnum, Variation } from '../i import { getMD5Hash } from '../obfuscation'; import { AttributeType } from '../types'; -import EppoClient, { checkTypeMatch, FlagConfigurationRequestParameters } from './eppo-client'; +import EppoClient, { + checkTypeMatch, + FlagConfigurationRequestParameters, + IAssignmentDetails, +} from './eppo-client'; import { initConfiguration } from './test-utils'; // Use a known salt to produce deterministic hashes @@ -352,7 +357,7 @@ describe('EppoClient E2E test', () => { let assignments: { subject: SubjectTestCase; - assignment: string | boolean | number | null | object; + assignment: AssignmentVariationValue; }[] = []; const typeAssignmentFunctions = assignmentWithDetails @@ -375,8 +380,8 @@ describe('EppoClient E2E test', () => { flagKey: string, subjectKey: string, subjectAttributes: Record, - defaultValue: boolean | string | number | object, - ) => never; + defaultValue: AssignmentVariationValue, + ) => AssignmentVariationValue | IAssignmentDetails; if (!assignmentFn) { throw new Error(`Unknown variation type: ${variationType}`); } diff --git a/test/testHelpers.ts b/test/testHelpers.ts index 2b547aa..dadbc74 100644 --- a/test/testHelpers.ts +++ b/test/testHelpers.ts @@ -2,7 +2,13 @@ import * as fs from 'fs'; import { isEqual } from 'lodash'; -import { AttributeType, ContextAttributes, IAssignmentDetails, VariationType } from '../src'; +import { + AttributeType, + ContextAttributes, + IAssignmentDetails, + Variation, + VariationType, +} from '../src'; import { IFlagEvaluationDetails } from '../src/flag-evaluation-details-builder'; import { IBanditParametersResponse, IUniversalFlagConfigResponse } from '../src/http-client'; @@ -20,17 +26,19 @@ const MOCK_PRECOMPUTED_FILENAME = 'precomputed-v1'; export const MOCK_PRECOMPUTED_WIRE_FILE = `${MOCK_PRECOMPUTED_FILENAME}.json`; export const MOCK_DEOBFUSCATED_PRECOMPUTED_RESPONSE_FILE = `${MOCK_PRECOMPUTED_FILENAME}-deobfuscated.json`; +export type AssignmentVariationValue = Variation['value'] | object; + export interface SubjectTestCase { subjectKey: string; subjectAttributes: Record; - assignment: string | number | boolean | object; + assignment: AssignmentVariationValue; evaluationDetails: IFlagEvaluationDetails; } export interface IAssignmentTestCase { flag: string; variationType: VariationType; - defaultValue: string | number | boolean | object; + defaultValue: AssignmentVariationValue; subjects: SubjectTestCase[]; } @@ -85,21 +93,15 @@ export function getTestAssignments( flagKey: string, subjectKey: string, subjectAttributes: Record, - defaultValue: string | number | boolean | object, - ) => never, + defaultValue: AssignmentVariationValue, + ) => AssignmentVariationValue | IAssignmentDetails, ): { subject: SubjectTestCase; - assignment: - | string - | boolean - | number - | null - | object - | IAssignmentDetails; + assignment: AssignmentVariationValue | IAssignmentDetails; }[] { const assignments: { subject: SubjectTestCase; - assignment: string | boolean | number | null | object; + assignment: AssignmentVariationValue; }[] = []; for (const subject of testCase.subjects) { const assignment = assignmentFn( @@ -121,13 +123,7 @@ const testHelperInstantiationDate = new Date(); export function validateTestAssignments( assignments: { subject: SubjectTestCase; - assignment: - | string - | boolean - | number - | object - | null - | IAssignmentDetails; + assignment: AssignmentVariationValue | IAssignmentDetails; }[], flag: string, withDetails: boolean,