Use new SDK test data (#8)

* Use new SDK test data

* fix directory path

* fix linter error

* use separate directory for test data

Author: ploomiz

.github/workflows/lint-test-sdk.yml

@@ -17,6 +17,8 @@ jobs:
         with:
           path: './node_modules'
           key: ${{ runner.os }}-root-node-modules-${{ hashFiles('./yarn.lock') }}
+      - name: 'Set up GCP SDK for downloading test data'
+        uses: 'google-github-actions/setup-gcloud@v0'
       - name: Install SDK dependencies
         run: yarn --frozen-lockfile
         working-directory: ./

.gitignore

@@ -7,4 +7,4 @@ temp
 .env
 yarn-error.log
 
-test/assignmentTestData
+test/data

Makefile

@@ -0,0 +1,42 @@
+# Make settings - @see https://tech.davis-hansson.com/p/make/
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+
+# Log levels
+DEBUG := $(shell printf "\e[2D\e[35m")
+INFO  := $(shell printf "\e[2D\e[36m🔵 ")
+OK    := $(shell printf "\e[2D\e[32m🟢 ")
+WARN  := $(shell printf "\e[2D\e[33m🟡 ")
+ERROR := $(shell printf "\e[2D\e[31m🔴 ")
+END   := $(shell printf "\e[0m")
+
+.PHONY: default
+default: help
+
+## help - Print help message.
+.PHONY: help
+help: Makefile
+	@echo "usage: make <target>"
+	@sed -n 's/^##//p' $<
+
+## test-data
+testDataDir := test/data/
+.PHONY: test-data
+test-data:
+	rm -rf $(testDataDir)
+	mkdir -p $(testDataDir)
+	gsutil cp gs://sdk-test-data/rac-experiments.json $(testDataDir)
+	gsutil cp -r gs://sdk-test-data/assignment-v2 $(testDataDir)
+
+## prepare
+.PHONY: prepare
+prepare: test-data
+	 husky install
+	 rm -rf dist/
+	 tsc
+	 webpack
+	 api-extractor run --local

jest.config.js

@@ -9,7 +9,6 @@ module.exports = {
   transform: {
     '^.+\\.(t|j)s$': 'ts-jest',
   },
-  globalSetup: './test/globalSetup.ts',
   collectCoverageFrom: ['**/*.(t|j)s'],
   coverageDirectory: 'coverage/',
   testEnvironment: 'node',

package.json

@@ -11,7 +11,7 @@
     "lint": "eslint '**/*.{ts,tsx}' '**/*.d.{ts,tsx}' --cache",
     "lint:fix": "eslint --fix '**/*.{ts,tsx}' --cache",
     "lint:fix-pre-commit": "eslint -c .eslintrc.pre-commit.js --fix '**/*.{ts,tsx}' --no-eslintrc --cache",
-    "prepare": "husky install && rm -rf dist/ && tsc && webpack && api-extractor run --local",
+    "prepare": "make prepare",
     "pre-commit": "lint-staged && tsc && yarn docs",
     "typecheck": "tsc",
     "test": "yarn test:unit",
@@ -30,7 +30,6 @@
   },
   "homepage": "https://github.com/Eppo-exp/js-client-sdk#readme",
   "devDependencies": {
-    "@google-cloud/storage": "^6.1.0",
     "@microsoft/api-documenter": "^7.17.17",
     "@microsoft/api-extractor": "^7.25.0",
     "@types/jest": "^28.1.1",

src/eppo-client.spec.ts

@@ -4,13 +4,15 @@
 import * as td from 'testdouble';
 import mock from 'xhr-mock';
 
-import { IAssignmentTestCase, readAssignmentTestData } from '../test/testHelpers';
+import {
+  IAssignmentTestCase,
+  readAssignmentTestData,
+  readMockRacResponse,
+} from '../test/testHelpers';
 
 import { IAssignmentLogger } from './assignment-logger';
 import { MAX_EVENT_QUEUE_SIZE } from './constants';
 import EppoClient from './eppo-client';
-import { IExperimentConfiguration } from './experiment/experiment-configuration';
-import { IVariation } from './experiment/variation';
 import { EppoLocalStorage } from './local-storage';
 import { OperatorType } from './rule';
 import { EppoSessionStorage } from './session-storage';
@@ -26,20 +28,8 @@ describe('EppoClient E2E test', () => {
     window.sessionStorage.clear();
     mock.setup();
     mock.get(/randomized_assignment\/config*/, (_req, res) => {
-      const testCases: IAssignmentTestCase[] = readAssignmentTestData();
-      const assignmentConfig: Record<string, IExperimentConfiguration> = {};
-      testCases.forEach(({ experiment, percentExposure, variations }) => {
-        assignmentConfig[experiment] = {
-          name: experiment,
-          percentExposure,
-          enabled: true,
-          subjectShards: 10000,
-          variations,
-          overrides: {},
-          rules: [],
-        };
-      });
-      return res.status(200).body(JSON.stringify({ experiments: assignmentConfig }));
+      const rac = readMockRacResponse();
+      return res.status(200).body(JSON.stringify(rac));
     });
     const preloadedConfig = {
       name: preloadedConfigExperiment,
@@ -142,22 +132,16 @@ describe('EppoClient E2E test', () => {
     it.each(readAssignmentTestData())(
       'test variation assignment splits',
       async ({
-        variations,
         experiment,
-        percentExposure,
         subjects,
+        subjectsWithAttributes,
         expectedAssignments,
       }: IAssignmentTestCase) => {
         console.log(`---- Test Case for ${experiment} Experiment ----`);
-        const assignments = getAssignments(subjects, experiment);
-        // verify the assingments don't change across test runs (deterministic)
+        const assignments = subjectsWithAttributes
+          ? getAssignmentsWithSubjectAttributes(subjectsWithAttributes, experiment)
+          : getAssignments(subjects, experiment);
         expect(assignments).toEqual(expectedAssignments);
-        const expectedVariationSplitPercentage = percentExposure / variations.length;
-        const unassignedCount = assignments.filter((assignment) => assignment == null).length;
-        expectToBeCloseToPercentage(unassignedCount / assignments.length, 1 - percentExposure);
-        variations.forEach((variation) => {
-          validateAssignmentCounts(assignments, expectedVariationSplitPercentage, variation);
-        });
       },
     );
   });
@@ -286,30 +270,24 @@ describe('EppoClient E2E test', () => {
     expect(assignment).toEqual('control');
   });
 
-  function validateAssignmentCounts(
-    assignments: string[],
-    expectedPercentage: number,
-    variation: IVariation,
-  ) {
-    const assignedCount = assignments.filter((assignment) => assignment === variation.name).length;
-    console.log(
-      `Expect variation ${variation.name} percentage of ${
-        assignedCount / assignments.length
-      } to be close to ${expectedPercentage}`,
-    );
-    expectToBeCloseToPercentage(assignedCount / assignments.length, expectedPercentage);
-  }
-
-  // expect assignment count to be within 5 percentage points of the expected count (because the hash output is random)
-  function expectToBeCloseToPercentage(percentage: number, expectedPercentage: number) {
-    expect(percentage).toBeGreaterThanOrEqual(expectedPercentage - 0.05);
-    expect(percentage).toBeLessThanOrEqual(expectedPercentage + 0.05);
-  }
-
   function getAssignments(subjects: string[], experiment: string): string[] {
     const client = getInstance();
     return subjects.map((subjectKey) => {
       return client.getAssignment(subjectKey, experiment);
     });
   }
+
+  function getAssignmentsWithSubjectAttributes(
+    subjectsWithAttributes: {
+      subjectKey: string;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      subjectAttributes: Record<string, any>;
+    }[],
+    experiment: string,
+  ): string[] {
+    const client = getInstance();
+    return subjectsWithAttributes.map((subject) => {
+      return client.getAssignment(subject.subjectKey, experiment, subject.subjectAttributes);
+    });
+  }
 });

test/globalSetup.ts

@@ -1,23 +1,31 @@
 import * as fs from 'fs';
 
+import { IExperimentConfiguration } from '../src/experiment/experiment-configuration';
 import { IVariation } from '../src/experiment/variation';
 
-export const TEST_DATA_DIR = './test/assignmentTestData/';
+export const TEST_DATA_DIR = './test/data/';
+export const ASSIGNMENT_TEST_DATA_DIR = TEST_DATA_DIR + 'assignment-v2/';
+export const MOCK_RAC_RESPONSE_FILE = 'rac-experiments.json';
 
 export interface IAssignmentTestCase {
   experiment: string;
   percentExposure: number;
   variations: IVariation[];
   subjects: string[];
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  subjectsWithAttributes: { subjectKey: string; subjectAttributes: Record<string, any> }[];
   expectedAssignments: string[];
 }
 
+export function readMockRacResponse(): Record<string, IExperimentConfiguration> {
+  return JSON.parse(fs.readFileSync(TEST_DATA_DIR + MOCK_RAC_RESPONSE_FILE, 'utf-8'));
+}
+
 export function readAssignmentTestData(): IAssignmentTestCase[] {
-  const testDataDir = './test/assignmentTestData/';
   const testCaseData: IAssignmentTestCase[] = [];
-  const testCaseFiles = fs.readdirSync(testDataDir);
+  const testCaseFiles = fs.readdirSync(ASSIGNMENT_TEST_DATA_DIR);
   testCaseFiles.forEach((file) => {
-    const testCase = JSON.parse(fs.readFileSync(testDataDir + file, 'utf8'));
+    const testCase = JSON.parse(fs.readFileSync(ASSIGNMENT_TEST_DATA_DIR + file, 'utf8'));
     testCaseData.push(testCase);
   });
   return testCaseData;