Merge pull request #893 from rasmi/balanced-strategy

Add BALANCED_ASSIGNMENT variable config

Author: vivtsai

frontend/src/components/experiment_builder/variable_editor.ts

@@ -44,9 +44,10 @@ import {
   VariableConfigType,
   VariableType,
   RandomPermutationVariableConfig,
-  StaticVariableConfig,
-  createStaticVariableConfig,
-  VariableScope,
+  BalancedAssignmentVariableConfig,
+  BalanceStrategy,
+  BalanceAcross,
+  createBalancedAssignmentVariableConfig,
   createShuffleConfig,
 } from '@deliberation-lab/utils';
 
@@ -196,17 +197,23 @@ const PolicySchema = VariableType.object({
   ),
 });
 
-// Create a static variable config with the complete policy object
-const POLICY_STATIC_CONFIG: StaticVariableConfig = createStaticVariableConfig({
-  id: 'policy-static-config',
-  scope: VariableScope.EXPERIMENT,
-  definition: {
-    name: 'policy',
-    description: 'Policy debate topic',
-    schema: PolicySchema,
-  },
-  value: JSON.stringify(EXAMPLE_POLICY_A),
-});
+// Create a balanced assignment config for multi-policy experiments
+// Each participant is randomly assigned one policy with even distribution
+const POLICY_BALANCED_ASSIGNMENT_CONFIG: BalancedAssignmentVariableConfig =
+  createBalancedAssignmentVariableConfig({
+    id: 'policy-balanced-assignment',
+    definition: {
+      name: 'policy',
+      description: 'Randomly assigned policy for balanced conditions',
+      schema: PolicySchema,
+    },
+    values: [
+      JSON.stringify(EXAMPLE_POLICY_A),
+      JSON.stringify(EXAMPLE_POLICY_B),
+    ],
+    balanceStrategy: BalanceStrategy.ROUND_ROBIN,
+    balanceAcross: BalanceAcross.EXPERIMENT,
+  });
 
 const NO_SHUFFLE: ShuffleConfig = createShuffleConfig({
   shuffle: false,
@@ -222,11 +229,11 @@ const PARTICIPANT_SHUFFLE: ShuffleConfig = createShuffleConfig({
 // ****************************************************************************
 export function getPolicyExperimentTemplate(): ExperimentTemplate {
   const stageConfigs = getPolicyStageConfigs();
-  const variableTemplates: VariableConfig[] = [POLICY_STATIC_CONFIG];
+  const variableConfigs: VariableConfig[] = [POLICY_BALANCED_ASSIGNMENT_CONFIG];
   return createExperimentTemplate({
     experiment: createExperimentConfig(stageConfigs, {
       metadata: POLICY_METADATA,
-      variableConfigs: variableTemplates,
+      variableConfigs,
     }),
     stageConfigs,
     agentMediators: POLICY_MEDIATOR_AGENTS,

frontend/src/shared/templates/policy.ts

@@ -10,15 +10,15 @@
     "deploy": "firebase deploy --only functions",
     "logs": "firebase functions:log",
     "test": "npm run test:unit && npm run test:firestore",
-    "test:firestore": "firebase -c firebase-test.json emulators:exec --only firestore,functions --project=demo-deliberate-lab \"npx jest $npm_package_config_firestore_tests\"",
+    "test:firestore": "firebase -c firebase-test.json emulators:exec --only firestore,functions --project=demo-deliberate-lab \"npx jest --runInBand $npm_package_config_firestore_tests\"",
     "test:unit": "npx jest --testPathIgnorePatterns=$npm_package_config_firestore_tests",
     "typecheck": "tsc --noEmit"
   },
   "engines": {
     "node": "22"
   },
   "config": {
-    "firestore_tests": "src/log.utils.test.ts src/dl_api/experiments.dl_api.integration.test.ts"
+    "firestore_tests": "src/log.utils.test.ts src/dl_api/experiments.dl_api.integration.test.ts src/variables.utils.test.ts"
   },
   "main": "lib/index.js",
   "dependencies": {

functions/package.json

@@ -4,9 +4,9 @@ import {
   MediatorProfileExtended,
   StageConfig,
   createPublicDataFromStageConfigs,
-  generateVariablesForScope,
   VariableScope,
 } from '@deliberation-lab/utils';
+import {generateVariablesForScope} from './variables.utils';
 import {createMediatorsForCohort} from './mediator.utils';
 import {app} from './app';
 
@@ -67,7 +67,7 @@ export async function createCohortInternal(
   }
 
   // Add variable values at the cohort level
-  cohortConfig.variableMap = generateVariablesForScope(
+  cohortConfig.variableMap = await generateVariablesForScope(
     experiment.variableConfigs ?? [],
     {scope: VariableScope.COHORT, experimentId, cohortId: cohortConfig.id},
   );

functions/src/cohort.utils.ts

@@ -3,13 +3,13 @@
  * Handles generation, hashing, storage, and verification of API keys
  */
 
-import * as admin from 'firebase-admin';
 import {randomBytes, scrypt, createHash, timingSafeEqual} from 'crypto';
 import {promisify} from 'util';
 import {
   DeliberateLabAPIKeyPermission,
   DeliberateLabAPIKeyData,
 } from '@deliberation-lab/utils';
+import {app} from '../app';
 
 const scryptAsync = promisify(scrypt);
 
@@ -67,7 +67,6 @@ export async function createDeliberateLabAPIKey(
     DeliberateLabAPIKeyPermission.WRITE,
   ],
 ): Promise<{apiKey: string; keyId: string}> {
-  const app = admin.app();
   const firestore = app.firestore();
 
   // Generate the API key
@@ -104,7 +103,6 @@ export async function createDeliberateLabAPIKey(
 export async function verifyDeliberateLabAPIKey(
   apiKey: string,
 ): Promise<{valid: boolean; data?: DeliberateLabAPIKeyData}> {
-  const app = admin.app();
   const firestore = app.firestore();
 
   // Get key ID to look up the document
@@ -151,7 +149,6 @@ export async function revokeDeliberateLabAPIKey(
   keyId: string,
   experimenterId: string,
 ): Promise<boolean> {
-  const app = admin.app();
   const firestore = app.firestore();
 
   const doc = await firestore
@@ -195,7 +192,6 @@ export async function listDeliberateLabAPIKeys(experimenterId: string): Promise<
     permissions: DeliberateLabAPIKeyPermission[];
   }>
 > {
-  const app = admin.app();
   const firestore = app.firestore();
 
   const snapshot = await firestore

functions/src/dl_api/dl_api_key.utils.ts

@@ -4,22 +4,14 @@
  * These tests verify that experiments created via the REST API are equivalent
  * to experiments created using the traditional template system.
  *
- * This test assumes that a Firestore emulator is running.
- * Run using: npm run test:firestore experiments.api.integration.test.ts
- * Or: firebase emulators:exec --only firestore "npx jest experiments.api.integration.test.ts"
+ * This test requires a Firestore emulator running. Run via:
+ * npm run test:firestore
  */
 
-// Don't override GCLOUD_PROJECT - let firebase emulators:exec set it
-// If not set, use a demo project ID that the emulator will accept
-if (!process.env.GCLOUD_PROJECT) {
-  process.env.GCLOUD_PROJECT = 'demo-deliberate-lab';
-}
-
 import {
   initializeTestEnvironment,
   RulesTestEnvironment,
 } from '@firebase/rules-unit-testing';
-import * as admin from 'firebase-admin';
 import {
   createExperimentConfig,
   StageConfig,
@@ -53,8 +45,7 @@ describe('API Experiment Creation Integration Tests', () => {
   const createdExperimentIds: string[] = [];
 
   beforeAll(async () => {
-    // Use the same project ID that the emulator will use
-    const projectId = process.env.GCLOUD_PROJECT || 'demo-deliberate-lab';
+    const projectId = 'demo-deliberate-lab';
 
     testEnv = await initializeTestEnvironment({
       projectId,
@@ -68,13 +59,6 @@ describe('API Experiment Creation Integration Tests', () => {
     firestore = testEnv.unauthenticatedContext().firestore();
     firestore.settings({ignoreUndefinedProperties: true, merge: true});
 
-    // Initialize Firebase Admin SDK (will use emulator via FIRESTORE_EMULATOR_HOST environment variable)
-    if (!admin.apps.length) {
-      admin.initializeApp({
-        projectId,
-      });
-    }
-
     // Create test API key (this will be stored in the emulator)
     console.log('Creating API key...');
     const {apiKey} = await createDeliberateLabAPIKey(

functions/src/dl_api/experiments.dl_api.integration.test.ts

@@ -13,9 +13,9 @@ import {
   StageConfig,
   createExperimentConfig,
   createExperimentTemplate,
-  generateVariablesForScope,
   VariableScope,
 } from '@deliberation-lab/utils';
+import {generateVariablesForScope} from './variables.utils';
 import {getExperimentDownload} from './data';
 
 import {onCall, HttpsError} from 'firebase-functions/v2/https';
@@ -71,7 +71,7 @@ export const writeExperiment = onCall(async (request) => {
     }
 
     // Add variable values at the experiment level
-    experimentConfig.variableMap = generateVariablesForScope(
+    experimentConfig.variableMap = await generateVariablesForScope(
       experimentConfig.variableConfigs ?? [],
       {scope: VariableScope.EXPERIMENT, experimentId: experimentConfig.id},
     );

functions/src/experiment.endpoints.ts

@@ -15,7 +15,6 @@ import {
   StageConfig,
   TransferStageConfig,
   createParticipantProfileExtended,
-  generateVariablesForScope,
   setProfile,
   VariableScope,
 } from '@deliberation-lab/utils';
@@ -24,6 +23,7 @@ import {
   updateParticipantNextStage,
   handleAutomaticTransfer,
 } from './participant.utils';
+import {generateVariablesForScope} from './variables.utils';
 
 import {onCall, HttpsError} from 'firebase-functions/v2/https';
 
@@ -77,7 +77,13 @@ export const createParticipant = onCall(async (request) => {
     .collection('participants')
     .doc(participantConfig.privateId);
 
-  // Set random timeout to avoid data contention with transaction
+  // Set random timeout to avoid data contention with transaction.
+  // Note: This also mitigates (but doesn't eliminate) a race condition with
+  // BalancedAssignment variables. The count/query used to determine assignment
+  // happens inside the transaction, but Firestore transactions only lock
+  // documents that are read—not aggregation queries. Two concurrent participants
+  // could see the same count and receive the same assignment. For most experiments
+  // with moderate join rates, the random delay provides sufficient distribution.
   await new Promise((resolve) => {
     setTimeout(resolve, Math.random() * 2000);
   });
@@ -129,7 +135,7 @@ export const createParticipant = onCall(async (request) => {
     participantConfig.currentStageId = experiment.stageIds[0];
 
     // Add variable values at the participant level
-    participantConfig.variableMap = generateVariablesForScope(
+    participantConfig.variableMap = await generateVariablesForScope(
       experiment.variableConfigs ?? [],
       {
         scope: VariableScope.PARTICIPANT,