diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index b7710f29d0..c2c91d4dc5 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -36,6 +36,10 @@ jobs: cache: 'pnpm' - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Log Internal Experiments + run: | + # Log enabled experiments + node scripts/log-experiments.js - name: Forge numeric Nightly version id: version env: diff --git a/docs/internal-feature-flags.md b/docs/internal-feature-flags.md new file mode 100644 index 0000000000..d17e76d6b3 --- /dev/null +++ b/docs/internal-feature-flags.md @@ -0,0 +1,755 @@ +# Internal Feature Flags Guide + +## Overview + +Internal feature flags in Roo Code are designed for controlled rollout of architectural changes, refactors, and experimental features. These flags are primarily enabled in nightly builds and gradually rolled out to stable versions once proven. + +## Philosophy + +- **Not user-facing**: These flags primarily control internal behavior, however in specific cases they can also be used to test user-facing features +- **Nightly-first**: New features are tested in nightly builds before stable +- **Simple lifecycle**: Nightly โ†’ Stable โ†’ Removed (no alpha/beta stages) +- **Developer-focused**: Primarily for the development team, not end users + +## Key Concepts + +### Internal vs NightlyDefault + +- **`internal`**: Controls visibility - whether the flag appears in the user settings UI +- **`nightlyDefault`**: Controls the default value specifically in nightly builds + +These properties work independently: + +1. **`internal: true, nightlyDefault: true`** - Hidden from users, auto-enabled in nightly (most common for refactors) +2. **`internal: false, nightlyDefault: true`** - Visible to users, on by default in nightly (good for user-facing features needing feedback) +3. **`internal: true, nightlyDefault: false`** - Hidden from users, off even in nightly (for very experimental features) +4. **`internal: false, nightlyDefault: false`** - Standard experimental feature, users can enable if they want + +## Implementation + +### 1. Define Internal Feature Flags + +Add to `packages/types/src/experiment.ts`: + +```typescript +export const experimentIds = [ + // Existing experiments + "powerSteering", + "concurrentFileReads", + + // Internal feature flags (prefix with underscore for clarity) + "_nightlyTestBanner", + "_improvedFileReader", + "_asyncToolExecution", + "_enhancedDiffStrategy", +] as const +``` + +### 2. Configure in Experiments System + +In `src/shared/experiments.ts`: + +```typescript +import type { AssertEqual, Equals, Keys, Values, ExperimentId } from "@roo-code/types" + +export const EXPERIMENT_IDS = { + // User-facing experiments + POWER_STEERING: "powerSteering", + CONCURRENT_FILE_READS: "concurrentFileReads", + + // Internal flags (use underscore prefix) + _NIGHTLY_TEST_BANNER: "_nightlyTestBanner", + // Add more internal flags as needed +} as const satisfies Record + +interface ExperimentConfig { + enabled: boolean + internal?: boolean // Mark as internal + nightlyDefault?: boolean // Enable by default in nightly + description?: string +} + +export const experimentConfigsMap: Record = { + // User-facing experiments + POWER_STEERING: { enabled: false }, + CONCURRENT_FILE_READS: { enabled: false }, + + // Internal flags + _NIGHTLY_TEST_BANNER: { + enabled: false, + internal: false, // Currently visible to users + nightlyDefault: true, + description: "Internal: Shows a test banner in nightly builds", + }, + // Add more internal flags as needed +} +``` + +### 3. Nightly Build Detection + +The nightly build detection is implemented as follows: + +```typescript +// src/shared/experiments.ts +export function getExperimentDefaults(isNightly: boolean = false): Record { + const defaults: Record = {} as Record + + Object.entries(experimentConfigsMap).forEach(([key, config]) => { + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + + if (isNightly && config.nightlyDefault) { + defaults[experimentId] = true + } else { + defaults[experimentId] = config.enabled + } + }) + + return defaults +} + +// Check if running nightly build +export function isNightlyBuild(): boolean { + // The nightly build process defines PKG_NAME as "roo-code-nightly" at compile time + // This is the most reliable single indicator for nightly builds + return process.env.PKG_NAME === "roo-code-nightly" +} + +// Update experimentDefault to use nightly defaults when appropriate +export const experimentDefault = getExperimentDefaults(isNightlyBuild()) +``` + +### 4. Hide Internal Flags from UI + +To hide internal flags from the settings UI, filter experiments that start with underscore: + +```typescript +// webview-ui/src/components/settings/ExperimentalSettings.tsx +
+ {Object.entries(experimentConfigsMap) + .filter(([key, config]) => { + // Filter out internal experiments (those starting with underscore) + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + return !experimentId.startsWith('_') + }) + .map(([key, config]) => { + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + return ( + setExperimentEnabled(experimentId, enabled)} + /> + ) + })} +
+``` + +## CI/CD Integration + +### 1. Nightly Build Workflow + +The nightly build workflow (`.github/workflows/nightly-publish.yml`) includes: + +```yaml +- name: Log Internal Experiments + run: | + # Log enabled experiments + node scripts/log-experiments.js +``` + +### 2. Build Configuration + +The nightly build process sets the PKG_NAME environment variable in `apps/vscode-nightly/esbuild.mjs`: + +```javascript +define: { + "process.env.PKG_NAME": '"roo-code-nightly"', + "process.env.PKG_VERSION": `"${overrideJson.version}"`, + "process.env.PKG_OUTPUT_CHANNEL": '"Roo-Code-Nightly"', + // ... other defines +} +``` + +### 3. Experiment Logging Script + +The `scripts/log-experiments.js` script: + +- Checks if running in nightly mode via `process.env.PKG_NAME === "roo-code-nightly"` +- Parses the experiments configuration from the TypeScript source +- Logs which experiments are enabled based on build type +- Separates user-facing and internal experiments in the output + +### 4. Gradual Rollout Process + +For managing the lifecycle of internal flags, you can extend the configuration: + +```typescript +// src/shared/experiments.ts +interface ExperimentConfig { + enabled: boolean + internal?: boolean + nightlyDefault?: boolean + description?: string + stableRolloutDate?: string // When to enable in stable + removalDate?: string // When to remove the flag +} +``` + +## Monitoring and Telemetry + +### 1. Track Internal Flag Usage + +Using the existing TelemetryService from `@roo-code/telemetry`: + +```typescript +// src/core/tools/readFileTool.ts +import { TelemetryService } from "@roo-code/telemetry" +import { TelemetryEventName } from "@roo-code/types" + +export async function readFileWithExperiment( + path: string, + experiments: Record, + taskId: string, +): Promise { + const useImprovedReader = experiments.isEnabled(experiments, EXPERIMENT_IDS._IMPROVED_FILE_READER) + + const start = Date.now() + let error: Error | null = null + + try { + const content = useImprovedReader ? await readFileImproved(path) : await readFileLegacy(path) + + // Track success metrics + TelemetryService.instance.captureEvent(TelemetryEventName.INTERNAL_EXPERIMENT_SUCCESS, { + taskId, + experiment: "_improvedFileReader", + duration: Date.now() - start, + fileSize: content.length, + }) + + return content + } catch (e) { + error = e as Error + + // Track failure metrics + TelemetryService.instance.captureEvent(TelemetryEventName.INTERNAL_EXPERIMENT_ERROR, { + taskId, + experiment: "_improvedFileReader", + error: error.message, + }) + + // Fallback to legacy implementation + return readFileLegacy(path) + } +} +``` + +### 2. Performance Comparison in Task Execution + +```typescript +// src/core/task/Task.ts +import { TelemetryService } from "@roo-code/telemetry" + +private async executeToolWithMetrics( + toolName: ToolName, + toolParams: any +): Promise { + const useAsyncExecution = experiments.isEnabled( + this.options.experiments, + EXPERIMENT_IDS._ASYNC_TOOL_EXECUTION + ) + + const start = Date.now() + + try { + const result = useAsyncExecution + ? await this.executeToolAsync(toolName, toolParams) + : await this.executeToolSync(toolName, toolParams) + + // Track tool execution metrics + TelemetryService.instance.captureToolUsage(this.taskId, toolName) + + // Additional metrics for experimental features + if (useAsyncExecution) { + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: this.taskId, + experiment: "_asyncToolExecution", + tool: toolName, + duration: Date.now() - start, + success: true + }) + } + + return result + } catch (error) { + // Track failures + if (useAsyncExecution) { + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: this.taskId, + experiment: "_asyncToolExecution", + tool: toolName, + duration: Date.now() - start, + success: false, + error: error.message + }) + } + throw error + } +} +``` + +## Lifecycle Management + +### 1. Introduction (Nightly Only) + +```typescript +// Initial state +{ + id: "_improvedFileReader", + internal: true, + enabled: false, + nightlyDefault: true, + stableRolloutDate: "2025-02-01" +} +``` + +### 2. Stable Rollout + +```typescript +// After testing in nightly +{ + id: "_improvedFileReader", + internal: true, + enabled: true, // Now enabled by default + nightlyDefault: true, + removalDate: "2025-03-01" // Plan for removal +} +``` + +### 3. Flag Removal + +After the feature is stable: + +1. Remove the flag checks from code +2. Remove the experiment definition +3. Update CHANGELOG.md + +## Best Practices + +1. **Prefix with underscore**: All internal flags should start with `_` +2. **No UI exposure**: Internal flags should not appear in settings +3. **Measure impact**: Always add telemetry to compare old vs new +4. **Set timelines**: Define rollout and removal dates +5. **Document changes**: Keep a log of what each flag controls +6. **Clean up promptly**: Remove flags once features are stable +7. **Use existing telemetry**: Leverage `TelemetryService` from `@roo-code/telemetry` + +## Testing Strategy + +### 1. Unit Tests + +```typescript +// src/core/tools/__tests__/readFileTool.test.ts +import { TelemetryService } from "@roo-code/telemetry" + +describe("File Reader with internal flags", () => { + beforeEach(() => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + }) + + it("should use legacy reader when flag is disabled", async () => { + const experiments = { _improvedFileReader: false } + const spy = jest.spyOn(FileReader.prototype as any, "readFileLegacy") + + await readFile("test.txt", experiments) + + expect(spy).toHaveBeenCalled() + }) + + it("should use improved reader when flag is enabled", async () => { + const experiments = { _improvedFileReader: true } + const spy = jest.spyOn(FileReader.prototype as any, "readFileImproved") + + await readFile("test.txt", experiments) + + expect(spy).toHaveBeenCalled() + }) +}) +``` + +### 2. Nightly-Specific Tests + +```typescript +// e2e/nightly/internal-features.test.ts +describe("Nightly Internal Features", () => { + beforeAll(() => { + process.env.ROO_CODE_NIGHTLY = "true" + }) + + test("internal flags are enabled by default", () => { + const defaults = getExperimentDefaults(true) + expect(defaults._improvedFileReader).toBe(true) + expect(defaults._asyncToolExecution).toBe(true) + }) +}) +``` + +## Migration Checklist + +When moving an internal feature from nightly to stable: + +- [ ] Feature has been in nightly for at least 2 weeks +- [ ] No critical bugs reported +- [ ] Performance metrics are acceptable (check telemetry) +- [ ] Error rates are within tolerance +- [ ] Update `enabled: true` in experiment config +- [ ] Set `removalDate` for flag cleanup +- [ ] Update CHANGELOG.md +- [ ] Notify team of rollout + +## Usage Examples + +### Example 1: Refactoring File Reading with Concurrent Reads + +```typescript +// src/core/tools/readFileTool.ts +import { experiments, EXPERIMENT_IDS } from "../../shared/experiments" +import type { ExperimentId } from "@roo-code/types" + +export async function readFiles( + paths: string[], + experiments: Record, +): Promise> { + const useConcurrentReads = experiments.isEnabled(experiments, EXPERIMENT_IDS.CONCURRENT_FILE_READS) + + if (useConcurrentReads) { + // Read files in parallel + const results = await Promise.all( + paths.map(async (path) => ({ + path, + content: await readFileContent(path), + })), + ) + return new Map(results.map((r) => [r.path, r.content])) + } else { + // Read files sequentially + const results = new Map() + for (const path of paths) { + results.set(path, await readFileContent(path)) + } + return results + } +} +``` + +### Example 2: Enhanced Diff Strategy + +```typescript +// src/core/diff/strategies/multi-search-replace.ts +import { experiments, EXPERIMENT_IDS } from "../../../shared/experiments" +import { DiffStrategy } from "../../../shared/tools" + +export class MultiSearchReplaceDiffStrategy { + constructor(private experiments: Record) {} + + async applyDiff(content: string, diff: string): Promise { + const useEnhancedStrategy = experiments.isEnabled(this.experiments, EXPERIMENT_IDS._ENHANCED_DIFF_STRATEGY) + + if (useEnhancedStrategy) { + // New implementation with better conflict resolution + return this.applyEnhancedDiff(content, diff) + } else { + // Current stable implementation + return this.applyStandardDiff(content, diff) + } + } +} +``` + +## Complete Implementation Example + +### Step 1: Update Type Definitions + +```typescript +// packages/types/src/experiment.ts +export const experimentIds = [ + "powerSteering", + "concurrentFileReads", + // Add new internal flag + "_enhancedDiffStrategy", +] as const + +export type ExperimentId = (typeof experimentIds)[number] +``` + +### Step 2: Update Experiments Configuration + +```typescript +// src/shared/experiments.ts +import type { AssertEqual, Equals, Keys, Values, ExperimentId } from "@roo-code/types" + +export const EXPERIMENT_IDS = { + POWER_STEERING: "powerSteering", + CONCURRENT_FILE_READS: "concurrentFileReads", + // Add new internal flag constant + _ENHANCED_DIFF_STRATEGY: "_enhancedDiffStrategy", +} as const satisfies Record + +export const experimentConfigsMap: Record = { + POWER_STEERING: { enabled: false }, + CONCURRENT_FILE_READS: { enabled: false }, + // Add configuration for internal flag + _ENHANCED_DIFF_STRATEGY: { + enabled: false, + internal: true, + nightlyDefault: true, + description: "Internal: Improved diff application with better conflict resolution", + }, +} +``` + +### Step 3: Update UI to Filter Internal Flags + +```typescript +// webview-ui/src/components/settings/ExperimentalSettings.tsx +import { experimentConfigsMap, EXPERIMENT_IDS } from "../../../src/shared/experiments" + +export function ExperimentalSettings({ experiments, setExperimentEnabled }) { + return ( +
+ {Object.entries(experimentConfigsMap) + .filter(([key, config]) => { + // Filter out internal experiments + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + return !experimentId.startsWith('_') + }) + .map(([key, config]) => { + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + return ( + setExperimentEnabled(experimentId, enabled)} + /> + ) + })} +
+ ) +} +``` + +### Step 4: Use the Internal Flag in Code + +```typescript +// src/core/tools/applyDiffTool.ts +import { experiments, EXPERIMENT_IDS } from "../../shared/experiments" +import { TelemetryService } from "@roo-code/telemetry" +import type { ExperimentId } from "@roo-code/types" + +export async function applyDiff( + cline: Task, + filePath: string, + diffContent: string, + experiments: Record, +): Promise { + const useEnhancedStrategy = experiments.isEnabled(experiments, EXPERIMENT_IDS._ENHANCED_DIFF_STRATEGY) + + const start = Date.now() + + try { + if (useEnhancedStrategy) { + // New implementation with better conflict resolution + await applyEnhancedDiff(filePath, diffContent) + } else { + // Current stable implementation + await applyStandardDiff(filePath, diffContent) + } + + // Track success + if (useEnhancedStrategy) { + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: cline.taskId, + experiment: "_enhancedDiffStrategy", + duration: Date.now() - start, + success: true, + }) + } + } catch (error) { + // Track failure and capture diff application error + TelemetryService.instance.captureDiffApplicationError(cline.taskId, 1) + + if (useEnhancedStrategy) { + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: cline.taskId, + experiment: "_enhancedDiffStrategy", + duration: Date.now() - start, + success: false, + error: error.message, + }) + } + + throw error + } +} +``` + +### Step 5: Add Telemetry Integration + +```typescript +// packages/types/src/telemetry.ts +export enum TelemetryEventName { + // ... existing events + EXPERIMENT_METRIC = "experiment_metric", + INTERNAL_EXPERIMENT_SUCCESS = "internal_experiment_success", + INTERNAL_EXPERIMENT_ERROR = "internal_experiment_error", +} + +// src/core/task/Task.ts +import { TelemetryService } from "@roo-code/telemetry" + +private async executeToolWithExperiment( + toolName: ToolName, + toolParams: any +): Promise { + const useAsyncExecution = experiments.isEnabled( + this.options.experiments, + EXPERIMENT_IDS._ASYNC_TOOL_EXECUTION + ) + + if (useAsyncExecution) { + const start = Date.now() + try { + await this.executeToolAsync(toolName, toolParams) + + // Track async execution success + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: this.taskId, + experiment: "_asyncToolExecution", + tool: toolName, + duration: Date.now() - start, + success: true + }) + } catch (error) { + // Track async execution failure + TelemetryService.instance.captureEvent(TelemetryEventName.EXPERIMENT_METRIC, { + taskId: this.taskId, + experiment: "_asyncToolExecution", + tool: toolName, + duration: Date.now() - start, + success: false, + error: error.message + }) + throw error + } + } else { + await this.executeToolSync(toolName, toolParams) + } +} +``` + +### Step 6: Testing + +```typescript +// src/core/tools/__tests__/applyDiffTool.test.ts +import { TelemetryService } from "@roo-code/telemetry" +import { applyDiff } from "../applyDiffTool" + +describe("Apply Diff with internal flags", () => { + let mockCline: any + + beforeEach(() => { + if (!TelemetryService.hasInstance()) { + TelemetryService.createInstance([]) + } + + mockCline = { + taskId: "test-task-123", + say: jest.fn(), + ask: jest.fn(), + } + }) + + it("uses standard diff when flag is disabled", async () => { + const experiments = { _enhancedDiffStrategy: false } + const spy = jest.spyOn(global as any, "applyStandardDiff") + + await applyDiff(mockCline, "test.ts", "diff content", experiments) + + expect(spy).toHaveBeenCalled() + }) + + it("uses enhanced diff when flag is enabled", async () => { + const experiments = { _enhancedDiffStrategy: true } + const spy = jest.spyOn(global as any, "applyEnhancedDiff") + + await applyDiff(mockCline, "test.ts", "diff content", experiments) + + expect(spy).toHaveBeenCalled() + }) + + it("tracks telemetry for enhanced diff", async () => { + const experiments = { _enhancedDiffStrategy: true } + const telemetrySpy = jest.spyOn(TelemetryService.instance, "captureEvent") + + await applyDiff(mockCline, "test.ts", "diff content", experiments) + + expect(telemetrySpy).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + taskId: "test-task-123", + experiment: "_enhancedDiffStrategy", + success: true, + }), + ) + }) +}) +``` + +### Manual Testing in Development + +To test internal flags during development: + +1. **Temporarily enable in code**: + +```typescript +// For testing, temporarily modify the config +experimentConfigsMap._ENHANCED_DIFF_STRATEGY.enabled = true +``` + +2. **Use environment variable**: + +```bash +# Set PKG_NAME to simulate nightly build +PKG_NAME=roo-code-nightly npm run dev +``` + +3. **Modify the isNightlyBuild function temporarily**: + +```typescript +export function isNightlyBuild(): boolean { + // For testing only - remove before committing + return true +} +``` + +## Rollout Timeline + +1. **Week 1-2**: Deploy to nightly builds only +2. **Week 3-4**: Monitor telemetry, fix any issues +3. **Week 5**: Enable by default in stable (change `enabled: true`) +4. **Week 8**: Remove flag and make permanent + +## Cleanup Process + +When removing a flag after stable rollout: + +- [ ] Remove from `experimentIds` in types +- [ ] Remove from `EXPERIMENT_IDS` constant +- [ ] Remove from `experimentConfigsMap` +- [ ] Remove conditional logic in code +- [ ] Update tests to remove flag checks +- [ ] Add entry to CHANGELOG.md documenting the change diff --git a/packages/types/src/experiment.ts b/packages/types/src/experiment.ts index 0e0db7276e..1dd1403f83 100644 --- a/packages/types/src/experiment.ts +++ b/packages/types/src/experiment.ts @@ -6,7 +6,7 @@ import type { Keys, Equals, AssertEqual } from "./type-fu.js" * ExperimentId */ -export const experimentIds = ["powerSteering", "concurrentFileReads"] as const +export const experimentIds = ["powerSteering", "concurrentFileReads", "_nightlyTestBanner"] as const export const experimentIdsSchema = z.enum(experimentIds) @@ -19,6 +19,7 @@ export type ExperimentId = z.infer export const experimentsSchema = z.object({ powerSteering: z.boolean(), concurrentFileReads: z.boolean(), + _nightlyTestBanner: z.boolean(), }) export type Experiments = z.infer diff --git a/packages/types/src/telemetry.ts b/packages/types/src/telemetry.ts index 9c10a63b5e..a384b534ce 100644 --- a/packages/types/src/telemetry.ts +++ b/packages/types/src/telemetry.ts @@ -45,6 +45,11 @@ export enum TelemetryEventName { DIFF_APPLICATION_ERROR = "Diff Application Error", SHELL_INTEGRATION_ERROR = "Shell Integration Error", CONSECUTIVE_MISTAKE_ERROR = "Consecutive Mistake Error", + + // Internal experiment metrics + EXPERIMENT_METRIC = "Experiment Metric", + INTERNAL_EXPERIMENT_SUCCESS = "Internal Experiment Success", + INTERNAL_EXPERIMENT_ERROR = "Internal Experiment Error", } /** @@ -112,6 +117,9 @@ export const rooCodeTelemetryEventSchema = z.discriminatedUnion("type", [ TelemetryEventName.CONSECUTIVE_MISTAKE_ERROR, TelemetryEventName.CONTEXT_CONDENSED, TelemetryEventName.SLIDING_WINDOW_TRUNCATION, + TelemetryEventName.EXPERIMENT_METRIC, + TelemetryEventName.INTERNAL_EXPERIMENT_SUCCESS, + TelemetryEventName.INTERNAL_EXPERIMENT_ERROR, ]), properties: telemetryPropertiesSchema, }), diff --git a/scripts/log-experiments.js b/scripts/log-experiments.js new file mode 100755 index 0000000000..f08745ccab --- /dev/null +++ b/scripts/log-experiments.js @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +const path = require("path") +const fs = require("fs") + +// Check if running in nightly mode +const isNightly = process.env.PKG_NAME === "roo-code-nightly" + +// Read the experiments configuration +const experimentsPath = path.join(__dirname, "../src/shared/experiments.ts") +const experimentsContent = fs.readFileSync(experimentsPath, "utf8") + +// Extract EXPERIMENT_IDS +const experimentIdsMatch = experimentsContent.match(/export const EXPERIMENT_IDS = \{([^}]+)\}/s) +if (!experimentIdsMatch) { + console.error("โŒ Could not find EXPERIMENT_IDS in experiments.ts") + process.exit(1) +} + +// Extract experimentConfigsMap +const configsMatch = experimentsContent.match(/export const experimentConfigsMap[^{]+\{(.*)\}/s) +if (!configsMatch) { + console.error("โŒ Could not find experimentConfigsMap in experiments.ts") + process.exit(1) +} + +// Parse experiment IDs +const experimentIds = {} +const idsContent = experimentIdsMatch[1] +const idMatches = idsContent.matchAll(/(\w+):\s*"([^"]+)"/g) +for (const match of idMatches) { + experimentIds[match[1]] = match[2] +} + +// Parse experiment configs +const configsContent = configsMatch[1] +const configMatches = configsContent.matchAll(/(\w+):\s*\{([^}]+)\}/g) + +const experiments = [] +for (const match of configMatches) { + const key = match[1] + const configStr = match[2] + const enabled = configStr.includes("enabled: true") + const isInternal = configStr.includes("internal: true") + const nightlyDefault = configStr.includes("nightlyDefault: true") + + const id = experimentIds[key] + experiments.push({ + key, + id, + enabled, + internal: isInternal, + nightlyDefault, + effectiveEnabled: isNightly && nightlyDefault ? true : enabled, + }) +} + +console.log(`\n๐Ÿงช Experiments Configuration (${isNightly ? "NIGHTLY" : "STABLE"} build):\n`) +console.log("User-facing experiments:") +experiments + .filter((exp) => !exp.internal) + .forEach(({ id, effectiveEnabled }) => { + console.log(` - ${id}: ${effectiveEnabled ? "โœ… ENABLED" : "โŒ DISABLED"}`) + }) + +const internalExperiments = experiments.filter((exp) => exp.internal) +if (internalExperiments.length > 0) { + console.log("\nInternal feature flags:") + internalExperiments.forEach(({ id, effectiveEnabled, nightlyDefault }) => { + const status = effectiveEnabled ? "โœ… ENABLED" : "โŒ DISABLED" + const nightlyInfo = nightlyDefault ? " (nightly default)" : "" + console.log(` - ${id}: ${status}${nightlyInfo}`) + }) +} + +console.log("\n") diff --git a/src/shared/__tests__/experiments.test.ts b/src/shared/__tests__/experiments.test.ts index e50bb18878..e59ae01e9c 100644 --- a/src/shared/__tests__/experiments.test.ts +++ b/src/shared/__tests__/experiments.test.ts @@ -1,10 +1,24 @@ -// npx jest src/shared/__tests__/experiments.test.ts +import { ExperimentId } from "@roo-code/types" +import { getExperimentDefaults, isNightlyBuild } from "../experiments" +import { EXPERIMENT_IDS, experimentConfigsMap, experiments as Experiments } from "../experiments" -import type { ExperimentId } from "@roo-code/types" +describe("Internal Feature Flags", () => { + let originalEnv: string | undefined -import { EXPERIMENT_IDS, experimentConfigsMap, experiments as Experiments } from "../experiments" + beforeEach(() => { + // Save original environment variable + originalEnv = process.env.ROO_CODE_NIGHTLY + }) + + afterEach(() => { + // Restore original environment variable + if (originalEnv !== undefined) { + process.env.ROO_CODE_NIGHTLY = originalEnv + } else { + delete process.env.ROO_CODE_NIGHTLY + } + }) -describe("experiments", () => { describe("POWER_STEERING", () => { it("is configured correctly", () => { expect(EXPERIMENT_IDS.POWER_STEERING).toBe("powerSteering") @@ -19,6 +33,7 @@ describe("experiments", () => { const experiments: Record = { powerSteering: false, concurrentFileReads: false, + _nightlyTestBanner: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.POWER_STEERING)).toBe(false) }) @@ -27,6 +42,7 @@ describe("experiments", () => { const experiments: Record = { powerSteering: true, concurrentFileReads: false, + _nightlyTestBanner: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.POWER_STEERING)).toBe(true) }) @@ -35,8 +51,70 @@ describe("experiments", () => { const experiments: Record = { powerSteering: false, concurrentFileReads: false, + _nightlyTestBanner: false, } expect(Experiments.isEnabled(experiments, EXPERIMENT_IDS.POWER_STEERING)).toBe(false) }) }) + + describe("isNightlyBuild", () => { + const originalPkgName = process.env.PKG_NAME + + afterEach(() => { + // Restore original PKG_NAME + if (originalPkgName !== undefined) { + process.env.PKG_NAME = originalPkgName + } else { + delete process.env.PKG_NAME + } + }) + + it("should return true when PKG_NAME is roo-code-nightly", () => { + process.env.PKG_NAME = "roo-code-nightly" + expect(isNightlyBuild()).toBe(true) + }) + + it("should return false when PKG_NAME is not roo-code-nightly", () => { + process.env.PKG_NAME = "roo-cline" + expect(isNightlyBuild()).toBe(false) + }) + + it("should return false when PKG_NAME is undefined", () => { + delete process.env.PKG_NAME + expect(isNightlyBuild()).toBe(false) + }) + }) + + describe("getExperimentDefaults", () => { + it("should return default values for stable build", () => { + const defaults = getExperimentDefaults(false) + expect(defaults.powerSteering).toBe(false) + expect(defaults.concurrentFileReads).toBe(false) + expect(defaults._nightlyTestBanner).toBe(false) + }) + + it("should respect nightlyDefault for nightly builds", () => { + const stableDefaults = getExperimentDefaults(false) + const nightlyDefaults = getExperimentDefaults(true) + + // Stable defaults + expect(stableDefaults.powerSteering).toBe(false) + expect(stableDefaults._nightlyTestBanner).toBe(false) + + // Nightly defaults - these have nightlyDefault: true + expect(nightlyDefaults.powerSteering).toBe(true) + expect(nightlyDefaults._nightlyTestBanner).toBe(true) + }) + }) + + describe("Internal flag filtering", () => { + it("should filter out flags starting with underscore in UI", () => { + // This is tested in the UI component, but we can verify the convention + const internalFlagExample = "_internalFeature" + const userFacingFlag = "userFeature" + + expect(internalFlagExample.startsWith("_")).toBe(true) + expect(userFacingFlag.startsWith("_")).toBe(false) + }) + }) }) diff --git a/src/shared/experiments.ts b/src/shared/experiments.ts index e387f7ddcd..17c90a5f12 100644 --- a/src/shared/experiments.ts +++ b/src/shared/experiments.ts @@ -3,6 +3,7 @@ import type { AssertEqual, Equals, Keys, Values, ExperimentId } from "@roo-code/ export const EXPERIMENT_IDS = { POWER_STEERING: "powerSteering", CONCURRENT_FILE_READS: "concurrentFileReads", + _NIGHTLY_TEST_BANNER: "_nightlyTestBanner", } as const satisfies Record type _AssertExperimentIds = AssertEqual>> @@ -11,19 +12,59 @@ type ExperimentKey = Keys interface ExperimentConfig { enabled: boolean + internal?: boolean // Mark as internal + nightlyDefault?: boolean // Enable by default in nightly + description?: string } export const experimentConfigsMap: Record = { POWER_STEERING: { enabled: false }, CONCURRENT_FILE_READS: { enabled: false }, + _NIGHTLY_TEST_BANNER: { + enabled: false, + internal: false, + nightlyDefault: true, + description: "Internal: Shows a test banner in nightly builds", + }, } -export const experimentDefault = Object.fromEntries( - Object.entries(experimentConfigsMap).map(([_, config]) => [ - EXPERIMENT_IDS[_ as keyof typeof EXPERIMENT_IDS] as ExperimentId, - config.enabled, - ]), -) as Record +/** + * Gets the default values for all experiments based on build type + * + * For nightly builds: + * - Experiments with nightlyDefault=true will be enabled by default + * - Other experiments use their configured enabled value + * + * For stable builds: + * - All experiments use their configured enabled value + * + * This allows features to be automatically enabled in nightly builds + * for testing before being enabled in stable builds. + */ +export function getExperimentDefaults(isNightly: boolean = false): Record { + const defaults: Record = {} as Record + + Object.entries(experimentConfigsMap).forEach(([key, config]) => { + const experimentId = EXPERIMENT_IDS[key as keyof typeof EXPERIMENT_IDS] + + if (isNightly && config.nightlyDefault) { + defaults[experimentId] = true + } else { + defaults[experimentId] = config.enabled + } + }) + + return defaults +} + +// Check if running nightly build +export function isNightlyBuild(): boolean { + // The nightly build process defines PKG_NAME as "roo-code-nightly" at compile time + // This is the most reliable single indicator for nightly builds + return process.env.PKG_NAME === "roo-code-nightly" +} + +export const experimentDefault = getExperimentDefaults(isNightlyBuild()) export const experiments = { get: (id: ExperimentKey): ExperimentConfig | undefined => experimentConfigsMap[id], diff --git a/webview-ui/src/App.tsx b/webview-ui/src/App.tsx index a04d0dd40e..871101159b 100644 --- a/webview-ui/src/App.tsx +++ b/webview-ui/src/App.tsx @@ -16,6 +16,7 @@ import McpView from "./components/mcp/McpView" import ModesView from "./components/modes/ModesView" import { HumanRelayDialog } from "./components/human-relay/HumanRelayDialog" import { AccountView } from "./components/account/AccountView" +import { NightlyTestBanner } from "./components/NightlyTestBanner" type Tab = "settings" | "history" | "mcp" | "modes" | "chat" | "account" @@ -37,6 +38,7 @@ const App = () => { telemetryKey, machineId, cloudUserInfo, + experiments, } = useExtensionState() const [showAnnouncement, setShowAnnouncement] = useState(false) @@ -121,6 +123,7 @@ const App = () => { ) : ( <> + {tab === "modes" && switchTab("chat")} />} {tab === "mcp" && switchTab("chat")} />} {tab === "history" && switchTab("chat")} />} diff --git a/webview-ui/src/components/NightlyTestBanner.tsx b/webview-ui/src/components/NightlyTestBanner.tsx new file mode 100644 index 0000000000..36bd3ed05f --- /dev/null +++ b/webview-ui/src/components/NightlyTestBanner.tsx @@ -0,0 +1,32 @@ +import React from "react" +import { AlertCircle } from "lucide-react" +import type { ExperimentId } from "@roo-code/types" +import { EXPERIMENT_IDS } from "@roo/experiments" + +interface NightlyTestBannerProps { + experiments: Record +} + +export const NightlyTestBanner: React.FC = ({ experiments }) => { + // Check if the internal nightly test banner experiment is enabled + const showBanner = experiments[EXPERIMENT_IDS._NIGHTLY_TEST_BANNER] ?? false + + if (!showBanner) { + return null + } + + return ( +
+ +
+

+ ๐Ÿงช Nightly Build Test Feature +

+

+ This banner is an internal test feature that only appears in nightly builds. It demonstrates that + internal experiments with nightlyDefault=true are working correctly. +

+
+
+ ) +} diff --git a/webview-ui/src/components/settings/ExperimentalSettings.tsx b/webview-ui/src/components/settings/ExperimentalSettings.tsx index 5525a842b2..c6a2b65f3c 100644 --- a/webview-ui/src/components/settings/ExperimentalSettings.tsx +++ b/webview-ui/src/components/settings/ExperimentalSettings.tsx @@ -55,7 +55,15 @@ export const ExperimentalSettings = ({
{Object.entries(experimentConfigsMap) - .filter((config) => config[0] !== "DIFF_STRATEGY" && config[0] !== "MULTI_SEARCH_AND_REPLACE") + .filter((config) => { + // Filter out internal experiments (those starting with underscore) + const experimentId = EXPERIMENT_IDS[config[0] as keyof typeof EXPERIMENT_IDS] + return ( + !experimentId.startsWith("_") && + config[0] !== "DIFF_STRATEGY" && + config[0] !== "MULTI_SEARCH_AND_REPLACE" + ) + }) .map((config) => { if (config[0] === "CONCURRENT_FILE_READS") { return ( @@ -78,7 +86,10 @@ export const ExperimentalSettings = ({ experimentKey={config[0]} enabled={experiments[EXPERIMENT_IDS[config[0] as keyof typeof EXPERIMENT_IDS]] ?? false} onChange={(enabled) => - setExperimentEnabled(EXPERIMENT_IDS[config[0] as keyof typeof EXPERIMENT_IDS], enabled) + setExperimentEnabled( + EXPERIMENT_IDS[config[0] as keyof typeof EXPERIMENT_IDS], + enabled, + ) } /> ) diff --git a/webview-ui/src/context/__tests__/ExtensionStateContext.test.tsx b/webview-ui/src/context/__tests__/ExtensionStateContext.test.tsx index ff29b9669e..1e3a84733c 100644 --- a/webview-ui/src/context/__tests__/ExtensionStateContext.test.tsx +++ b/webview-ui/src/context/__tests__/ExtensionStateContext.test.tsx @@ -220,8 +220,8 @@ describe("mergeExtensionState", () => { apiConfiguration: { modelMaxThinkingTokens: 456, modelTemperature: 0.3 }, experiments: { powerSteering: true, - autoCondenseContext: true, concurrentFileReads: true, + _nightlyTestBanner: false, } as Record, }