Skip to content

Commit cde1cd1

Browse files
authored
Merge branch 'master' into samedson-gcli-evals-tool-2
2 parents f46cd93 + 522e957 commit cde1cd1

File tree

22 files changed

+582
-31
lines changed

22 files changed

+582
-31
lines changed

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
- Add a command `firebase firestore:databases:clone` to clone a Firestore database (#9262).
12
- Add JSON format support for Cloud Functions secrets with `--format json` flag and auto-detection from file extensions (#1745)
23
- `firebase dataconnect:sdk:generate` will run `init dataconnect:sdk` automatically if no SDKs are configured (#9325).
34
- Tighten --only filter resolution for functions deployment to prefer codebase names (#9353)
5+
- Add `disallowLegacyRuntimeConfig` option to `firebase.json` to optionally skip fetching legacy Runtime Config during function deploys (#9354)

schema/firebase-config.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,9 @@
872872
"configDir": {
873873
"type": "string"
874874
},
875+
"disallowLegacyRuntimeConfig": {
876+
"type": "boolean"
877+
},
875878
"ignore": {
876879
"items": {
877880
"type": "string"

scripts/agent-evals/README.md

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# Agent Evals
2+
3+
This codebase evaluates the Firebase MCP server running in various coding agents.
4+
5+
## Running Tests
6+
7+
Agent Evals use [mocha](https://www.npmjs.com/package/mocha) to run tests, similar to how the Firebase CLI unit tests are implemented. The test commands will automatically instrument the Firebase MCP Server.
8+
9+
WARNING: Running evals will remove any existing Firebase MCP Servers and the Firebase Gemini CLI Extension from your user account so that they don't interfere with the test.
10+
11+
For running tests during development, run:
12+
13+
```bash
14+
# Link and build the CLI so that the `firebase` is built with your changes
15+
$ npm link
16+
$ npm run build:watch
17+
18+
# In a separate terminal, run the test suite.
19+
# Running test:dev will skip rebuilding the Firebase CLI (because your watch
20+
# command is doing that for you)
21+
$ cd scripts/agent-evals
22+
$ npm run test:dev
23+
```
24+
25+
For running in CI, the eval system will do a clean install of the Firebase CLI before running tests:
26+
27+
```bash
28+
$ npm run test
29+
```
30+
31+
## Writing Tests
32+
33+
Add a new file in `src/tests`:
34+
35+
```typescript
36+
import { startAgentTest } from "../runner/index.js";
37+
import { AgentTestRunner } from "../runner/index.js";
38+
39+
// Ensure you import hooks which instruments an afterEach block that cleans up
40+
// the agent and the pseudo terminal.
41+
import "../helpers/hooks.js";
42+
43+
describe("<prompt-or-tool-name>", function (this: Mocha.Suite) {
44+
// Recommend setting retries > 0 because LLMs are nondeterministic
45+
this.retries(2);
46+
47+
it("<use-case>", async function (this: Mocha.Context) {
48+
// Start the AgentTestRunner, which will start up the coding agent in a
49+
// pseudo-terminal, and wait for it to load the Firebase MCP server, and
50+
// start accepting keystrokes
51+
const run: AgentTestRunner = await startAgentTest(this);
52+
53+
// Simulate typing in the terminal. This will await until the "turn" is over
54+
// so any assertions on what happened will happen on the current "turn"
55+
await run.type("/firebase:init");
56+
// Assert that the agent outputted "Backend Services"
57+
await run.expectText("Backend Services");
58+
59+
await run.type("Use Firebase Project `project-id-1000`");
60+
// Assert that a tool was called with the given arguments, and that it was
61+
// successful
62+
await run.expectToolCalls([
63+
"firebase_update_environment",
64+
argumentContains: "project-id-1000",
65+
isSuccess: true,
66+
]);
67+
68+
// Important: Expectations apply to the last "turn". Each time you type, it
69+
// creates a new turn. This ensures you are only asserting against the most
70+
// recent actions of the agent
71+
await run.type("Hello world");
72+
// This will fail, because "Hello World" doesn't trigger a tool call
73+
await run.expectToolCalls([
74+
"firebase_update_environment",
75+
argumentContains: "project-id-1000",
76+
isSuccess: true,
77+
]);
78+
});
79+
});
80+
```
Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import * as clc from "colorette";
2+
3+
import { Command } from "../command";
4+
import * as fsi from "../firestore/api";
5+
import * as types from "../firestore/api-types";
6+
import { getCurrentMinuteAsIsoString, parseDatabaseName } from "../firestore/util";
7+
import { logger } from "../logger";
8+
import { requirePermissions } from "../requirePermissions";
9+
import { Emulators } from "../emulator/types";
10+
import { warnEmulatorNotSupported } from "../emulator/commandUtils";
11+
import { EncryptionType, FirestoreOptions } from "../firestore/options";
12+
import { PrettyPrint } from "../firestore/pretty-print";
13+
import { FirebaseError } from "../error";
14+
15+
export const command = new Command("firestore:databases:clone <sourceDatabase> <targetDatabase>")
16+
.description("clone one Firestore database to another")
17+
.option(
18+
"-e, --encryption-type <encryptionType>",
19+
`encryption method of the cloned database; one of ${EncryptionType.USE_SOURCE_ENCRYPTION} (default), ` +
20+
`${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}, ${EncryptionType.GOOGLE_DEFAULT_ENCRYPTION}`,
21+
)
22+
// TODO(b/356137854): Remove allowlist only message once feature is public GA.
23+
.option(
24+
"-k, --kms-key-name <kmsKeyName>",
25+
"resource ID of the Cloud KMS key to encrypt the cloned database. This " +
26+
"feature is allowlist only in initial launch",
27+
)
28+
.option(
29+
"-s, --snapshot-time <snapshotTime>",
30+
"snapshot time of the source database to use, in ISO 8601 format. Can be any minutely snapshot after the database's earliest version time. If unspecified, takes the most recent available snapshot",
31+
)
32+
.before(requirePermissions, ["datastore.databases.clone"])
33+
.before(warnEmulatorNotSupported, Emulators.FIRESTORE)
34+
.action(async (sourceDatabase: string, targetDatabase: string, options: FirestoreOptions) => {
35+
const api = new fsi.FirestoreApi();
36+
const printer = new PrettyPrint();
37+
const helpCommandText = "See firebase firestore:databases:clone --help for more info.";
38+
39+
if (options.database) {
40+
throw new FirebaseError(
41+
`--database is not a supported flag for 'firestoree:databases:clone'. ${helpCommandText}`,
42+
);
43+
}
44+
45+
let snapshotTime: string;
46+
if (options.snapshotTime) {
47+
snapshotTime = options.snapshotTime;
48+
} else {
49+
snapshotTime = getCurrentMinuteAsIsoString();
50+
}
51+
52+
let encryptionConfig: types.EncryptionConfig | undefined = undefined;
53+
switch (options.encryptionType) {
54+
case EncryptionType.GOOGLE_DEFAULT_ENCRYPTION:
55+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
56+
encryptionConfig = { googleDefaultEncryption: {} };
57+
break;
58+
case EncryptionType.USE_SOURCE_ENCRYPTION:
59+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
60+
encryptionConfig = { useSourceEncryption: {} };
61+
break;
62+
case EncryptionType.CUSTOMER_MANAGED_ENCRYPTION:
63+
encryptionConfig = {
64+
customerManagedEncryption: { kmsKeyName: getKmsKeyOrThrow(options.kmsKeyName) },
65+
};
66+
break;
67+
case undefined:
68+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
69+
break;
70+
default:
71+
throw new FirebaseError(`Invalid value for flag --encryption-type. ${helpCommandText}`);
72+
}
73+
74+
// projects must be the same
75+
const targetDatabaseName = parseDatabaseName(targetDatabase);
76+
const parentProject = targetDatabaseName.projectId;
77+
const targetDatabaseId = targetDatabaseName.databaseId;
78+
const sourceProject = parseDatabaseName(sourceDatabase).projectId;
79+
if (parentProject !== sourceProject) {
80+
throw new FirebaseError(`Cloning across projects is not supported.`);
81+
}
82+
const lro: types.Operation = await api.cloneDatabase(
83+
sourceProject,
84+
{
85+
database: sourceDatabase,
86+
snapshotTime,
87+
},
88+
targetDatabaseId,
89+
encryptionConfig,
90+
);
91+
92+
if (lro.error) {
93+
logger.error(
94+
clc.bold(
95+
`Clone to ${printer.prettyDatabaseString(targetDatabase)} failed. See below for details.`,
96+
),
97+
);
98+
printer.prettyPrintOperation(lro);
99+
} else {
100+
logger.info(
101+
clc.bold(`Successfully initiated clone to ${printer.prettyDatabaseString(targetDatabase)}`),
102+
);
103+
logger.info(
104+
"Please be sure to configure Firebase rules in your Firebase config file for\n" +
105+
"the new database. By default, created databases will have closed rules that\n" +
106+
"block any incoming third-party traffic.",
107+
);
108+
logger.info();
109+
logger.info(`You can monitor the progress of this clone by executing this command:`);
110+
logger.info();
111+
logger.info(
112+
`firebase firestore:operations:describe --database="${targetDatabaseId}" ${lro.name}`,
113+
);
114+
logger.info();
115+
logger.info(
116+
`Once the clone is complete, your database may be viewed at ${printer.firebaseConsoleDatabaseUrl(options.project, targetDatabaseId)}`,
117+
);
118+
}
119+
120+
return lro;
121+
122+
function throwIfKmsKeyNameIsSet(kmsKeyName: string | undefined): void {
123+
if (kmsKeyName) {
124+
throw new FirebaseError(
125+
"--kms-key-name can only be set when specifying an --encryption-type " +
126+
`of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
127+
);
128+
}
129+
}
130+
131+
function getKmsKeyOrThrow(kmsKeyName: string | undefined): string {
132+
if (kmsKeyName) return kmsKeyName;
133+
134+
throw new FirebaseError(
135+
"--kms-key-name must be provided when specifying an --encryption-type " +
136+
`of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
137+
);
138+
}
139+
});

src/commands/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ export function load(client: any): any {
115115
client.firestore.databases.update = loadCommand("firestore-databases-update");
116116
client.firestore.databases.delete = loadCommand("firestore-databases-delete");
117117
client.firestore.databases.restore = loadCommand("firestore-databases-restore");
118+
client.firestore.databases.clone = loadCommand("firestore-databases-clone");
118119
client.firestore.backups = {};
119120
client.firestore.backups.schedules = {};
120121
client.firestore.backups.list = loadCommand("firestore-backups-list");

src/commands/internaltesting-functions-discover.ts

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@ import { Command } from "../command";
22
import { Options } from "../options";
33
import { logger } from "../logger";
44
import { loadCodebases } from "../deploy/functions/prepare";
5-
import { normalizeAndValidate } from "../functions/projectConfig";
5+
import { normalizeAndValidate, shouldUseRuntimeConfig } from "../functions/projectConfig";
66
import { getProjectAdminSdkConfigOrCached } from "../emulator/adminSdkConfig";
77
import { needProjectId } from "../projectUtils";
88
import { FirebaseError } from "../error";
99
import * as ensureApiEnabled from "../ensureApiEnabled";
1010
import { runtimeconfigOrigin } from "../api";
11-
import * as experiments from "../experiments";
1211
import { getFunctionsConfig } from "../deploy/functions/prepareFunctionsUpload";
1312

1413
export const command = new Command("internaltesting:functions:discover")
@@ -24,9 +23,8 @@ export const command = new Command("internaltesting:functions:discover")
2423
}
2524

2625
let runtimeConfig: Record<string, unknown> = { firebase: firebaseConfig };
27-
const allowFunctionsConfig = experiments.isEnabled("dangerouslyAllowFunctionsConfig");
2826

29-
if (allowFunctionsConfig) {
27+
if (fnConfig.some(shouldUseRuntimeConfig)) {
3028
try {
3129
const runtimeConfigApiEnabled = await ensureApiEnabled.check(
3230
projectId,

src/deploy/functions/prepare.spec.ts

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,58 @@ describe("prepare", () => {
9999

100100
expect(builds.codebase.runtime).to.equal("nodejs20");
101101
});
102+
103+
it("should pass only firebase config when disallowLegacyRuntimeConfig is true", async () => {
104+
const config: ValidatedConfig = [
105+
{
106+
source: "source",
107+
codebase: "codebase",
108+
disallowLegacyRuntimeConfig: true,
109+
runtime: "nodejs22",
110+
},
111+
];
112+
const options = {
113+
config: {
114+
path: (p: string) => p,
115+
},
116+
projectId: "project",
117+
} as unknown as Options;
118+
const firebaseConfig = { projectId: "project" };
119+
const runtimeConfig = { firebase: firebaseConfig, customKey: "customValue" };
120+
121+
await prepare.loadCodebases(config, options, firebaseConfig, runtimeConfig);
122+
123+
expect(discoverBuildStub.calledOnce).to.be.true;
124+
const callArgs = discoverBuildStub.firstCall.args;
125+
expect(callArgs[0]).to.deep.equal({ firebase: firebaseConfig });
126+
expect(callArgs[0]).to.not.have.property("customKey");
127+
});
128+
129+
it("should pass full runtime config when disallowLegacyRuntimeConfig is false", async () => {
130+
const config: ValidatedConfig = [
131+
{
132+
source: "source",
133+
codebase: "codebase",
134+
disallowLegacyRuntimeConfig: false,
135+
runtime: "nodejs22",
136+
},
137+
];
138+
const options = {
139+
config: {
140+
path: (p: string) => p,
141+
},
142+
projectId: "project",
143+
} as unknown as Options;
144+
const firebaseConfig = { projectId: "project" };
145+
const runtimeConfig = { firebase: firebaseConfig, customKey: "customValue" };
146+
147+
await prepare.loadCodebases(config, options, firebaseConfig, runtimeConfig);
148+
149+
expect(discoverBuildStub.calledOnce).to.be.true;
150+
const callArgs = discoverBuildStub.firstCall.args;
151+
expect(callArgs[0]).to.deep.equal(runtimeConfig);
152+
expect(callArgs[0]).to.have.property("customKey", "customValue");
153+
});
102154
});
103155

104156
describe("inferDetailsFromExisting", () => {

src/deploy/functions/prepare.ts

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import * as runtimes from "./runtimes";
1111
import * as supported from "./runtimes/supported";
1212
import * as validate from "./validate";
1313
import * as ensure from "./ensure";
14-
import * as experiments from "../../experiments";
1514
import {
1615
functionsOrigin,
1716
artifactRegistryDomain,
@@ -43,6 +42,7 @@ import {
4342
normalizeAndValidate,
4443
ValidatedConfig,
4544
requireLocal,
45+
shouldUseRuntimeConfig,
4646
} from "../../functions/projectConfig";
4747
import { AUTH_BLOCKING_EVENTS } from "../../functions/events/v1";
4848
import { generateServiceIdentity } from "../../gcp/serviceusage";
@@ -94,10 +94,11 @@ export async function prepare(
9494

9595
// ===Phase 1. Load codebases from source with optional runtime config.
9696
let runtimeConfig: Record<string, unknown> = { firebase: firebaseConfig };
97-
const allowFunctionsConfig = experiments.isEnabled("dangerouslyAllowFunctionsConfig");
9897

99-
// Load runtime config if experiment allows it and API is enabled
100-
if (allowFunctionsConfig && checkAPIsEnabled[1]) {
98+
const targetedCodebaseConfigs = context.config!.filter((cfg) => codebases.includes(cfg.codebase));
99+
100+
// Load runtime config if API is enabled and at least one targeted codebase uses it
101+
if (checkAPIsEnabled[1] && targetedCodebaseConfigs.some(shouldUseRuntimeConfig)) {
101102
runtimeConfig = { ...runtimeConfig, ...(await getFunctionsConfig(projectId)) };
102103
}
103104

@@ -228,7 +229,8 @@ export async function prepare(
228229
source.functionsSourceV2Hash = packagedSource?.hash;
229230
}
230231
if (backend.someEndpoint(wantBackend, (e) => e.platform === "gcfv1")) {
231-
const packagedSource = await prepareFunctionsUpload(sourceDir, localCfg, runtimeConfig);
232+
const configForUpload = shouldUseRuntimeConfig(localCfg) ? runtimeConfig : undefined;
233+
const packagedSource = await prepareFunctionsUpload(sourceDir, localCfg, configForUpload);
232234
source.functionsSourceV1 = packagedSource?.pathToSource;
233235
source.functionsSourceV1Hash = packagedSource?.hash;
234236
}
@@ -486,7 +488,12 @@ export async function loadCodebases(
486488
"functions",
487489
`Loading and analyzing source code for codebase ${codebase} to determine what to deploy`,
488490
);
489-
const discoveredBuild = await runtimeDelegate.discoverBuild(runtimeConfig, {
491+
492+
const codebaseRuntimeConfig = shouldUseRuntimeConfig(codebaseConfig)
493+
? runtimeConfig
494+
: { firebase: firebaseConfig };
495+
496+
const discoveredBuild = await runtimeDelegate.discoverBuild(codebaseRuntimeConfig, {
490497
...firebaseEnvs,
491498
// Quota project is required when using GCP's Client-based APIs
492499
// Some GCP client SDKs, like Vertex AI, requires appropriate quota project setup

src/experiments.ts

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -57,14 +57,6 @@ export const ALL_EXPERIMENTS = experiments({
5757
"of how that image was created.",
5858
public: true,
5959
},
60-
dangerouslyAllowFunctionsConfig: {
61-
shortDescription: "Allows the use of deprecated functions.config() API",
62-
fullDescription:
63-
"The functions.config() API is deprecated and will be removed on December 31, 2025. " +
64-
"This experiment allows continued use of the API during the migration period.",
65-
default: true,
66-
public: true,
67-
},
6860
runfunctions: {
6961
shortDescription:
7062
"Functions created using the V2 API target Cloud Run Functions (not production ready)",

0 commit comments

Comments
 (0)