Skip to content

Commit ea02e90

Browse files
inlinedinlined
committed
Add new apphosting:env:import commmand
Approved by committee ages ago. This will expand to remote env by just adding a --backend flag and can also include a dialog if neither backend nor apphosting.yaml is found. Only controversial change is that I updated the discoverBackendRoot to also look for .env files. This seems to also be a pretty good indication of the root of a website and helps make sure that this command is always operating on the same directory.
1 parent af64adf commit ea02e90

File tree

6 files changed

+464
-2
lines changed

6 files changed

+464
-2
lines changed

‎src/apphosting/config.spec.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@ describe("config", () => {
5555

5656
expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd");
5757
});
58+
59+
it("discovers backend root from any env file", () => {
60+
fs.listFiles.withArgs("/parent/cwd").returns([".env"]);
61+
62+
expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd");
63+
});
5864
});
5965

6066
describe("get/setEnv", () => {

‎src/apphosting/config.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { join, dirname } from "path";
1+
import { join, dirname, basename } from "path";
22
import { writeFileSync } from "fs";
33
import * as yaml from "yaml";
44
import * as clc from "colorette";
@@ -63,7 +63,7 @@ export function discoverBackendRoot(cwd: string): string | null {
6363

6464
while (true) {
6565
const files = fs.listFiles(dir);
66-
if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file))) {
66+
if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file) || basename(file) === ".env")) {
6767
return dir;
6868
}
6969

‎src/apphosting/env.spec.ts‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import { expect } from "chai";
2+
import * as sinon from "sinon";
3+
import * as env from "./env";
4+
import * as promptNS from "../prompt";
5+
import * as config from "./config";
6+
import * as gcsmNS from "../gcp/secretManager";
7+
import * as secretsNS from "./secrets";
8+
import * as utilsNS from "../utils";
9+
import { Document } from "yaml";
10+
11+
describe("env", () => {
12+
let prompt: sinon.SinonStubbedInstance<typeof promptNS>;
13+
let gcsm: sinon.SinonStubbedInstance<typeof gcsmNS>;
14+
let secrets: sinon.SinonStubbedInstance<typeof secretsNS>;
15+
let utils: sinon.SinonStubbedInstance<typeof utilsNS>;
16+
17+
function makeDocument(...envs: config.Env[]): Document {
18+
const doc = new Document();
19+
for (const e of envs) {
20+
config.upsertEnv(doc, e);
21+
}
22+
return doc;
23+
}
24+
25+
beforeEach(() => {
26+
prompt = sinon.stub(promptNS);
27+
gcsm = sinon.stub(gcsmNS);
28+
secrets = sinon.stub(secretsNS);
29+
utils = sinon.stub(utilsNS);
30+
31+
utils.logLabeledWarning.resolves();
32+
prompt.input.rejects(new Error("Should not be called"));
33+
gcsm.accessSecretVersion.rejects(new Error("Should not be called"));
34+
gcsm.addVersion.rejects(new Error("Should not be called"));
35+
secrets.upsertSecret.rejects(new Error("Should not be called"));
36+
});
37+
38+
afterEach(() => {
39+
sinon.verifyAndRestore();
40+
});
41+
42+
it("should diffEnvs", async () => {
43+
gcsm.accessSecretVersion
44+
.withArgs("test-project", "matching-secret", "latest")
45+
.resolves("unchanged");
46+
gcsm.accessSecretVersion
47+
.withArgs("test-project", "changed-secret", "latest")
48+
.resolves("original-value");
49+
gcsm.accessSecretVersion
50+
.withArgs("test-project", "error-secret", "latest")
51+
.rejects(new Error("Cannot access secret"));
52+
53+
const existingEnv = makeDocument(
54+
{ variable: "MATCHING_PLAIN", value: "existing" },
55+
{ variable: "CHANGED_PLAIN", value: "original-value" },
56+
{ variable: "UNREFERENCED_PLAIN", value: "existing" },
57+
58+
{ variable: "MATCHING_SECRET", secret: "matching-secret" },
59+
{ variable: "UNREFERENCED_SECRET", secret: "unreferenced-secret" },
60+
{ variable: "CHANGED_SECRET", secret: "changed-secret" },
61+
{ variable: "ERROR_SECRET", secret: "error-secret" },
62+
);
63+
64+
const importingEnv = {
65+
MATCHING_PLAIN: "existing",
66+
CHANGED_PLAIN: "new-value",
67+
NEW_PLAIN: "new",
68+
69+
MATCHING_SECRET: "unchanged",
70+
NEW_SECRET: "new",
71+
CHANGED_SECRET: "changed-value",
72+
ERROR_SECRET: "attempted-value",
73+
};
74+
75+
await expect(env.diffEnvs("test-project", importingEnv, existingEnv)).to.eventually.deep.equal({
76+
newVars: ["NEW_PLAIN", "NEW_SECRET"],
77+
matched: ["MATCHING_PLAIN", "MATCHING_SECRET"],
78+
conflicts: ["CHANGED_PLAIN", "CHANGED_SECRET", "ERROR_SECRET"],
79+
});
80+
expect(gcsm.accessSecretVersion).to.have.been.calledThrice;
81+
expect(utils.logLabeledWarning).to.have.been.calledWith(
82+
"apphosting",
83+
"Cannot read value of existing secret error-secret to see if it has changed. Assuming it has changed.",
84+
);
85+
});
86+
87+
describe("confirmConflicts", () => {
88+
it("should return an empty array if no conflicts", async () => {
89+
const result = await env.confirmConflicts([]);
90+
expect(result).to.be.empty;
91+
});
92+
93+
it("should prompt the user to resolve conflicts", async () => {
94+
prompt.checkbox.resolves(["FOO"]);
95+
const result = await env.confirmConflicts(["FOO", "BAZ"]);
96+
expect(result).to.deep.equal(["FOO"]);
97+
expect(prompt.checkbox).to.have.been.calledOnce;
98+
});
99+
});
100+
101+
describe("chooseNewSecrets", () => {
102+
it("should return an empty array if no vars", async () => {
103+
const result = await env.chooseNewSecrets([]);
104+
expect(result).to.be.empty;
105+
});
106+
107+
it("should suggest which values to store as secrets", async () => {
108+
prompt.checkbox.resolves(["MY_KEY"]);
109+
const result = await env.chooseNewSecrets(["FOO", "BAZ", "MY_KEY", "MY_SECRET"]);
110+
expect(result).to.deep.equal(["MY_KEY"]);
111+
expect(prompt.checkbox).to.have.been.calledWithMatch({
112+
message:
113+
"Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?",
114+
choices: [
115+
{ value: "FOO", checked: false },
116+
{ value: "BAZ", checked: false },
117+
{ value: "MY_KEY", checked: true },
118+
{ value: "MY_SECRET", checked: true },
119+
],
120+
});
121+
});
122+
});
123+
124+
describe("importEnv", () => {
125+
// We could break this into multiple tests, but the same code is execrcised in all cases.
126+
it("should keep existing secrets as secrets, prompt for new vars to be secrets, and store only selected info", async () => {
127+
const existingEnv = makeDocument(
128+
{ variable: "EXISTING_PLAIN1", value: "existing" },
129+
{ variable: "EXISTING_PLAIN2", value: "existing" },
130+
{ variable: "EXISTING_SECRET1", secret: "existing-secret1" },
131+
{ variable: "EXISTING_SECRET2", secret: "existing-secret2" },
132+
);
133+
134+
const importingEnv = {
135+
EXISTING_PLAIN1: "new",
136+
EXISTING_PLAIN2: "new",
137+
NEW_PLAIN: "new",
138+
EXISTING_SECRET1: "new",
139+
EXISTING_SECRET2: "new",
140+
NEW_SECRET: "new",
141+
};
142+
143+
sinon.stub(env, "diffEnvs").resolves({
144+
newVars: ["NEW_PLAIN", "NEW_SECRET"],
145+
conflicts: ["EXISTING_PLAIN1", "EXISTING_PLAIN2", "EXISTING_SECRET1", "EXISTING_SECRET2"],
146+
matched: [],
147+
});
148+
// Leave #2 alone and verify that they haven't been modified
149+
sinon.stub(env, "confirmConflicts").resolves(["EXISTING_PLAIN1", "EXISTING_SECRET1"]);
150+
// Verify that only new variables are offered to be stored as secrets
151+
sinon.stub(env, "chooseNewSecrets").resolves(["NEW_SECRET"]);
152+
secrets.upsertSecret.withArgs("test-project", "NEW_SECRET").resolves();
153+
gcsm.addVersion.withArgs("test-project", "NEW_SECRET", "new").resolves();
154+
gcsm.addVersion.withArgs("test-project", "existing-secret1", "new").resolves();
155+
156+
const createdSecrets = await env.importEnv("test-project", importingEnv, existingEnv);
157+
158+
// Confirm new variables are not part of the confirm prompt
159+
expect(env.confirmConflicts).calledWithMatch([
160+
"EXISTING_PLAIN1",
161+
"EXISTING_PLAIN2",
162+
"EXISTING_SECRET1",
163+
"EXISTING_SECRET2",
164+
]);
165+
166+
// Confirm that variables which already existed are not asked to be stored as secrets
167+
expect(env.chooseNewSecrets).calledWithMatch(["NEW_PLAIN", "NEW_SECRET"]);
168+
169+
// Confirm that we don't unnecessarily upsert existing secrets
170+
expect(secrets.upsertSecret).to.have.been.calledOnceWith("test-project", "NEW_SECRET");
171+
172+
// Confirm that we updated the versions of the secrets we were asked to update
173+
expect(gcsm.addVersion).to.have.been.calledWith("test-project", "NEW_SECRET", "new");
174+
expect(gcsm.addVersion).to.have.been.calledWith("test-project", "existing-secret1", "new");
175+
176+
// Confirm that we return the list of created secrets for furhter IAM granting
177+
expect(createdSecrets).to.deep.equal(["NEW_SECRET"]);
178+
179+
// Confirm that the existing env was properly updated
180+
expect(config.findEnv(existingEnv, "EXISTING_PLAIN1")?.value).to.equal("new");
181+
expect(config.findEnv(existingEnv, "EXISTING_PLAIN2")?.value).to.equal("existing");
182+
expect(config.findEnv(existingEnv, "NEW_PLAIN")?.value).to.equal("new");
183+
expect(config.findEnv(existingEnv, "EXISTING_SECRET1")?.secret).to.equal("existing-secret1");
184+
expect(config.findEnv(existingEnv, "EXISTING_SECRET2")?.secret).to.equal("existing-secret2");
185+
expect(config.findEnv(existingEnv, "NEW_SECRET")?.secret).to.equal("NEW_SECRET");
186+
});
187+
});
188+
});

‎src/apphosting/env.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import * as clc from "colorette";
2+
3+
import { FirebaseError } from "../error";
4+
import * as secretManager from "../gcp/secretManager";
5+
import * as prompt from "../prompt";
6+
import * as config from "./config";
7+
import { Document } from "yaml";
8+
import * as secrets from "./secrets";
9+
import * as utils from "../utils";
10+
11+
const dynamicDispatch = exports as {
12+
diffEnvs: typeof diffEnvs;
13+
confirmConflicts: typeof confirmConflicts;
14+
chooseNewSecrets: typeof chooseNewSecrets;
15+
};
16+
17+
export interface DiffResults {
18+
newVars: string[];
19+
matched: string[];
20+
conflicts: string[];
21+
}
22+
23+
export async function diffEnvs(
24+
projectId: string,
25+
envs: Record<string, string>,
26+
doc: Document,
27+
): Promise<DiffResults> {
28+
const newVars: string[] = [];
29+
const matched: string[] = [];
30+
const conflicts: string[] = [];
31+
32+
// Note: Can conceivably optimize this by parallelizing lookups of secret values with fetchSecrets.
33+
// Unlikely to actually cause noticeable benefits.
34+
for (const [key, value] of Object.entries(envs)) {
35+
const existingEnv = config.findEnv(doc, key);
36+
if (!existingEnv) {
37+
newVars.push(key);
38+
continue;
39+
}
40+
41+
let match = false;
42+
if (existingEnv.value) {
43+
match = existingEnv.value === value;
44+
} else {
45+
try {
46+
match =
47+
value ===
48+
(await secretManager.accessSecretVersion(projectId, existingEnv.secret!, "latest"));
49+
} catch (err) {
50+
utils.logLabeledWarning(
51+
"apphosting",
52+
`Cannot read value of existing secret ${existingEnv.secret!} to see if it has changed. Assuming it has changed.`,
53+
);
54+
}
55+
}
56+
57+
(match ? matched : conflicts).push(key);
58+
}
59+
return { newVars, matched, conflicts };
60+
}
61+
62+
export async function confirmConflicts(conflicts: string[]): Promise<string[]> {
63+
if (!conflicts.length) {
64+
return conflicts;
65+
}
66+
67+
const overwrite = await prompt.checkbox<string>({
68+
message:
69+
"The following variables have different values in apphosting.yaml. Which would you like to overwrite?",
70+
choices: conflicts,
71+
});
72+
return overwrite;
73+
}
74+
75+
export async function chooseNewSecrets(vars: string[]): Promise<string[]> {
76+
if (!vars.length) {
77+
return vars;
78+
}
79+
80+
return await prompt.checkbox<string>({
81+
message:
82+
"Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?",
83+
choices: vars.map((name) => ({
84+
value: name,
85+
checked: name.includes("KEY") || name.includes("SECRET"),
86+
})),
87+
});
88+
}
89+
90+
/**
91+
* Merges a .env file with a YAML document including uploading, but not necessarily granting permission, to secrets.
92+
* We're using a YAML doc and not worrying about file saving or granting permissions so that the caller can swap out whether
93+
* this is a local yaml (for which env) or whether this is for remote env.
94+
* @returns A list of secrets which were created and may need access granted.
95+
*/
96+
export async function importEnv(
97+
projectId: string,
98+
envs: Record<string, string>,
99+
doc: Document,
100+
): Promise<string[]> {
101+
let { newVars, conflicts } = await dynamicDispatch.diffEnvs(projectId, envs, doc);
102+
103+
conflicts = await dynamicDispatch.confirmConflicts(conflicts);
104+
const newSecrets = await dynamicDispatch.chooseNewSecrets(newVars);
105+
106+
for (const key of conflicts) {
107+
const existingEnv = config.findEnv(doc, key);
108+
if (!existingEnv) {
109+
throw new FirebaseError(`Internal error: expected existing env for ${key}`, { exit: 1 });
110+
}
111+
if (existingEnv.value) {
112+
existingEnv.value = envs[key];
113+
config.upsertEnv(doc, existingEnv);
114+
} else {
115+
const secretValue = envs[key];
116+
const version = await secretManager.addVersion(projectId, existingEnv.secret!, secretValue);
117+
utils.logSuccess(
118+
`Created new secret version ${secretManager.toSecretVersionResourceName(version)}`,
119+
);
120+
// TODO: What do we do if the YAML is pinned to a specific version?
121+
}
122+
}
123+
124+
const newPlaintext = newVars.filter((v) => !newSecrets.includes(v));
125+
for (const key of newPlaintext) {
126+
config.upsertEnv(doc, { variable: key, value: envs[key] });
127+
}
128+
129+
// NOTE: not doing this in parallel to avoid interleaving log lines in a way that might be confusing.
130+
for (const key of newSecrets) {
131+
// TODO: (How) do we support secrets in a specific location? Not investing deeply right now since everything in App Hosting
132+
// is curreently global jurrisdiction and we may be chaging to REP managing secrets locality instead of UMMR anyway.
133+
const created = await secrets.upsertSecret(projectId, key);
134+
if (created) {
135+
utils.logSuccess(`Created new secret projects/${projectId}/secrets/${key}`);
136+
}
137+
138+
const version = await secretManager.addVersion(projectId, key, envs[key]);
139+
utils.logSuccess(
140+
`Created new secret version ${secretManager.toSecretVersionResourceName(version)}`,
141+
);
142+
utils.logBullet(
143+
`You can access the contents of the secret's latest value with ${clc.bold(`firebase apphosting:secrets:access ${key}\n`)}`,
144+
);
145+
146+
config.upsertEnv(doc, { variable: key, secret: key });
147+
}
148+
return newSecrets;
149+
}

0 commit comments

Comments
 (0)