diff --git a/src/apphosting/config.spec.ts b/src/apphosting/config.spec.ts index b4e87a15488..2db4e5ef3f4 100644 --- a/src/apphosting/config.spec.ts +++ b/src/apphosting/config.spec.ts @@ -55,6 +55,12 @@ describe("config", () => { expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd"); }); + + it("discovers backend root from any env file", () => { + fs.listFiles.withArgs("/parent/cwd").returns([".env"]); + + expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd"); + }); }); describe("get/setEnv", () => { diff --git a/src/apphosting/config.ts b/src/apphosting/config.ts index c8487626980..09636d9f9f3 100644 --- a/src/apphosting/config.ts +++ b/src/apphosting/config.ts @@ -63,7 +63,7 @@ export function discoverBackendRoot(cwd: string): string | null { while (true) { const files = fs.listFiles(dir); - if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file))) { + if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file) || file === ".env")) { return dir; } diff --git a/src/apphosting/env.spec.ts b/src/apphosting/env.spec.ts new file mode 100644 index 00000000000..8db28f8de25 --- /dev/null +++ b/src/apphosting/env.spec.ts @@ -0,0 +1,188 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as env from "./env"; +import * as promptNS from "../prompt"; +import * as config from "./config"; +import * as gcsmNS from "../gcp/secretManager"; +import * as secretsNS from "./secrets"; +import * as utilsNS from "../utils"; +import { Document } from "yaml"; + +describe("env", () => { + let prompt: sinon.SinonStubbedInstance; + let gcsm: sinon.SinonStubbedInstance; + let secrets: sinon.SinonStubbedInstance; + let utils: sinon.SinonStubbedInstance; + + function makeDocument(...envs: config.Env[]): Document { + const doc = new Document(); + for (const e of envs) { + config.upsertEnv(doc, e); + } + return doc; + } + + beforeEach(() => { + prompt = sinon.stub(promptNS); + gcsm = sinon.stub(gcsmNS); + secrets = sinon.stub(secretsNS); + utils = sinon.stub(utilsNS); + + utils.logLabeledWarning.resolves(); + prompt.input.rejects(new Error("Should not be called")); + gcsm.accessSecretVersion.rejects(new Error("Should not be called")); + gcsm.addVersion.rejects(new Error("Should not be called")); + secrets.upsertSecret.rejects(new Error("Should not be called")); + }); + + afterEach(() => { + sinon.verifyAndRestore(); + }); + + it("should diffEnvs", async () => { + gcsm.accessSecretVersion + .withArgs("test-project", "matching-secret", "latest") + .resolves("unchanged"); + gcsm.accessSecretVersion + .withArgs("test-project", "changed-secret", "latest") + .resolves("original-value"); + gcsm.accessSecretVersion + .withArgs("test-project", "error-secret", "latest") + .rejects(new Error("Cannot access secret")); + + const existingEnv = makeDocument( + { variable: "MATCHING_PLAIN", value: "existing" }, + { variable: "CHANGED_PLAIN", value: "original-value" }, + { variable: "UNREFERENCED_PLAIN", value: "existing" }, + + { variable: "MATCHING_SECRET", secret: "matching-secret" }, + { variable: "UNREFERENCED_SECRET", secret: "unreferenced-secret" }, + { variable: "CHANGED_SECRET", secret: "changed-secret" }, + { variable: "ERROR_SECRET", secret: "error-secret" }, + ); + + const importingEnv = { + MATCHING_PLAIN: "existing", + CHANGED_PLAIN: "new-value", + NEW_PLAIN: "new", + + MATCHING_SECRET: "unchanged", + NEW_SECRET: "new", + CHANGED_SECRET: "changed-value", + ERROR_SECRET: "attempted-value", + }; + + await expect(env.diffEnvs("test-project", importingEnv, existingEnv)).to.eventually.deep.equal({ + newVars: ["NEW_PLAIN", "NEW_SECRET"], + matched: ["MATCHING_PLAIN", "MATCHING_SECRET"], + conflicts: ["CHANGED_PLAIN", "CHANGED_SECRET", "ERROR_SECRET"], + }); + expect(gcsm.accessSecretVersion).to.have.been.calledThrice; + expect(utils.logLabeledWarning).to.have.been.calledWith( + "apphosting", + "Cannot read value of existing secret error-secret to see if it has changed. Assuming it has changed.", + ); + }); + + describe("confirmConflicts", () => { + it("should return an empty array if no conflicts", async () => { + const result = await env.confirmConflicts([]); + expect(result).to.be.empty; + }); + + it("should prompt the user to resolve conflicts", async () => { + prompt.checkbox.resolves(["FOO"]); + const result = await env.confirmConflicts(["FOO", "BAZ"]); + expect(result).to.deep.equal(["FOO"]); + expect(prompt.checkbox).to.have.been.calledOnce; + }); + }); + + describe("chooseNewSecrets", () => { + it("should return an empty array if no vars", async () => { + const result = await env.chooseNewSecrets([]); + expect(result).to.be.empty; + }); + + it("should suggest which values to store as secrets", async () => { + prompt.checkbox.resolves(["MY_KEY"]); + const result = await env.chooseNewSecrets(["FOO", "BAZ", "MY_KEY", "MY_SECRET"]); + expect(result).to.deep.equal(["MY_KEY"]); + expect(prompt.checkbox).to.have.been.calledWithMatch({ + message: + "Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?", + choices: [ + { value: "FOO", checked: false }, + { value: "BAZ", checked: false }, + { value: "MY_KEY", checked: true }, + { value: "MY_SECRET", checked: true }, + ], + }); + }); + }); + + describe("importEnv", () => { + // We could break this into multiple tests, but the same code is execrcised in all cases. + it("should keep existing secrets as secrets, prompt for new vars to be secrets, and store only selected info", async () => { + const existingEnv = makeDocument( + { variable: "EXISTING_PLAIN1", value: "existing" }, + { variable: "EXISTING_PLAIN2", value: "existing" }, + { variable: "EXISTING_SECRET1", secret: "existing-secret1" }, + { variable: "EXISTING_SECRET2", secret: "existing-secret2" }, + ); + + const importingEnv = { + EXISTING_PLAIN1: "new", + EXISTING_PLAIN2: "new", + NEW_PLAIN: "new", + EXISTING_SECRET1: "new", + EXISTING_SECRET2: "new", + NEW_SECRET: "new", + }; + + sinon.stub(env, "diffEnvs").resolves({ + newVars: ["NEW_PLAIN", "NEW_SECRET"], + conflicts: ["EXISTING_PLAIN1", "EXISTING_PLAIN2", "EXISTING_SECRET1", "EXISTING_SECRET2"], + matched: [], + }); + // Leave #2 alone and verify that they haven't been modified + sinon.stub(env, "confirmConflicts").resolves(["EXISTING_PLAIN1", "EXISTING_SECRET1"]); + // Verify that only new variables are offered to be stored as secrets + sinon.stub(env, "chooseNewSecrets").resolves(["NEW_SECRET"]); + secrets.upsertSecret.withArgs("test-project", "NEW_SECRET").resolves(); + gcsm.addVersion.withArgs("test-project", "NEW_SECRET", "new").resolves(); + gcsm.addVersion.withArgs("test-project", "existing-secret1", "new").resolves(); + + const createdSecrets = await env.importEnv("test-project", importingEnv, existingEnv); + + // Confirm new variables are not part of the confirm prompt + expect(env.confirmConflicts).calledWithMatch([ + "EXISTING_PLAIN1", + "EXISTING_PLAIN2", + "EXISTING_SECRET1", + "EXISTING_SECRET2", + ]); + + // Confirm that variables which already existed are not asked to be stored as secrets + expect(env.chooseNewSecrets).calledWithMatch(["NEW_PLAIN", "NEW_SECRET"]); + + // Confirm that we don't unnecessarily upsert existing secrets + expect(secrets.upsertSecret).to.have.been.calledOnceWith("test-project", "NEW_SECRET"); + + // Confirm that we updated the versions of the secrets we were asked to update + expect(gcsm.addVersion).to.have.been.calledWith("test-project", "NEW_SECRET", "new"); + expect(gcsm.addVersion).to.have.been.calledWith("test-project", "existing-secret1", "new"); + + // Confirm that we return the list of created secrets for furhter IAM granting + expect(createdSecrets).to.deep.equal(["NEW_SECRET"]); + + // Confirm that the existing env was properly updated + expect(config.findEnv(existingEnv, "EXISTING_PLAIN1")?.value).to.equal("new"); + expect(config.findEnv(existingEnv, "EXISTING_PLAIN2")?.value).to.equal("existing"); + expect(config.findEnv(existingEnv, "NEW_PLAIN")?.value).to.equal("new"); + expect(config.findEnv(existingEnv, "EXISTING_SECRET1")?.secret).to.equal("existing-secret1"); + expect(config.findEnv(existingEnv, "EXISTING_SECRET2")?.secret).to.equal("existing-secret2"); + expect(config.findEnv(existingEnv, "NEW_SECRET")?.secret).to.equal("NEW_SECRET"); + }); + }); +}); diff --git a/src/apphosting/env.ts b/src/apphosting/env.ts new file mode 100644 index 00000000000..b8aa42096f6 --- /dev/null +++ b/src/apphosting/env.ts @@ -0,0 +1,149 @@ +import * as clc from "colorette"; + +import { FirebaseError } from "../error"; +import * as secretManager from "../gcp/secretManager"; +import * as prompt from "../prompt"; +import * as config from "./config"; +import { Document } from "yaml"; +import * as secrets from "./secrets"; +import * as utils from "../utils"; + +const dynamicDispatch = exports as { + diffEnvs: typeof diffEnvs; + confirmConflicts: typeof confirmConflicts; + chooseNewSecrets: typeof chooseNewSecrets; +}; + +export interface DiffResults { + newVars: string[]; + matched: string[]; + conflicts: string[]; +} + +export async function diffEnvs( + projectId: string, + envs: Record, + doc: Document, +): Promise { + const newVars: string[] = []; + const matched: string[] = []; + const conflicts: string[] = []; + + await Promise.all( + Object.entries(envs).map(async ([key, value]) => { + const existingEnv = config.findEnv(doc, key); + if (!existingEnv) { + newVars.push(key); + return; + } + + let match = false; + if (existingEnv.value) { + match = existingEnv.value === value; + } else { + try { + match = + value === + (await secretManager.accessSecretVersion(projectId, existingEnv.secret!, "latest")); + } catch (err) { + utils.logLabeledWarning( + "apphosting", + `Cannot read value of existing secret ${existingEnv.secret!} to see if it has changed. Assuming it has changed.`, + ); + } + } + + (match ? matched : conflicts).push(key); + }), + ); + + return { newVars, matched, conflicts }; +} + +export async function confirmConflicts(conflicts: string[]): Promise { + if (!conflicts.length) { + return conflicts; + } + + const overwrite = await prompt.checkbox({ + message: + "The following variables have different values in apphosting.yaml. Which would you like to overwrite?", + choices: conflicts, + }); + return overwrite; +} + +export async function chooseNewSecrets(vars: string[]): Promise { + if (!vars.length) { + return vars; + } + + return await prompt.checkbox({ + message: + "Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?", + choices: vars.map((name) => ({ + value: name, + checked: name.includes("KEY") || name.includes("SECRET"), + })), + }); +} + +/** + * Merges a .env file with a YAML document including uploading, but not necessarily granting permission, to secrets. + * We're using a YAML doc and not worrying about file saving or granting permissions so that the caller can swap out whether + * this is a local yaml (for which env) or whether this is for remote env. + * @returns A list of secrets which were created and may need access granted. + */ +export async function importEnv( + projectId: string, + envs: Record, + doc: Document, +): Promise { + let { newVars, conflicts } = await dynamicDispatch.diffEnvs(projectId, envs, doc); + + conflicts = await dynamicDispatch.confirmConflicts(conflicts); + const newSecrets = await dynamicDispatch.chooseNewSecrets(newVars); + + for (const key of conflicts) { + const existingEnv = config.findEnv(doc, key); + if (!existingEnv) { + throw new FirebaseError(`Internal error: expected existing env for ${key}`, { exit: 1 }); + } + if (existingEnv.value) { + existingEnv.value = envs[key]; + config.upsertEnv(doc, existingEnv); + } else { + const secretValue = envs[key]; + const version = await secretManager.addVersion(projectId, existingEnv.secret!, secretValue); + utils.logSuccess( + `Created new secret version ${secretManager.toSecretVersionResourceName(version)}`, + ); + } + } + + const newPlaintext = newVars.filter((v) => !newSecrets.includes(v)); + for (const key of newPlaintext) { + config.upsertEnv(doc, { variable: key, value: envs[key] }); + } + + // NOTE: not doing this in parallel to avoid interleaving log lines in a way that might be confusing. + for (const key of newSecrets) { + // TODO: (How) do we support secrets in a specific location? Not investing deeply right now since everything in App Hosting + // is curreently global jurrisdiction and we may be chaging to REP managing secrets locality instead of UMMR anyway. + const created = await secrets.upsertSecret(projectId, key); + if (created) { + utils.logSuccess(`Created new secret projects/${projectId}/secrets/${key}`); + } + + const version = await secretManager.addVersion(projectId, key, envs[key]); + utils.logSuccess( + `Created new secret version ${secretManager.toSecretVersionResourceName(version)}`, + ); + utils.logBullet( + `You can access the contents of the secret's latest value with ${clc.bold(`firebase apphosting:secrets:access ${key}\n`)}`, + ); + + config.upsertEnv(doc, { variable: key, secret: key }); + } + return newSecrets; +} diff --git a/src/commands/apphosting-env-import.ts b/src/commands/apphosting-env-import.ts new file mode 100644 index 00000000000..c1af0b1877d --- /dev/null +++ b/src/commands/apphosting-env-import.ts @@ -0,0 +1,118 @@ +import * as clc from "colorette"; + +import { Command } from "../command"; +import { Options } from "../options"; +import { requireAuth } from "../requireAuth"; +import { importEnv } from "../apphosting/env"; +import { needProjectId, needProjectNumber } from "../projectUtils"; +import { fileExistsSync } from "../fsutils"; +import { FirebaseError } from "../error"; +import { promises as fs } from "fs"; +import * as path from "path"; +import * as functionsEnv from "../functions/env"; +import * as config from "../apphosting/config"; +import * as prompt from "../prompt"; +import { requirePermissions } from "../requirePermissions"; +import * as gcsm from "../gcp/secretManager"; +import * as secrets from "../apphosting/secrets"; +import * as dialogs from "../apphosting/secrets/dialogs"; +import * as utils from "../utils"; + +export const command = new Command("apphosting:env:import") + .description("import environment variables from a .env file into your apphosting.yaml") + .option("--source ", "path to .env file", "") + .option("--output ", "path to apphosting yaml file", "") + .before(requireAuth) + .before(gcsm.ensureApi) + .before(requirePermissions, [ + "secretmanager.secrets.create", + "secretmanager.secrets.get", + "secretmanager.secrets.update", + "secretmanager.versions.add", + "secretmanager.secrets.getIamPolicy", + "secretmanager.secrets.setIamPolicy", + ]) + .action(async (options: Options) => { + const projectId = needProjectId(options); + const projectNumber = await needProjectNumber(options); + const source = options.source as string; + let envFilePath: string; + let projectRoot: string; + if (source) { + envFilePath = path.resolve(source); + projectRoot = path.dirname(envFilePath); + } else { + const temp = config.discoverBackendRoot(process.cwd()); + if (!temp) { + throw new FirebaseError( + "Could not find .env file. Please specify the path to your .env file with the --source flag.", + ); + } + projectRoot = temp; + envFilePath = path.join(projectRoot, ".env"); + } + + if (!fileExistsSync(envFilePath)) { + throw new FirebaseError("Could not find .env file. Please specify with the --source flag."); + } + + const envFileContent = await fs.readFile(envFilePath, "utf8"); + const { envs, errors } = functionsEnv.parse(envFileContent); + + if (errors.length > 0) { + throw new FirebaseError(`Invalid .env file: ${errors.join(", ")}`); + } + + // NOTE: When we add a --backend option, we can use a yaml.Document in memory with the same utilities, + // but then just publish the values to the server. + let outputFile = options.output as string; + if (!outputFile) { + const environment = await prompt.input( + "What environment would you like to import to? Leave blank for all environments, use 'emulator' to affect the emulator", + ); + outputFile = environment ? `apphosting.${environment}.yaml` : "apphosting.yaml"; + } + + if (!path.isAbsolute(outputFile)) { + outputFile = path.resolve(projectRoot, outputFile); + } + const doc = config.load(outputFile); + + const newSecrets = await importEnv(projectId, envs, doc); + + if (outputFile.endsWith(".local.yaml") || outputFile.endsWith(".emulator.yaml")) { + const emailList = await prompt.input({ + message: + "Please enter a comma separated list of user or groups who should have access to this secret:", + }); + if (emailList.length) { + await secrets.grantEmailsSecretAccess(projectId, newSecrets, emailList.split(",")); + } else { + utils.logBullet( + "To grant access in the future run " + + clc.bold( + `firebase apphosting:secrets:grantaccess ${newSecrets.join(",")} --emails [email list]`, + ), + ); + } + config.store(outputFile, doc); + return; + } + + const accounts = await dialogs.selectBackendServiceAccounts(projectNumber, projectId, options); + + // If we're not granting permissions, there's no point in adding to YAML either. + if (!accounts.buildServiceAccounts.length && !accounts.runServiceAccounts.length) { + utils.logWarning( + `To use this secret in your backend, you must grant access. You can do so in the future with ${clc.bold("firebase apphosting:secrets:grantaccess")}`, + ); + } else { + await Promise.all( + newSecrets.map((secretName) => + secrets.grantSecretAccess(projectId, projectNumber, secretName, accounts), + ), + ); + } + + config.store(outputFile, doc); + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index 7e346426623..d4192e25a26 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -193,6 +193,8 @@ export function load(client: any): any { client.apphosting.repos.create = loadCommand("apphosting-repos-create"); client.apphosting.rollouts.list = loadCommand("apphosting-rollouts-list"); } + client.apphosting.env = {}; + client.apphosting.env.import = loadCommand("apphosting-env-import"); } client.login = loadCommand("login"); client.login.add = loadCommand("login-add");