Add new apphosting:env:import commmand

src/apphosting/config.spec.ts

@@ -55,6 +55,12 @@ describe("config", () => {
 
       expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd");
     });
+
+    it("discovers backend root from any env file", () => {
+      fs.listFiles.withArgs("/parent/cwd").returns([".env"]);
+
+      expect(config.discoverBackendRoot("/parent/cwd")).equals("/parent/cwd");
+    });
   });
 
   describe("get/setEnv", () => {

src/apphosting/config.ts

@@ -1,4 +1,4 @@
-import { join, dirname } from "path";
+import { join, dirname, basename } from "path";
 import { writeFileSync } from "fs";
 import * as yaml from "yaml";
 import * as clc from "colorette";
@@ -61,9 +61,9 @@
 export function discoverBackendRoot(cwd: string): string | null {
   let dir = cwd;
 
   while (true) {
     const files = fs.listFiles(dir);
-    if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file))) {
+    if (files.some((file) => APPHOSTING_YAML_FILE_REGEX.test(file) || basename(file) === ".env")) {
       return dir;
     }
 
@@ -100,8 +100,8 @@
   let raw: string;
   try {
     raw = fs.readFile(yamlPath);
   } catch (err: any) {
     if (err.code !== "ENOENT") {
       throw new FirebaseError(`Unexpected error trying to load ${yamlPath}`, {
         original: getError(err),
       });
@@ -316,7 +316,7 @@
         default: suggestedTestKeyName(name),
       });
 
       if (await csm.secretExists(projectId!, secretRef)) {
         action = await prompt.select<"reuse" | "pick-new">({
           message:
             "This secret reference already exists, would you like to reuse it or create a new one?",
@@ -330,7 +330,7 @@
       }
     }
 
     newEnv[name] = { variable: name, secret: secretRef! };
     if (action === "reuse") {
       continue;
     }
@@ -339,13 +339,13 @@
       `What new value would you like for secret ${name} [input is hidden]?`,
     );
     // TODO: Do we need to support overriding locations? Inferring them from the original?
     await csm.createSecret(projectId!, secretRef!, { [csm.FIREBASE_MANAGED]: "apphosting" });
     await csm.addVersion(projectId!, secretRef!, secretValue);
   }
 
   return newEnv;
 }
 
 export function suggestedTestKeyName(variable: string): string {
   return "test-" + variable.replace(/_/g, "-").toLowerCase();
 }

src/apphosting/env.spec.ts

@@ -0,0 +1,188 @@
+import { expect } from "chai";
+import * as sinon from "sinon";
+import * as env from "./env";
+import * as promptNS from "../prompt";
+import * as config from "./config";
+import * as gcsmNS from "../gcp/secretManager";
+import * as secretsNS from "./secrets";
+import * as utilsNS from "../utils";
+import { Document } from "yaml";
+
+describe("env", () => {
+  let prompt: sinon.SinonStubbedInstance<typeof promptNS>;
+  let gcsm: sinon.SinonStubbedInstance<typeof gcsmNS>;
+  let secrets: sinon.SinonStubbedInstance<typeof secretsNS>;
+  let utils: sinon.SinonStubbedInstance<typeof utilsNS>;
+
+  function makeDocument(...envs: config.Env[]): Document {
+    const doc = new Document();
+    for (const e of envs) {
+      config.upsertEnv(doc, e);
+    }
+    return doc;
+  }
+
+  beforeEach(() => {
+    prompt = sinon.stub(promptNS);
+    gcsm = sinon.stub(gcsmNS);
+    secrets = sinon.stub(secretsNS);
+    utils = sinon.stub(utilsNS);
+
+    utils.logLabeledWarning.resolves();
+    prompt.input.rejects(new Error("Should not be called"));
+    gcsm.accessSecretVersion.rejects(new Error("Should not be called"));
+    gcsm.addVersion.rejects(new Error("Should not be called"));
+    secrets.upsertSecret.rejects(new Error("Should not be called"));
+  });
+
+  afterEach(() => {
+    sinon.verifyAndRestore();
+  });
+
+  it("should diffEnvs", async () => {
+    gcsm.accessSecretVersion
+      .withArgs("test-project", "matching-secret", "latest")
+      .resolves("unchanged");
+    gcsm.accessSecretVersion
+      .withArgs("test-project", "changed-secret", "latest")
+      .resolves("original-value");
+    gcsm.accessSecretVersion
+      .withArgs("test-project", "error-secret", "latest")
+      .rejects(new Error("Cannot access secret"));
+
+    const existingEnv = makeDocument(
+      { variable: "MATCHING_PLAIN", value: "existing" },
+      { variable: "CHANGED_PLAIN", value: "original-value" },
+      { variable: "UNREFERENCED_PLAIN", value: "existing" },
+
+      { variable: "MATCHING_SECRET", secret: "matching-secret" },
+      { variable: "UNREFERENCED_SECRET", secret: "unreferenced-secret" },
+      { variable: "CHANGED_SECRET", secret: "changed-secret" },
+      { variable: "ERROR_SECRET", secret: "error-secret" },
+    );
+
+    const importingEnv = {
+      MATCHING_PLAIN: "existing",
+      CHANGED_PLAIN: "new-value",
+      NEW_PLAIN: "new",
+
+      MATCHING_SECRET: "unchanged",
+      NEW_SECRET: "new",
+      CHANGED_SECRET: "changed-value",
+      ERROR_SECRET: "attempted-value",
+    };
+
+    await expect(env.diffEnvs("test-project", importingEnv, existingEnv)).to.eventually.deep.equal({
+      newVars: ["NEW_PLAIN", "NEW_SECRET"],
+      matched: ["MATCHING_PLAIN", "MATCHING_SECRET"],
+      conflicts: ["CHANGED_PLAIN", "CHANGED_SECRET", "ERROR_SECRET"],
+    });
+    expect(gcsm.accessSecretVersion).to.have.been.calledThrice;
+    expect(utils.logLabeledWarning).to.have.been.calledWith(
+      "apphosting",
+      "Cannot read value of existing secret error-secret to see if it has changed. Assuming it has changed.",
+    );
+  });
+
+  describe("confirmConflicts", () => {
+    it("should return an empty array if no conflicts", async () => {
+      const result = await env.confirmConflicts([]);
+      expect(result).to.be.empty;
+    });
+
+    it("should prompt the user to resolve conflicts", async () => {
+      prompt.checkbox.resolves(["FOO"]);
+      const result = await env.confirmConflicts(["FOO", "BAZ"]);
+      expect(result).to.deep.equal(["FOO"]);
+      expect(prompt.checkbox).to.have.been.calledOnce;
+    });
+  });
+
+  describe("chooseNewSecrets", () => {
+    it("should return an empty array if no vars", async () => {
+      const result = await env.chooseNewSecrets([]);
+      expect(result).to.be.empty;
+    });
+
+    it("should suggest which values to store as secrets", async () => {
+      prompt.checkbox.resolves(["MY_KEY"]);
+      const result = await env.chooseNewSecrets(["FOO", "BAZ", "MY_KEY", "MY_SECRET"]);
+      expect(result).to.deep.equal(["MY_KEY"]);
+      expect(prompt.checkbox).to.have.been.calledWithMatch({
+        message:
+          "Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?",
+        choices: [
+          { value: "FOO", checked: false },
+          { value: "BAZ", checked: false },
+          { value: "MY_KEY", checked: true },
+          { value: "MY_SECRET", checked: true },
+        ],
+      });
+    });
+  });
+
+  describe("importEnv", () => {
+    // We could break this into multiple tests, but the same code is execrcised in all cases.
+    it("should keep existing secrets as secrets, prompt for new vars to be secrets, and store only selected info", async () => {
+      const existingEnv = makeDocument(
+        { variable: "EXISTING_PLAIN1", value: "existing" },
+        { variable: "EXISTING_PLAIN2", value: "existing" },
+        { variable: "EXISTING_SECRET1", secret: "existing-secret1" },
+        { variable: "EXISTING_SECRET2", secret: "existing-secret2" },
+      );
+
+      const importingEnv = {
+        EXISTING_PLAIN1: "new",
+        EXISTING_PLAIN2: "new",
+        NEW_PLAIN: "new",
+        EXISTING_SECRET1: "new",
+        EXISTING_SECRET2: "new",
+        NEW_SECRET: "new",
+      };
+
+      sinon.stub(env, "diffEnvs").resolves({
+        newVars: ["NEW_PLAIN", "NEW_SECRET"],
+        conflicts: ["EXISTING_PLAIN1", "EXISTING_PLAIN2", "EXISTING_SECRET1", "EXISTING_SECRET2"],
+        matched: [],
+      });
+      // Leave #2 alone and verify that they haven't been modified
+      sinon.stub(env, "confirmConflicts").resolves(["EXISTING_PLAIN1", "EXISTING_SECRET1"]);
+      // Verify that only new variables are offered to be stored as secrets
+      sinon.stub(env, "chooseNewSecrets").resolves(["NEW_SECRET"]);
+      secrets.upsertSecret.withArgs("test-project", "NEW_SECRET").resolves();
+      gcsm.addVersion.withArgs("test-project", "NEW_SECRET", "new").resolves();
+      gcsm.addVersion.withArgs("test-project", "existing-secret1", "new").resolves();
+
+      const createdSecrets = await env.importEnv("test-project", importingEnv, existingEnv);
+
+      // Confirm new variables are not part of the confirm prompt
+      expect(env.confirmConflicts).calledWithMatch([
+        "EXISTING_PLAIN1",
+        "EXISTING_PLAIN2",
+        "EXISTING_SECRET1",
+        "EXISTING_SECRET2",
+      ]);
+
+      // Confirm that variables which already existed are not asked to be stored as secrets
+      expect(env.chooseNewSecrets).calledWithMatch(["NEW_PLAIN", "NEW_SECRET"]);
+
+      // Confirm that we don't unnecessarily upsert existing secrets
+      expect(secrets.upsertSecret).to.have.been.calledOnceWith("test-project", "NEW_SECRET");
+
+      // Confirm that we updated the versions of the secrets we were asked to update
+      expect(gcsm.addVersion).to.have.been.calledWith("test-project", "NEW_SECRET", "new");
+      expect(gcsm.addVersion).to.have.been.calledWith("test-project", "existing-secret1", "new");
+
+      // Confirm that we return the list of created secrets for furhter IAM granting
+      expect(createdSecrets).to.deep.equal(["NEW_SECRET"]);
+
+      // Confirm that the existing env was properly updated
+      expect(config.findEnv(existingEnv, "EXISTING_PLAIN1")?.value).to.equal("new");
+      expect(config.findEnv(existingEnv, "EXISTING_PLAIN2")?.value).to.equal("existing");
+      expect(config.findEnv(existingEnv, "NEW_PLAIN")?.value).to.equal("new");
+      expect(config.findEnv(existingEnv, "EXISTING_SECRET1")?.secret).to.equal("existing-secret1");
+      expect(config.findEnv(existingEnv, "EXISTING_SECRET2")?.secret).to.equal("existing-secret2");
+      expect(config.findEnv(existingEnv, "NEW_SECRET")?.secret).to.equal("NEW_SECRET");
+    });
+  });
+});

src/apphosting/env.ts

@@ -0,0 +1,148 @@
+import * as clc from "colorette";
+
+import { FirebaseError } from "../error";
+import * as secretManager from "../gcp/secretManager";
+import * as prompt from "../prompt";
+import * as config from "./config";
+import { Document } from "yaml";
+import * as secrets from "./secrets";
+import * as utils from "../utils";
+
+const dynamicDispatch = exports as {
+  diffEnvs: typeof diffEnvs;
+  confirmConflicts: typeof confirmConflicts;
+  chooseNewSecrets: typeof chooseNewSecrets;
+};
+
+export interface DiffResults {
+  newVars: string[];
+  matched: string[];
+  conflicts: string[];
+}
+
+export async function diffEnvs(
+  projectId: string,
+  envs: Record<string, string>,
+  doc: Document,
+): Promise<DiffResults> {
+  const newVars: string[] = [];
+  const matched: string[] = [];
+  const conflicts: string[] = [];
+
+  // Note: Can conceivably optimize this by parallelizing lookups of secret values with fetchSecrets.
+  // Unlikely to actually cause noticeable benefits.
+  for (const [key, value] of Object.entries(envs)) {
+    const existingEnv = config.findEnv(doc, key);
+    if (!existingEnv) {
+      newVars.push(key);
+      continue;
+    }
+
+    let match = false;
+    if (existingEnv.value) {
+      match = existingEnv.value === value;
+    } else {
+      try {
+        match =
+          value ===
+          (await secretManager.accessSecretVersion(projectId, existingEnv.secret!, "latest"));
+      } catch (err) {
+        utils.logLabeledWarning(
+          "apphosting",
+          `Cannot read value of existing secret ${existingEnv.secret!} to see if it has changed. Assuming it has changed.`,
+        );
+      }
+    }
+
+    (match ? matched : conflicts).push(key);
+  }
+  return { newVars, matched, conflicts };
+}
+
+export async function confirmConflicts(conflicts: string[]): Promise<string[]> {
+  if (!conflicts.length) {
+    return conflicts;
+  }
+
+  const overwrite = await prompt.checkbox<string>({
+    message:
+      "The following variables have different values in apphosting.yaml. Which would you like to overwrite?",
+    choices: conflicts,
+  });
+  return overwrite;
+}
+
+export async function chooseNewSecrets(vars: string[]): Promise<string[]> {
+  if (!vars.length) {
+    return vars;
+  }
+
+  return await prompt.checkbox<string>({
+    message:
+      "Sensitive data should be stored in Cloud Secrets Manager so that access to its value is protected. Which variables are sensitive?",
+    choices: vars.map((name) => ({
+      value: name,
+      checked: name.includes("KEY") || name.includes("SECRET"),
+    })),
+  });
+}
+
+/**
+ * Merges a .env file with a YAML document including uploading, but not necessarily granting permission, to secrets.
+ * We're using a YAML doc and not worrying about file saving or granting permissions so that the caller can swap out whether
+ * this is a local yaml (for which env) or whether this is for remote env.
+ * @returns A list of secrets which were created and may need access granted.
+ */
+export async function importEnv(
+  projectId: string,
+  envs: Record<string, string>,
+  doc: Document,
+): Promise<string[]> {
+  let { newVars, conflicts } = await dynamicDispatch.diffEnvs(projectId, envs, doc);
+
+  conflicts = await dynamicDispatch.confirmConflicts(conflicts);
+  const newSecrets = await dynamicDispatch.chooseNewSecrets(newVars);
+
+  for (const key of conflicts) {
+    const existingEnv = config.findEnv(doc, key);
+    if (!existingEnv) {
+      throw new FirebaseError(`Internal error: expected existing env for ${key}`, { exit: 1 });
+    }
+    if (existingEnv.value) {
+      existingEnv.value = envs[key];
+      config.upsertEnv(doc, existingEnv);
+    } else {
+      const secretValue = envs[key];
+      const version = await secretManager.addVersion(projectId, existingEnv.secret!, secretValue);
+      utils.logSuccess(
+        `Created new secret version ${secretManager.toSecretVersionResourceName(version)}`,
+      );
+    }
+  }
+
+  const newPlaintext = newVars.filter((v) => !newSecrets.includes(v));
+  for (const key of newPlaintext) {
+    config.upsertEnv(doc, { variable: key, value: envs[key] });
+  }
+
+  // NOTE: not doing this in parallel to avoid interleaving log lines in a way that might be confusing.
+  for (const key of newSecrets) {
+    // TODO: (How) do we support secrets in a specific location? Not investing deeply right now since everything in App Hosting
+    // is curreently global jurrisdiction and we may be chaging to REP managing secrets locality instead of UMMR anyway.
+    const created = await secrets.upsertSecret(projectId, key);
+    if (created) {
+      utils.logSuccess(`Created new secret projects/${projectId}/secrets/${key}`);
+    }
+
+    const version = await secretManager.addVersion(projectId, key, envs[key]);
+    utils.logSuccess(
+      `Created new secret version ${secretManager.toSecretVersionResourceName(version)}`,
+    );
+    utils.logBullet(
+      `You can access the contents of the secret's latest value with ${clc.bold(`firebase apphosting:secrets:access ${key}\n`)}`,
+    );
+
+    config.upsertEnv(doc, { variable: key, secret: key });
+  }
+  return newSecrets;
+}