Support webhooks in Data Connect behind an experiment flag. (#9440)

* Support webhooks in Data Connect behind an experiment flag.

* Update existing logic to specifically refer to the main schema.

* Fix build errors.

* lint

* Support loading schema sources from either `schema` or `schemas` field.

* Fix firebase init

* Properly deploy secondary schemas as well.

* Make `getSchema` take a schema ID, move secondary schema upsert after main schema upsert, and add some TODOs.

* Fix listSchemas in `firebase init`.

* Fix unit tests.

* Actually fix unit tests.

* Set default parameter for getSchema schemaId.

* Specify just top-level fields in listSchema call.

* Fix VSCode unit tests.

* Try to fix import path in VSCode.

Author: rosalyntan

firebase-vscode/src/data-connect/config.ts

@@ -1,6 +1,6 @@
 import { isPathInside } from "./file-utils";
 import { DeepReadOnly } from "../metaprogramming";
-import { ConnectorYaml, DataConnectYaml } from "../dataconnect/types";
+import { ConnectorYaml, DataConnectYaml, mainSchemaYaml } from "../../../src/dataconnect/types";
 import { Result, ResultValue } from "../result";
 import { computed, effect, signal } from "@preact/signals-core";
 import {
@@ -265,7 +265,7 @@ export class ResolvedDataConnectConfig {
   }
 
   get schemaDir(): string {
-    return this.value.schema.source;
+    return mainSchemaYaml(this.value).source;
   }
 
   get relativePath(): string {

src/commands/dataconnect-services-list.ts

@@ -8,6 +8,7 @@ import { requirePermissions } from "../requirePermissions";
 import { ensureApis } from "../dataconnect/ensureApis";
 import * as Table from "cli-table3";
 
+// TODO: Update this command to also list secondary schema information.
 export const command = new Command("dataconnect:services:list")
   .description("list all deployed Data Connect services")
   .before(requirePermissions, [

src/commands/dataconnect-sql-diff.ts

@@ -6,6 +6,7 @@ import { requirePermissions } from "../requirePermissions";
 import { pickService } from "../dataconnect/load";
 import { diffSchema } from "../dataconnect/schemaMigration";
 import { requireAuth } from "../requireAuth";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:diff [serviceId]")
   .description(
@@ -24,8 +25,8 @@ export const command = new Command("dataconnect:sql:diff [serviceId]")
 
     const diffs = await diffSchema(
       options,
-      serviceInfo.schema,
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation,
+      mainSchema(serviceInfo.schemas),
+      mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.schemaValidation,
     );
     return { projectId, serviceId, diffs };
   });

src/commands/dataconnect-sql-grant.ts

@@ -9,6 +9,7 @@ import { requireAuth } from "../requireAuth";
 import { FirebaseError } from "../error";
 import { fdcSqlRoleMap } from "../gcp/cloudsql/permissionsSetup";
 import { iamUserIsCSQLAdmin } from "../gcp/cloudsql/cloudsqladmin";
+import { mainSchema } from "../dataconnect/types";
 
 const allowedRoles = Object.keys(fdcSqlRoleMap);
 
@@ -51,6 +52,6 @@ export const command = new Command("dataconnect:sql:grant [serviceId]")
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
 
-    await grantRoleToUserInSchema(options, serviceInfo.schema);
+    await grantRoleToUserInSchema(options, mainSchema(serviceInfo.schemas));
     return { projectId, serviceId };
   });

src/commands/dataconnect-sql-migrate.ts

@@ -8,6 +8,7 @@ import { requireAuth } from "../requireAuth";
 import { requirePermissions } from "../requirePermissions";
 import { ensureApis } from "../dataconnect/ensureApis";
 import { logLabeledSuccess } from "../utils";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:migrate [serviceId]")
   .description("migrate your CloudSQL database's schema to match your local Data Connect schema")
@@ -23,18 +24,19 @@ export const command = new Command("dataconnect:sql:migrate [serviceId]")
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const instanceId =
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId;
+    const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql
+      .instanceId;
     if (!instanceId) {
       throw new FirebaseError(
         "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId",
       );
     }
     const diffs = await migrateSchema({
       options,
-      schema: serviceInfo.schema,
+      schema: mainSchema(serviceInfo.schemas),
       validateOnly: true,
-      schemaValidation: serviceInfo.dataConnectYaml.schema.datasource.postgresql?.schemaValidation,
+      schemaValidation: mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql
+        ?.schemaValidation,
     });
     if (diffs.length) {
       logLabeledSuccess(

src/commands/dataconnect-sql-setup.ts

@@ -10,6 +10,7 @@ import { setupSQLPermissions, getSchemaMetadata } from "../gcp/cloudsql/permissi
 import { DEFAULT_SCHEMA } from "../gcp/cloudsql/permissions";
 import { getIdentifiers, ensureServiceIsConnectedToCloudSql } from "../dataconnect/schemaMigration";
 import { setupIAMUsers } from "../gcp/cloudsql/connect";
+import { mainSchema, mainSchemaYaml } from "../dataconnect/types";
 
 export const command = new Command("dataconnect:sql:setup [serviceId]")
   .description("set up your CloudSQL database")
@@ -24,15 +25,17 @@ export const command = new Command("dataconnect:sql:setup [serviceId]")
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const instanceId =
-      serviceInfo.dataConnectYaml.schema.datasource.postgresql?.cloudSql.instanceId;
+    const instanceId = mainSchemaYaml(serviceInfo.dataConnectYaml).datasource.postgresql?.cloudSql
+      .instanceId;
     if (!instanceId) {
       throw new FirebaseError(
         "dataconnect.yaml is missing field schema.datasource.postgresql.cloudsql.instanceId",
       );
     }
 
-    const { serviceName, instanceName, databaseId } = getIdentifiers(serviceInfo.schema);
+    const { serviceName, instanceName, databaseId } = getIdentifiers(
+      mainSchema(serviceInfo.schemas),
+    );
     await ensureServiceIsConnectedToCloudSql(
       serviceName,
       instanceName,

src/commands/dataconnect-sql-shell.ts

@@ -17,6 +17,7 @@ import { logger } from "../logger";
 import { FirebaseError } from "../error";
 import { FBToolsAuthClient } from "../gcp/cloudsql/fbToolsAuthClient";
 import { confirmDangerousQuery, interactiveExecuteQuery } from "../gcp/cloudsql/interactive";
+import { mainSchema } from "../dataconnect/types";
 
 // Not a comprehensive list, used for keyword coloring.
 const sqlKeywords = [
@@ -91,7 +92,7 @@ export const command = new Command("dataconnect:sql:shell [serviceId]")
     const projectId = needProjectId(options);
     await ensureApis(projectId);
     const serviceInfo = await pickService(projectId, options.config, serviceId);
-    const { instanceId, databaseId } = getIdentifiers(serviceInfo.schema);
+    const { instanceId, databaseId } = getIdentifiers(mainSchema(serviceInfo.schemas));
     const { user: username } = await getIAMUser(options);
     const instance = await cloudSqlAdminClient.getInstance(projectId, instanceId);
 

src/dataconnect/client.spec.ts

@@ -124,6 +124,13 @@ describe("client", () => {
       expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/main");
     });
 
+    it("getSchema with schemaId", async () => {
+      getStub.resolves({ body: { name: "schema" } });
+      const schema = await client.getSchema("projects/p/locations/l/services/s", "schemaId");
+      expect(schema).to.deep.equal({ name: "schema" });
+      expect(getStub).to.be.calledWith("projects/p/locations/l/services/s/schemas/schemaId");
+    });
+
     it("getSchema returns undefined if not found", async () => {
       getStub.rejects(new FirebaseError("err", { status: 404 }));
       const schema = await client.getSchema("projects/p/locations/l/services/s");

src/dataconnect/client.ts

@@ -83,11 +83,12 @@ export async function deleteService(serviceName: string): Promise<types.Service>
 
 /** Schema methods */
 
-export async function getSchema(serviceName: string): Promise<types.Schema | undefined> {
+export async function getSchema(
+  serviceName: string,
+  schemaId: string = types.MAIN_SCHEMA_ID,
+): Promise<types.Schema | undefined> {
   try {
-    const res = await dataconnectClient().get<types.Schema>(
-      `${serviceName}/schemas/${types.SCHEMA_ID}`,
-    );
+    const res = await dataconnectClient().get<types.Schema>(`${serviceName}/schemas/${schemaId}`);
     return res.body;
   } catch (err: any) {
     if (err.status !== 404) {
@@ -146,7 +147,7 @@ export async function upsertSchema(
 
 export async function deleteSchema(serviceName: string): Promise<void> {
   const op = await dataconnectClient().delete<types.Schema>(
-    `${serviceName}/schemas/${types.SCHEMA_ID}`,
+    `${serviceName}/schemas/${types.MAIN_SCHEMA_ID}`,
   );
   await operationPoller.pollOperation<void>({
     apiOrigin: dataconnectOrigin(),

src/dataconnect/load.ts

@@ -7,7 +7,7 @@ import { Config } from "../config";
 import { FirebaseError } from "../error";
 import {
   toDatasource,
-  SCHEMA_ID,
+  MAIN_SCHEMA_ID,
   ConnectorYaml,
   DataConnectYaml,
   File,
@@ -16,6 +16,7 @@ import {
 } from "./types";
 import { readFileFromDirectory, wrappedSafeLoad } from "../utils";
 import { DataConnectMultiple } from "../firebaseConfig";
+import * as experiments from "../experiments";
 
 // pickService reads firebase.json and returns all services with a given serviceId.
 // If serviceID is not provided and there is a single service, return that.
@@ -77,8 +78,20 @@ export async function load(
   const resolvedDir = config.path(sourceDirectory);
   const dataConnectYaml = await readDataConnectYaml(resolvedDir);
   const serviceName = `projects/${projectId}/locations/${dataConnectYaml.location}/services/${dataConnectYaml.serviceId}`;
-  const schemaDir = path.join(resolvedDir, dataConnectYaml.schema.source);
-  const schemaGQLs = await readGQLFiles(schemaDir);
+  const schemaYamls = dataConnectYaml.schema ? [dataConnectYaml.schema] : dataConnectYaml.schemas;
+  const schemas = await Promise.all(
+    schemaYamls!.map(async (yaml) => {
+      const schemaDir = path.join(resolvedDir, yaml.source);
+      const schemaGQLs = await readGQLFiles(schemaDir);
+      return {
+        name: `${serviceName}/schemas/${yaml.id || MAIN_SCHEMA_ID}`,
+        datasources: [toDatasource(projectId, dataConnectYaml.location, yaml.datasource)],
+        source: {
+          files: schemaGQLs,
+        },
+      };
+    }),
+  );
   const connectorInfo = await Promise.all(
     dataConnectYaml.connectorDirs.map(async (dir) => {
       const connectorDir = path.join(resolvedDir, dir);
@@ -100,15 +113,7 @@ export async function load(
   return {
     serviceName,
     sourceDirectory: resolvedDir,
-    schema: {
-      name: `${serviceName}/schemas/${SCHEMA_ID}`,
-      datasources: [
-        toDatasource(projectId, dataConnectYaml.location, dataConnectYaml.schema.datasource),
-      ],
-      source: {
-        files: schemaGQLs,
-      },
-    },
+    schemas: schemas,
     dataConnectYaml,
     connectorInfo,
   };
@@ -149,6 +154,12 @@ function validateDataConnectYaml(unvalidated: any): DataConnectYaml {
   if (!unvalidated["location"]) {
     throw new FirebaseError("Missing required field 'location' in dataconnect.yaml");
   }
+  if (!experiments.isEnabled("fdcwebhooks") && unvalidated["schemas"]) {
+    throw new FirebaseError("Unsupported field 'schemas' in dataconnect.yaml");
+  }
+  if (!unvalidated["schema"] && !unvalidated["schemas"]) {
+    throw new FirebaseError("Either 'schema' or 'schemas' is required in dataconnect.yaml");
+  }
   return unvalidated as DataConnectYaml;
 }