[FDC] Support private IP Cloud SQL with VPC (#9200)

* allow private IP

* m

* remove log

* changelog

* comments

* m

* bold Cloud SQL instance name

* m

* m

* m

* m

* m

* m

* m

* m

* m

* m

* m

* timeout

* clean

* clean

* clean

* clean

* Update CHANGELOG.md

Co-authored-by: Yuchen Shi <[email protected]>

---------

Co-authored-by: Yuchen Shi <[email protected]>

Author: fredzqm

CHANGELOG.md

@@ -3,3 +3,4 @@
 - `firebase_update_environment` MCP tool supports accepting Gemini in Firebase Terms of Service.
 - Fixed a bug when `firebase init dataconnect` failed to create a React app when launched from VS Code extension (#9171).
 - Improved the clarity of the `firebase apptesting:execute` command when you have zero or multiple apps.
+- `firebase dataconnect:sql:migrate` now supports Cloud SQL instances with only private IPs. The command must be run in the same VPC of the instance to work. (##9200)

src/dataconnect/client.ts

@@ -140,7 +140,7 @@ export async function upsertSchema(
     apiOrigin: dataconnectOrigin(),
     apiVersion: DATACONNECT_API_VERSION,
     operationResourceName: op.body.name,
-    masterTimeout: 120000,
+    masterTimeout: 10000,
   });
 }
 

src/dataconnect/provisionCloudSql.ts

@@ -1,5 +1,6 @@
 import * as cloudSqlAdminClient from "../gcp/cloudsql/cloudsqladmin";
 import * as utils from "../utils";
+import * as clc from "colorette";
 import { grantRolesToCloudSqlServiceAccount } from "./checkIam";
 import { Instance } from "../gcp/cloudsql/types";
 import { promiseWithSpinner } from "../utils";
@@ -35,20 +36,23 @@ async function upsertInstance(args: {
   const { projectId, instanceId, requireGoogleMlIntegration, dryRun } = args;
   try {
     const existingInstance = await cloudSqlAdminClient.getInstance(projectId, instanceId);
-    utils.logLabeledBullet("dataconnect", `Found existing Cloud SQL instance ${instanceId}.`);
+    utils.logLabeledBullet(
+      "dataconnect",
+      `Found existing Cloud SQL instance ${clc.bold(instanceId)}.`,
+    );
     const why = getUpdateReason(existingInstance, requireGoogleMlIntegration);
     if (why) {
       if (dryRun) {
         utils.logLabeledBullet(
           "dataconnect",
-          `Cloud SQL instance ${instanceId} settings not compatible with Firebase Data Connect. ` +
+          `Cloud SQL instance ${clc.bold(instanceId)} settings are not compatible with Firebase Data Connect. ` +
             `It will be updated on your next deploy.` +
             why,
         );
       } else {
         utils.logLabeledBullet(
           "dataconnect",
-          `Cloud SQL instance ${instanceId} settings not compatible with Firebase Data Connect. ` +
+          `Cloud SQL instance ${clc.bold(instanceId)} settings are not compatible with Firebase Data Connect. ` +
             why,
         );
         await promiseWithSpinner(
@@ -83,7 +87,7 @@ async function createInstance(args: {
   if (dryRun) {
     utils.logLabeledBullet(
       "dataconnect",
-      `Cloud SQL Instance ${instanceId} not found. It will be created on your next deploy.`,
+      `Cloud SQL Instance ${clc.bold(instanceId)} not found. It will be created on your next deploy.`,
     );
   } else {
     await cloudSqlAdminClient.createInstance({
@@ -109,7 +113,7 @@ export function cloudSQLBeingCreated(
   includeFreeTrialToS?: boolean,
 ): string {
   return (
-    `Cloud SQL Instance ${instanceId} is being created.` +
+    `Cloud SQL Instance ${clc.bold(instanceId)} is being created.` +
     (includeFreeTrialToS
       ? `\nThis instance is provided under the terms of the Data Connect no-cost trial ${freeTrialTermsLink()}`
       : "") +
@@ -157,9 +161,19 @@ async function upsertDatabase(args: {
 export function getUpdateReason(instance: Instance, requireGoogleMlIntegration: boolean): string {
   let reason = "";
   const settings = instance.settings;
-  // Cloud SQL instances must have public IP enabled to be used with Firebase Data Connect.
   if (!settings.ipConfiguration?.ipv4Enabled) {
-    reason += "\n - to enable public IP.";
+    utils.logLabeledWarning(
+      "dataconnect",
+      `Cloud SQL instance ${clc.bold(instance.name)} does not have a public IP.
+    ${clc.bold("firebase dataconnect:sql:migrate")} will only work within its VPC (e.g. GCE, GKE).`,
+    );
+    if (
+      settings.ipConfiguration?.privateNetwork &&
+      !settings.ipConfiguration?.enablePrivatePathForGoogleCloudServices
+    ) {
+      // Cloud SQL instances with only private IP must enable PSC for Data Connect backend to connect to it.
+      reason += "\n - to enable Private Path for Google Cloud Services.";
+    }
   }
 
   if (requireGoogleMlIntegration) {

src/gcp/cloudsql/cloudsqladmin.ts

@@ -1,5 +1,6 @@
 import { Client } from "../../apiv2";
 import { cloudSQLAdminOrigin } from "../../api";
+import * as clc from "colorette";
 import * as operationPoller from "../../operation-poller";
 import { Instance, Database, User, UserType, DatabaseFlag } from "./types";
 import { needProjectId } from "../../projectUtils";
@@ -47,7 +48,7 @@ export async function getInstance(projectId: string, instanceId: string): Promis
   const res = await client.get<Instance>(`projects/${projectId}/instances/${instanceId}`);
   if (res.body.state === "FAILED") {
     throw new FirebaseError(
-      `Cloud SQL instance ${instanceId} is in a failed state.\nGo to ${instanceConsoleLink(projectId, instanceId)} to repair or delete it.`,
+      `Cloud SQL instance ${clc.bold(instanceId)} is in a failed state.\nGo to ${instanceConsoleLink(projectId, instanceId)} to repair or delete it.`,
     );
   }
   return res.body;
@@ -121,7 +122,8 @@ export async function updateInstanceForDataConnect(
     {
       settings: {
         ipConfiguration: {
-          ipv4Enabled: true,
+          enablePrivatePathForGoogleCloudServices:
+            !!instance?.settings?.ipConfiguration?.privateNetwork,
         },
         databaseFlags: dbFlags,
         enableGoogleMlIntegration,

src/gcp/cloudsql/connect.ts

@@ -34,39 +34,21 @@ export async function execute(
     );
   }
   let connector: Connector;
-  let pool: pg.Pool;
+  let authType: AuthTypes;
   switch (user.type) {
     case "CLOUD_IAM_USER": {
       connector = new Connector({
         auth: new FBToolsAuthClient(),
       });
-      const clientOpts = await connector.getOptions({
-        instanceConnectionName: connectionName,
-        ipType: IpAddressTypes.PUBLIC,
-        authType: AuthTypes.IAM,
-      });
-      pool = new pg.Pool({
-        ...clientOpts,
-        user: opts.username,
-        database: opts.databaseId,
-      });
+      authType = AuthTypes.IAM;
       break;
     }
     case "CLOUD_IAM_SERVICE_ACCOUNT": {
       connector = new Connector();
+      authType = AuthTypes.IAM;
       // Currently, this only works with Application Default credentials
       // https://github.com/GoogleCloudPlatform/cloud-sql-nodejs-connector/issues/61 is an open
       // FR to add support for OAuth2 tokens.
-      const clientOpts = await connector.getOptions({
-        instanceConnectionName: connectionName,
-        ipType: IpAddressTypes.PUBLIC,
-        authType: AuthTypes.IAM,
-      });
-      pool = new pg.Pool({
-        ...clientOpts,
-        user: opts.username,
-        database: opts.databaseId,
-      });
       break;
     }
     default: {
@@ -77,19 +59,24 @@ export async function execute(
       connector = new Connector({
         auth: new FBToolsAuthClient(),
       });
-      const clientOpts = await connector.getOptions({
-        instanceConnectionName: connectionName,
-        ipType: IpAddressTypes.PUBLIC,
-      });
-      pool = new pg.Pool({
-        ...clientOpts,
-        user: opts.username,
-        password: opts.password,
-        database: opts.databaseId,
-      });
+      authType = AuthTypes.PASSWORD;
       break;
     }
   }
+  const connectionOpts = {
+    instanceConnectionName: connectionName,
+    ipType: instance.ipAddresses.some((ip) => ip.type === "PRIMARY")
+      ? IpAddressTypes.PUBLIC
+      : IpAddressTypes.PRIVATE,
+    authType: authType,
+  };
+  const pool = new pg.Pool({
+    ...(await connector.getOptions(connectionOpts)),
+    connectionTimeoutMillis: 1000,
+    password: opts.password,
+    user: opts.username,
+    database: opts.databaseId,
+  });
 
   const cleanUpFn = async () => {
     conn.release();

src/gcp/cloudsql/types.ts

@@ -23,6 +23,7 @@ export interface IpConfiguration {
     allowedConsumerProjects: string[];
     pscEnabled: boolean;
   };
+  enablePrivatePathForGoogleCloudServices?: boolean;
 }
 
 export interface InstanceSettings {