feat(firestore-bigquery-export): add bigquery-project-id param to gen-schema-view script (#1999)

Author: jauntybrain

firestore-bigquery-export/scripts/gen-schema-view/src/index.ts

@@ -26,6 +26,7 @@ import { readSchemas } from "./schema-loader-utils";
 
 const BIGQUERY_VALID_CHARACTERS = /^[a-zA-Z0-9_]+$/;
 const FIRESTORE_VALID_CHARACTERS = /^[^\/]+$/;
+const GCP_PROJECT_VALID_CHARACTERS = /^[a-z][a-z0-9-]{0,29}$/;
 
 const validateInput = (value: any, name: string, regex: RegExp) => {
   if (!value || value === "" || value.trim() === "") {
@@ -58,7 +59,7 @@ program
   )
   .option(
     "-B, --big-query-project <big-query-project>",
-    "Google Cloud Project ID for BigQuery."
+    "Google Cloud Project ID for BigQuery (can be the same as the Firebase project ID)."
   )
   .option(
     "-d, --dataset <dataset>",
@@ -79,10 +80,20 @@ const questions = [
   {
     message: "What is your Firebase project ID?",
     name: "project",
+    default: process.env.PROJECT_ID,
     type: "input",
     validate: (value) =>
       validateInput(value, "project ID", FIRESTORE_VALID_CHARACTERS),
   },
+  {
+    message:
+      "What is your Google Cloud Project ID for BigQuery? (can be the same as the Firebase project ID)",
+    name: "bigQueryProject",
+    default: process.env.PROJECT_ID,
+    type: "input",
+    validate: (value) =>
+      validateInput(value, "BigQuery project ID", GCP_PROJECT_VALID_CHARACTERS),
+  },
   {
     message:
       "What is the ID of the BigQuery dataset the raw changelog lives in? (The dataset and the raw changelog must already exist!)",
@@ -109,6 +120,7 @@ const questions = [
 
 interface CliConfig {
   projectId: string;
+  bigQueryProjectId: string;
   datasetId: string;
   tableNamePrefix: string;
   schemas: { [schemaName: string]: FirestoreSchema };
@@ -121,7 +133,7 @@ async function run(): Promise<number> {
   // Set project ID so it can be used in BigQuery intialization
   process.env.PROJECT_ID = config.projectId;
   // BigQuery actually requires this variable to set the project correctly.
-  process.env.GOOGLE_CLOUD_PROJECT = config.projectId;
+  process.env.GOOGLE_CLOUD_PROJECT = config.bigQueryProjectId;
 
   // Initialize Firebase
   if (!firebase.apps.length) {
@@ -135,7 +147,9 @@ async function run(): Promise<number> {
   if (Object.keys(config.schemas).length === 0) {
     console.log(`No schema files found!`);
   }
-  const viewFactory = new FirestoreBigQuerySchemaViewFactory();
+  const viewFactory = new FirestoreBigQuerySchemaViewFactory(
+    config.bigQueryProjectId
+  );
   for (const schemaName in config.schemas) {
     await viewFactory.initializeSchemaViewResources(
       config.datasetId,
@@ -152,6 +166,7 @@ async function parseConfig(): Promise<CliConfig> {
   if (program.nonInteractive) {
     if (
       program.project === undefined ||
+      program.bigQueryProject === undefined ||
       program.dataset === undefined ||
       program.tableNamePrefix === undefined ||
       program.schemaFiles.length === 0
@@ -161,15 +176,18 @@ async function parseConfig(): Promise<CliConfig> {
     }
     return {
       projectId: program.project,
+      bigQueryProjectId: program.bigQueryProject,
       datasetId: program.dataset,
       tableNamePrefix: program.tableNamePrefix,
       schemas: readSchemas(program.schemaFiles),
     };
   }
-  const { project, dataset, tableNamePrefix, schemaFiles } =
+  const { project, bigQueryProject, dataset, tableNamePrefix, schemaFiles } =
     await inquirer.prompt(questions);
+
   return {
     projectId: project,
+    bigQueryProjectId: bigQueryProject,
     datasetId: dataset,
     tableNamePrefix: tableNamePrefix,
     schemas: readSchemas(

firestore-bigquery-export/scripts/gen-schema-view/src/schema/index.ts

@@ -87,8 +87,8 @@ export const firestoreToBigQueryFieldType: {
 export class FirestoreBigQuerySchemaViewFactory {
   bq: bigquery.BigQuery;
 
-  constructor() {
-    this.bq = new bigquery.BigQuery();
+  constructor(bqProjectId: string) {
+    this.bq = new bigquery.BigQuery({ projectId: bqProjectId });
   }
 
   /**