Google Cloud BigQuery bug fix (#18647)

michelle0927 · web-flow · commit 1744391047d3 · 2025-10-09T12:27:14.000-04:00
* updates

* updates

* version

* updates

* updates

* updates

* move @google-cloud/compute import

* versions

* version
diff --git a/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs b/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs
@@ -3,7 +3,7 @@ import utils from "../../common/utils.mjs";
 
 export default {
   name: "Bigquery Insert Rows",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/create-bucket/create-bucket.mjs b/components/google_cloud/actions/create-bucket/create-bucket.mjs
@@ -3,7 +3,7 @@ import storageClasses from "../../utils/storageClasses.mjs";
 
 export default {
   name: "Create Bucket",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs b/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs
@@ -12,7 +12,7 @@ export default {
   key: "google_cloud-create-scheduled-query",
   name: "Create Scheduled Query",
   description: "Creates a scheduled query in Google Cloud. [See the documentation](https://cloud.google.com/bigquery/docs/scheduling-queries)",
-  version: "0.0.2",
+  version: "0.0.3",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/get-bucket/get-bucket.mjs b/components/google_cloud/actions/get-bucket/get-bucket.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "Get Bucket Metadata",
-  version: "0.0.5",
+  version: "0.0.6",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/get-object/get-object.mjs b/components/google_cloud/actions/get-object/get-object.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "Get Object",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/list-buckets/list-buckets.mjs b/components/google_cloud/actions/list-buckets/list-buckets.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "List Buckets",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/logging-write-log/logging-write-log.mjs b/components/google_cloud/actions/logging-write-log/logging-write-log.mjs
@@ -3,7 +3,7 @@ import logSeverity from "../../utils/logSeverity.mjs";
 
 export default {
   name: "Logging - Write Log",
-  version: "0.0.5",
+  version: "0.0.6",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/run-query/run-query.mjs b/components/google_cloud/actions/run-query/run-query.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "Run Query",
-  version: "0.0.2",
+  version: "0.0.3",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/search-objects/search-objects.mjs b/components/google_cloud/actions/search-objects/search-objects.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "Search Objects",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs b/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs
@@ -1,8 +1,14 @@
 import googleCloud from "../../google_cloud.app.mjs";
+import {
+  ZonesClient,
+  ZoneOperationsClient,
+  InstancesClient,
+} from "@google-cloud/compute";
+import { ConfigurationError } from "@pipedream/platform";
 
 export default {
   name: "Switch Instance Boot Status",
-  version: "0.0.3",
+  version: "0.0.4",
   annotations: {
     destructiveHint: true,
     openWorldHint: true,
@@ -14,19 +20,23 @@ export default {
   props: {
     googleCloud,
     zone: {
-      propDefinition: [
-        googleCloud,
-        "zoneName",
-      ],
+      label: "Zone",
+      description: "The unique zone name",
+      type: "string",
+      async options() {
+        const zones = await this.listZones();
+        return zones.map((item) => (item.name));
+      },
     },
     instanceName: {
-      propDefinition: [
-        googleCloud,
-        "instanceName",
-        ({ zone }) => ({
-          zone,
-        }),
-      ],
+      label: "Instance Name",
+      description: "The unique instance name",
+      type: "string",
+      async options({ zone }) {
+        if (!zone) { return []; }
+        const instances = await this.listVmInstancesByZone(zone);
+        return instances.map((item) => (item.name));
+      },
     },
     newInstanceStatus: {
       label: "New Instance Status",
@@ -44,6 +54,70 @@ export default {
       default: false,
     },
   },
+  methods: {
+    zonesClient() {
+      return new ZonesClient(this.googleCloud.sdkParams());
+    },
+    zoneOperationsClient() {
+      return new ZoneOperationsClient(this.googleCloud.sdkParams());
+    },
+    instancesClient() {
+      return new InstancesClient(this.googleCloud.sdkParams());
+    },
+    async listZones() {
+      const zonesClient = this.zonesClient();
+      const sdkParams = this.googleCloud.sdkParams();
+      const [
+        zones,
+      ] = await zonesClient.list({
+        project: sdkParams.projectId,
+      });
+      return zones;
+    },
+    async waitOperation(operation) {
+      const operationsClient = this.zoneOperationsClient();
+      const sdkParams = this.googleCloud.sdkParams();
+      while (operation.status !== "DONE") {
+        [
+          operation,
+        ] = await operationsClient.wait({
+          operation: operation.name,
+          project: sdkParams.projectId,
+          zone: operation.zone.split("/").pop(),
+        });
+      }
+      return operation;
+    },
+    async listVmInstancesByZone(zone) {
+      const instancesClient = this.instancesClient();
+      const sdkParams = this.googleCloud.sdkParams();
+      const [
+        instances,
+      ] = await instancesClient.list({
+        project: sdkParams.projectId,
+        zone,
+      });
+      return instances;
+    },
+    async switchInstanceBootStatus(zone, instance, newStatus) {
+      if (![
+        "start",
+        "stop",
+      ].includes(newStatus)) {
+        throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'.");
+      }
+      const instancesClient = this.instancesClient();
+      const sdkParams = this.googleCloud.sdkParams();
+      const [
+        response,
+      ] = await instancesClient[newStatus]({
+        project: sdkParams.projectId,
+        zone,
+        instance,
+      });
+      return response.latestResponse;
+    },
+  },
   async run({ $ }) {
     const {
       zone,
@@ -52,14 +126,14 @@ export default {
       newInstanceStatus,
     } = this;
 
-    let operation = await this.googleCloud.switchInstanceBootStatus(
+    let operation = await this.switchInstanceBootStatus(
       zone,
       instanceName,
       newInstanceStatus,
     );
 
     if (waitCompletion) {
-      operation = await this.googleCloud.waitOperation(operation);
+      operation = await this.waitOperation(operation);
     }
 
     $.export("$summary", `Instance ${instanceName} boot status was set to ${newInstanceStatus}.`);
diff --git a/components/google_cloud/actions/upload-object/upload-object.mjs b/components/google_cloud/actions/upload-object/upload-object.mjs
@@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs";
 
 export default {
   name: "Upload An Object",
-  version: "0.0.4",
+  version: "0.0.5",
   annotations: {
     destructiveHint: false,
     openWorldHint: true,
diff --git a/components/google_cloud/google_cloud.app.mjs b/components/google_cloud/google_cloud.app.mjs
@@ -2,13 +2,7 @@
 import { Logging } from "@google-cloud/logging";
 import { Storage } from "@google-cloud/storage";
 import { BigQuery } from "@google-cloud/bigquery";
-import {
-  ZonesClient,
-  ZoneOperationsClient,
-  InstancesClient,
-} from "@google-cloud/compute";
 import { v1 as bqdt } from "@google-cloud/bigquery-data-transfer";
-import { ConfigurationError } from "@pipedream/platform";
 
 export default {
   type: "app",
@@ -135,68 +129,6 @@ export default {
     storageClient() {
       return new Storage(this.sdkParams());
     },
-    instancesClient() {
-      return new InstancesClient(this.sdkParams());
-    },
-    zoneOperationsClient() {
-      return new ZoneOperationsClient(this.sdkParams());
-    },
-    zonesClient() {
-      return new ZonesClient(this.sdkParams());
-    },
-    async listVmInstancesByZone(zone) {
-      const instancesClient = this.instancesClient();
-      const sdkParams = this.sdkParams();
-      const [
-        instances,
-      ] = await instancesClient.list({
-        project: sdkParams.projectId,
-        zone,
-      });
-      return instances;
-    },
-    async listZones() {
-      const zonesClient = this.zonesClient();
-      const sdkParams = this.sdkParams();
-      const [
-        zones,
-      ] = await zonesClient.list({
-        project: sdkParams.projectId,
-      });
-      return zones;
-    },
-    async switchInstanceBootStatus(zone, instance, newStatus) {
-      if (![
-        "start",
-        "stop",
-      ].includes(newStatus)) {
-        throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'.");
-      }
-      const instancesClient = this.instancesClient();
-      const sdkParams = this.sdkParams();
-      const [
-        response,
-      ] = await instancesClient[newStatus]({
-        project: sdkParams.projectId,
-        zone,
-        instance,
-      });
-      return response.latestResponse;
-    },
-    async waitOperation(operation) {
-      const operationsClient = this.zoneOperationsClient();
-      const sdkParams = this.sdkParams();
-      while (operation.status !== "DONE") {
-        [
-          operation,
-        ] = await operationsClient.wait({
-          operation: operation.name,
-          project: sdkParams.projectId,
-          zone: operation.zone.split("/").pop(),
-        });
-      }
-      return operation;
-    },
     getBigQueryClient() {
       const credentials = this.authKeyJson();
       const { project_id: projectId } = credentials;
diff --git a/components/google_cloud/package.json b/components/google_cloud/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/google_cloud",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Pipedream Google_cloud Components",
   "main": "google_cloud.app.mjs",
   "keywords": [
diff --git a/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs b/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs
@@ -9,7 +9,7 @@ export default {
   // eslint-disable-next-line pipedream/source-name
   name: "BigQuery - New Row",
   description: "Emit new events when a new row is added to a table",
-  version: "0.1.8",
+  version: "0.1.9",
   dedupe: "unique",
   type: "source",
   props: {
@@ -56,9 +56,6 @@ export default {
       const lastResultId = await this._getIdOfLastRow();
       this._setLastResultId(lastResultId);
     },
-    deactivate() {
-      this._setLastResultId(null);
-    },
   },
   methods: {
     ...common.methods,
@@ -113,7 +110,18 @@ export default {
           limit,
         },
       };
-      const rows = await this.getRowsForQuery(queryOpts, this.datasetId);
+      const client = this.googleCloud
+        .getBigQueryClient()
+        .dataset(this.datasetId);
+
+      const [
+        job,
+      ] = await client.createQueryJob(queryOpts);
+
+      const [
+        rows,
+      ] = await job.getQueryResults();
+
       if (rows.length === 0) {
         console.log(`
           No records found in the target table, will start scanning from the beginning
@@ -126,15 +134,16 @@ export default {
     },
     getQueryOpts() {
       const lastResultId = this._getLastResultId();
-      const query = `
-        SELECT *
-        FROM \`${this.tableId}\`
-        WHERE \`${this.uniqueKey}\` >= @lastResultId
-        ORDER BY \`${this.uniqueKey}\` ASC
-      `;
-      const params = {
-        lastResultId,
-      };
+      let query = `SELECT * FROM \`${this.tableId}\``;
+      if (lastResultId) {
+        query += ` WHERE \`${this.uniqueKey}\` >= @lastResultId`;
+      }
+      query += ` ORDER BY \`${this.uniqueKey}\` DESC`;
+      const params = lastResultId
+        ? {
+          lastResultId,
+        }
+        : {};
       return {
         query,
         params,
diff --git a/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs b/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs
@@ -8,7 +8,7 @@ export default {
   // eslint-disable-next-line pipedream/source-name
   name: "BigQuery - Query Results",
   description: "Emit new events with the results of an arbitrary query",
-  version: "0.1.7",
+  version: "0.1.8",
   dedupe: "unique",
   type: "source",
   props: {
diff --git a/components/google_cloud/sources/common/bigquery.mjs b/components/google_cloud/sources/common/bigquery.mjs
diff --git a/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs b/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs
