diff --git a/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs b/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs index 09e64c11df480..872f7103bb513 100644 --- a/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs +++ b/components/google_cloud/actions/bigquery-insert-rows/bigquery-insert-rows.mjs @@ -3,7 +3,7 @@ import utils from "../../common/utils.mjs"; export default { name: "Bigquery Insert Rows", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/create-bucket/create-bucket.mjs b/components/google_cloud/actions/create-bucket/create-bucket.mjs index 003644961a3b8..197d1168a33d1 100644 --- a/components/google_cloud/actions/create-bucket/create-bucket.mjs +++ b/components/google_cloud/actions/create-bucket/create-bucket.mjs @@ -3,7 +3,7 @@ import storageClasses from "../../utils/storageClasses.mjs"; export default { name: "Create Bucket", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs b/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs index 99e912715d3d5..1a0e12d693403 100644 --- a/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs +++ b/components/google_cloud/actions/create-scheduled-query/create-scheduled-query.mjs @@ -12,7 +12,7 @@ export default { key: "google_cloud-create-scheduled-query", name: "Create Scheduled Query", description: "Creates a scheduled query in Google Cloud. [See the documentation](https://cloud.google.com/bigquery/docs/scheduling-queries)", - version: "0.0.2", + version: "0.0.3", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/get-bucket/get-bucket.mjs b/components/google_cloud/actions/get-bucket/get-bucket.mjs index bb3473f059bd7..4602f24fe4607 100644 --- a/components/google_cloud/actions/get-bucket/get-bucket.mjs +++ b/components/google_cloud/actions/get-bucket/get-bucket.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "Get Bucket Metadata", - version: "0.0.5", + version: "0.0.6", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/get-object/get-object.mjs b/components/google_cloud/actions/get-object/get-object.mjs index ca13af1c3761a..a859be9bd9942 100644 --- a/components/google_cloud/actions/get-object/get-object.mjs +++ b/components/google_cloud/actions/get-object/get-object.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "Get Object", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/list-buckets/list-buckets.mjs b/components/google_cloud/actions/list-buckets/list-buckets.mjs index e0f2f0d9b20cc..c52793305ac31 100644 --- a/components/google_cloud/actions/list-buckets/list-buckets.mjs +++ b/components/google_cloud/actions/list-buckets/list-buckets.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "List Buckets", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/logging-write-log/logging-write-log.mjs b/components/google_cloud/actions/logging-write-log/logging-write-log.mjs index c173cecc07c52..44ef739fab332 100644 --- a/components/google_cloud/actions/logging-write-log/logging-write-log.mjs +++ b/components/google_cloud/actions/logging-write-log/logging-write-log.mjs @@ -3,7 +3,7 @@ import logSeverity from "../../utils/logSeverity.mjs"; export default { name: "Logging - Write Log", - version: "0.0.5", + version: "0.0.6", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/run-query/run-query.mjs b/components/google_cloud/actions/run-query/run-query.mjs index c2d6104648a38..73b198701d332 100644 --- a/components/google_cloud/actions/run-query/run-query.mjs +++ b/components/google_cloud/actions/run-query/run-query.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "Run Query", - version: "0.0.2", + version: "0.0.3", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/search-objects/search-objects.mjs b/components/google_cloud/actions/search-objects/search-objects.mjs index 8b4a93f2941d8..265f1e3812c32 100644 --- a/components/google_cloud/actions/search-objects/search-objects.mjs +++ b/components/google_cloud/actions/search-objects/search-objects.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "Search Objects", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs b/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs index dabacb16585cd..a88fb2d35a6a4 100644 --- a/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs +++ b/components/google_cloud/actions/switch-instance-boot-status/switch-instance-boot-status.mjs @@ -1,8 +1,14 @@ import googleCloud from "../../google_cloud.app.mjs"; +import { + ZonesClient, + ZoneOperationsClient, + InstancesClient, +} from "@google-cloud/compute"; +import { ConfigurationError } from "@pipedream/platform"; export default { name: "Switch Instance Boot Status", - version: "0.0.3", + version: "0.0.4", annotations: { destructiveHint: true, openWorldHint: true, @@ -14,19 +20,23 @@ export default { props: { googleCloud, zone: { - propDefinition: [ - googleCloud, - "zoneName", - ], + label: "Zone", + description: "The unique zone name", + type: "string", + async options() { + const zones = await this.listZones(); + return zones.map((item) => (item.name)); + }, }, instanceName: { - propDefinition: [ - googleCloud, - "instanceName", - ({ zone }) => ({ - zone, - }), - ], + label: "Instance Name", + description: "The unique instance name", + type: "string", + async options({ zone }) { + if (!zone) { return []; } + const instances = await this.listVmInstancesByZone(zone); + return instances.map((item) => (item.name)); + }, }, newInstanceStatus: { label: "New Instance Status", @@ -44,6 +54,70 @@ export default { default: false, }, }, + methods: { + zonesClient() { + return new ZonesClient(this.googleCloud.sdkParams()); + }, + zoneOperationsClient() { + return new ZoneOperationsClient(this.googleCloud.sdkParams()); + }, + instancesClient() { + return new InstancesClient(this.googleCloud.sdkParams()); + }, + async listZones() { + const zonesClient = this.zonesClient(); + const sdkParams = this.googleCloud.sdkParams(); + const [ + zones, + ] = await zonesClient.list({ + project: sdkParams.projectId, + }); + return zones; + }, + async waitOperation(operation) { + const operationsClient = this.zoneOperationsClient(); + const sdkParams = this.googleCloud.sdkParams(); + while (operation.status !== "DONE") { + [ + operation, + ] = await operationsClient.wait({ + operation: operation.name, + project: sdkParams.projectId, + zone: operation.zone.split("/").pop(), + }); + } + return operation; + }, + async listVmInstancesByZone(zone) { + const instancesClient = this.instancesClient(); + const sdkParams = this.googleCloud.sdkParams(); + const [ + instances, + ] = await instancesClient.list({ + project: sdkParams.projectId, + zone, + }); + return instances; + }, + async switchInstanceBootStatus(zone, instance, newStatus) { + if (![ + "start", + "stop", + ].includes(newStatus)) { + throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'."); + } + const instancesClient = this.instancesClient(); + const sdkParams = this.googleCloud.sdkParams(); + const [ + response, + ] = await instancesClient[newStatus]({ + project: sdkParams.projectId, + zone, + instance, + }); + return response.latestResponse; + }, + }, async run({ $ }) { const { zone, @@ -52,14 +126,14 @@ export default { newInstanceStatus, } = this; - let operation = await this.googleCloud.switchInstanceBootStatus( + let operation = await this.switchInstanceBootStatus( zone, instanceName, newInstanceStatus, ); if (waitCompletion) { - operation = await this.googleCloud.waitOperation(operation); + operation = await this.waitOperation(operation); } $.export("$summary", `Instance ${instanceName} boot status was set to ${newInstanceStatus}.`); diff --git a/components/google_cloud/actions/upload-object/upload-object.mjs b/components/google_cloud/actions/upload-object/upload-object.mjs index c94b0c0d9b280..3fc5da99911b1 100644 --- a/components/google_cloud/actions/upload-object/upload-object.mjs +++ b/components/google_cloud/actions/upload-object/upload-object.mjs @@ -2,7 +2,7 @@ import googleCloud from "../../google_cloud.app.mjs"; export default { name: "Upload An Object", - version: "0.0.4", + version: "0.0.5", annotations: { destructiveHint: false, openWorldHint: true, diff --git a/components/google_cloud/google_cloud.app.mjs b/components/google_cloud/google_cloud.app.mjs index c53ff153634e1..6b4e36ae21548 100644 --- a/components/google_cloud/google_cloud.app.mjs +++ b/components/google_cloud/google_cloud.app.mjs @@ -2,13 +2,7 @@ import { Logging } from "@google-cloud/logging"; import { Storage } from "@google-cloud/storage"; import { BigQuery } from "@google-cloud/bigquery"; -import { - ZonesClient, - ZoneOperationsClient, - InstancesClient, -} from "@google-cloud/compute"; import { v1 as bqdt } from "@google-cloud/bigquery-data-transfer"; -import { ConfigurationError } from "@pipedream/platform"; export default { type: "app", @@ -135,68 +129,6 @@ export default { storageClient() { return new Storage(this.sdkParams()); }, - instancesClient() { - return new InstancesClient(this.sdkParams()); - }, - zoneOperationsClient() { - return new ZoneOperationsClient(this.sdkParams()); - }, - zonesClient() { - return new ZonesClient(this.sdkParams()); - }, - async listVmInstancesByZone(zone) { - const instancesClient = this.instancesClient(); - const sdkParams = this.sdkParams(); - const [ - instances, - ] = await instancesClient.list({ - project: sdkParams.projectId, - zone, - }); - return instances; - }, - async listZones() { - const zonesClient = this.zonesClient(); - const sdkParams = this.sdkParams(); - const [ - zones, - ] = await zonesClient.list({ - project: sdkParams.projectId, - }); - return zones; - }, - async switchInstanceBootStatus(zone, instance, newStatus) { - if (![ - "start", - "stop", - ].includes(newStatus)) { - throw new ConfigurationError("The new VM boot status must be 'start' or 'stop'."); - } - const instancesClient = this.instancesClient(); - const sdkParams = this.sdkParams(); - const [ - response, - ] = await instancesClient[newStatus]({ - project: sdkParams.projectId, - zone, - instance, - }); - return response.latestResponse; - }, - async waitOperation(operation) { - const operationsClient = this.zoneOperationsClient(); - const sdkParams = this.sdkParams(); - while (operation.status !== "DONE") { - [ - operation, - ] = await operationsClient.wait({ - operation: operation.name, - project: sdkParams.projectId, - zone: operation.zone.split("/").pop(), - }); - } - return operation; - }, getBigQueryClient() { const credentials = this.authKeyJson(); const { project_id: projectId } = credentials; diff --git a/components/google_cloud/package.json b/components/google_cloud/package.json index 2593e22b65e60..3e73d9118ef60 100644 --- a/components/google_cloud/package.json +++ b/components/google_cloud/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_cloud", - "version": "0.6.2", + "version": "0.6.3", "description": "Pipedream Google_cloud Components", "main": "google_cloud.app.mjs", "keywords": [ diff --git a/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs b/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs index 07a0efbc29f89..85098ea2c6cd2 100644 --- a/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs +++ b/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs @@ -9,7 +9,7 @@ export default { // eslint-disable-next-line pipedream/source-name name: "BigQuery - New Row", description: "Emit new events when a new row is added to a table", - version: "0.1.8", + version: "0.1.9", dedupe: "unique", type: "source", props: { @@ -56,9 +56,6 @@ export default { const lastResultId = await this._getIdOfLastRow(); this._setLastResultId(lastResultId); }, - deactivate() { - this._setLastResultId(null); - }, }, methods: { ...common.methods, @@ -113,7 +110,18 @@ export default { limit, }, }; - const rows = await this.getRowsForQuery(queryOpts, this.datasetId); + const client = this.googleCloud + .getBigQueryClient() + .dataset(this.datasetId); + + const [ + job, + ] = await client.createQueryJob(queryOpts); + + const [ + rows, + ] = await job.getQueryResults(); + if (rows.length === 0) { console.log(` No records found in the target table, will start scanning from the beginning @@ -126,15 +134,16 @@ export default { }, getQueryOpts() { const lastResultId = this._getLastResultId(); - const query = ` - SELECT * - FROM \`${this.tableId}\` - WHERE \`${this.uniqueKey}\` >= @lastResultId - ORDER BY \`${this.uniqueKey}\` ASC - `; - const params = { - lastResultId, - }; + let query = `SELECT * FROM \`${this.tableId}\``; + if (lastResultId) { + query += ` WHERE \`${this.uniqueKey}\` >= @lastResultId`; + } + query += ` ORDER BY \`${this.uniqueKey}\` DESC`; + const params = lastResultId + ? { + lastResultId, + } + : {}; return { query, params, diff --git a/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs b/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs index 99b8368725f4d..14bad5dcee471 100644 --- a/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs +++ b/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs @@ -8,7 +8,7 @@ export default { // eslint-disable-next-line pipedream/source-name name: "BigQuery - Query Results", description: "Emit new events with the results of an arbitrary query", - version: "0.1.7", + version: "0.1.8", dedupe: "unique", type: "source", props: { diff --git a/components/google_cloud/sources/common/bigquery.mjs b/components/google_cloud/sources/common/bigquery.mjs index b5df544d6ad38..51bccdb986ee2 100644 --- a/components/google_cloud/sources/common/bigquery.mjs +++ b/components/google_cloud/sources/common/bigquery.mjs @@ -163,9 +163,9 @@ export default { throw new Error("processEvent is not implemented"); }, }, - run(event) { + async run(event) { const { timestamp } = event; const queryOpts = this.getQueryOpts(event); - return this.processCollection(queryOpts, timestamp); + return await this.processCollection(queryOpts, timestamp); }, }; diff --git a/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs b/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs index eca05a0b1279f..589fd22897d6d 100644 --- a/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs +++ b/components/google_cloud/sources/new-pubsub-messages/new-pubsub-messages.mjs @@ -5,7 +5,7 @@ export default { key: "google_cloud-new-pubsub-messages", name: "New Pub/Sub Messages", description: "Emit new Pub/Sub topic in your GCP account. Messages published to this topic are emitted from the Pipedream source.", - version: "0.1.6", + version: "0.1.7", type: "source", dedupe: "unique", // Dedupe on Pub/Sub message ID props: {