diff --git a/components/google_cloud/package.json b/components/google_cloud/package.json index 6642e175013f0..01508632ca235 100644 --- a/components/google_cloud/package.json +++ b/components/google_cloud/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_cloud", - "version": "0.6.0", + "version": "0.6.1", "description": "Pipedream Google_cloud Components", "main": "google_cloud.app.mjs", "keywords": [ @@ -20,7 +20,7 @@ "@google-cloud/logging": "^10.0.3", "@google-cloud/pubsub": "^3.0.1", "@google-cloud/storage": "^6.0.1", - "@pipedream/platform": "^0.10.0", + "@pipedream/platform": "^3.1.0", "crypto": "^1.0.1", "lodash-es": "^4.17.21", "uuid": "^8.3.2" diff --git a/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs b/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs index ceaa1771ba829..9e20e95c25c6c 100644 --- a/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs +++ b/components/google_cloud/sources/bigquery-new-row/bigquery-new-row.mjs @@ -9,7 +9,7 @@ export default { // eslint-disable-next-line pipedream/source-name name: "BigQuery - New Row", description: "Emit new events when a new row is added to a table", - version: "0.1.6", + version: "0.1.7", dedupe: "unique", type: "source", props: { diff --git a/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs b/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs index 33a90f29b24b0..56af8ea87f529 100644 --- a/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs +++ b/components/google_cloud/sources/bigquery-query-results/bigquery-query-results.mjs @@ -8,7 +8,7 @@ export default { // eslint-disable-next-line pipedream/source-name name: "BigQuery - Query Results", description: "Emit new events with the results of an arbitrary query", - version: "0.1.5", + version: "0.1.6", dedupe: "unique", type: "source", props: { diff --git a/components/google_cloud/sources/common/bigquery.mjs b/components/google_cloud/sources/common/bigquery.mjs index ffbdfd04d1247..4ae2b1a6f3fd8 100644 --- a/components/google_cloud/sources/common/bigquery.mjs +++ b/components/google_cloud/sources/common/bigquery.mjs @@ -30,17 +30,33 @@ export default { }, }, methods: { - async getRowsForQuery(queryOpts) { + async createQueryJob(queryOpts) { const client = this.googleCloud .getBigQueryClient() .dataset(this.datasetId); const [ job, ] = await client.createQueryJob(queryOpts); + return job; + }, + async getRowsForQuery(job, pageSize = 1000, pageToken = null) { + const options = { + maxResults: pageSize, + }; + + if (pageToken) { + options.pageToken = pageToken; + } + const [ rows, - ] = await job.getQueryResults(); - return rows; + queryResults, + ] = await job.getQueryResults(options); + const nextPageToken = queryResults?.pageToken; + return { + rows, + pageToken: nextPageToken, + }; }, _updateLastResultId(rows) { const lastRow = rows.pop(); @@ -53,26 +69,49 @@ export default { this.db.set("lastResultId", newLastResultId); }, async processCollection(queryOpts, timestamp) { - const rows = await this.getRowsForQuery(queryOpts); - chunk(rows, this.eventSize) - .forEach((rows) => { - const meta = this.generateMetaForCollection(rows, timestamp); - const rowCount = rows.length; - const data = { - rows, - rowCount, - }; - this.$emit(data, meta); + const job = await this.createQueryJob(queryOpts); + + const pageSize = 1000, maxPages = 10; + let pageToken = null; + let allProcessed = false; + let pageCount = 0; + + while (!allProcessed) { + const { + rows, pageToken: nextPageToken, + } = await this.getRowsForQuery(job, pageSize, pageToken); + + if (rows.length === 0) { + allProcessed = true; + break; + } + + chunk(rows, this.eventSize).forEach((batch) => { + if (this.eventSize === 1) { + const meta = this.generateMeta(batch[0], timestamp); + this.$emit(batch[0], meta); + } else { + const meta = this.generateMetaForCollection(batch, timestamp); + const data = { + rows: batch, + rowCount: batch.length, + }; + this.$emit(data, meta); + } }); - if (this.uniqueKey) this._updateLastResultId(rows); - }, - async processSingle(queryOpts, timestamp) { - const rows = await this.getRowsForQuery(queryOpts); - rows.forEach((row) => { - const meta = this.generateMeta(row, timestamp); - this.$emit(row, meta); - }); - if (this.uniqueKey) this._updateLastResultId(rows); + + pageCount++; + if (pageCount >= maxPages) { + allProcessed = true; + } + if (this.uniqueKey) { + this._updateLastResultId(rows); + } + if (!nextPageToken) { + break; + } + pageToken = nextPageToken; + } }, getInitialEventCount() { return 10; @@ -93,8 +132,6 @@ export default { run(event) { const { timestamp } = event; const queryOpts = this.getQueryOpts(event); - return (this.eventSize === 1) ? - this.processSingle(queryOpts, timestamp) : - this.processCollection(queryOpts, timestamp); + return this.processCollection(queryOpts, timestamp); }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 77992798a25bf..939d6c9a66e09 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3312,8 +3312,7 @@ importers: specifier: ^3.1.0 version: 3.1.0 - components/cratedb_cloud: - specifiers: {} + components/cratedb_cloud: {} components/crawlbase: {} @@ -3438,8 +3437,7 @@ importers: components/cvr_dev: {} - components/cyberark_certificate_manager: - specifiers: {} + components/cyberark_certificate_manager: {} components/cyberimpact: dependencies: @@ -5888,8 +5886,8 @@ importers: specifier: ^6.0.1 version: 6.12.0 '@pipedream/platform': - specifier: ^0.10.0 - version: 0.10.0 + specifier: ^3.1.0 + version: 3.1.0 crypto: specifier: ^1.0.1 version: 1.0.1 @@ -30986,22 +30984,22 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to superagent v10.2.2+, see release notes at https://github.com/forwardemail/superagent/releases/tag/v10.2.2 - maintenance is supported by Forward Email @ https://forwardemail.net supports-color@10.0.0: resolution: {integrity: sha512-HRVVSbCCMbj7/kdWF9Q+bbckjBHLtHMEoJWlkmYzzdwhYMkjkOwubLM6t7NbWKjgKamGDrWL1++KrjUO1t9oAQ==}