From c4b9e1598e86baa54cc43d7301e603aef3405195 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 10:10:54 -0300 Subject: [PATCH 1/6] add underscore to private methods --- .../sources/updated-page/updated-page.mjs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 4c7b00f0a3357..4ecce4505b0b8 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -41,7 +41,7 @@ export default { }, hooks: { async deploy() { - const propertiesToCheck = await this.getPropertiesToCheck(); + const propertiesToCheck = await this._getPropertiesToCheck(); const propertyValues = {}; const params = this.lastUpdatedSortParam(); const pagesStream = this.notion.getPages(this.databaseId, params); @@ -49,7 +49,7 @@ export default { let lastUpdatedTimestamp = 0; for await (const page of pagesStream) { for (const propertyName of propertiesToCheck) { - const currentValue = this.maybeRemoveFileSubItems(page.properties[propertyName]); + const currentValue = this._maybeRemoveFileSubItems(page.properties[propertyName]); propertyValues[page.id] = { ...propertyValues[page.id], [propertyName]: currentValue, @@ -60,7 +60,7 @@ export default { Date.parse(page.last_edited_time), ); if (count++ < 25) { - this.emitEvent(page); + this._emitEvent(page); } } this._setPropertyValues(propertyValues); @@ -80,14 +80,14 @@ export default { const compressed = zlib.deflateSync(string).toString("base64"); this.db.set("propertyValues", compressed); }, - async getPropertiesToCheck() { + async _getPropertiesToCheck() { if (this.properties?.length) { return this.properties; } const { properties } = await this.notion.retrieveDatabase(this.databaseId); return Object.keys(properties); }, - maybeRemoveFileSubItems(property) { + _maybeRemoveFileSubItems(property) { // Files & Media type: // `url` and `expiry_time` are constantly updated by Notion, so ignore these fields if (property.type === "files") { @@ -101,7 +101,7 @@ export default { } return property; }, - generateMeta(obj, summary) { + _generateMeta(obj, summary) { const { id } = obj; const title = this.notion.extractPageTitle(obj); const ts = Date.now(); @@ -111,10 +111,10 @@ export default { ts, }; }, - emitEvent(page, changes = [], isNewPage = true) { + _emitEvent(page, changes = [], isNewPage = true) { const meta = isNewPage - ? this.generateMeta(page, constants.summaries.PAGE_ADDED) - : this.generateMeta(page, constants.summaries.PAGE_UPDATED); + ? this._generateMeta(page, constants.summaries.PAGE_ADDED) + : this._generateMeta(page, constants.summaries.PAGE_UPDATED); const event = { page, changes, @@ -136,7 +136,7 @@ export default { }, }; let newLastUpdatedTimestamp = lastCheckedTimestamp; - const propertiesToCheck = await this.getPropertiesToCheck(); + const propertiesToCheck = await this._getPropertiesToCheck(); const pagesStream = this.notion.getPages(this.databaseId, params); for await (const page of pagesStream) { @@ -156,7 +156,7 @@ export default { for (const propertyName of propertiesToCheck) { const previousValue = structuredClone(propertyValues[page.id]?.[propertyName]); // value used to compare and to save to this.db - const currentValueToSave = this.maybeRemoveFileSubItems(page.properties[propertyName]); + const currentValueToSave = this._maybeRemoveFileSubItems(page.properties[propertyName]); // (unmodified) value that should be emitted const currentValueToEmit = page.properties[propertyName]; @@ -197,7 +197,7 @@ export default { } if (propertyHasChanged) { - this.emitEvent(page, changes, isNewPage); + this._emitEvent(page, changes, isNewPage); } } From f71cb60ef91063c0c81aa874ed8732129e7b48f6 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 10:11:15 -0300 Subject: [PATCH 2/6] activate and deactivate hooks to restart state --- .../sources/updated-page/updated-page.mjs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 4ecce4505b0b8..b3f7fb722432a 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -66,6 +66,28 @@ export default { this._setPropertyValues(propertyValues); this.setLastUpdatedTimestamp(lastUpdatedTimestamp); }, + async activate() { + console.log("Restarting -- fetching all pages and properties"); + const now = new Date(); + const propertiesToCheck = await this._getPropertiesToCheck(); + const propertyValues = {}; + const params = this.lastUpdatedSortParam(); + const pagesStream = this.notion.getPages(this.databaseId, params); + for await (const page of pagesStream) { + for (const propertyName of propertiesToCheck) { + const currentValue = this._maybeRemoveFileSubItems(page.properties[propertyName]); + propertyValues[page.id] = { + ...propertyValues[page.id], + [propertyName]: currentValue, + }; + } + } + this._setPropertyValues(propertyValues); + this.setLastUpdatedTimestamp(now); + }, + async deactivate() { + this.setLastUpdatedTimestamp(null); + }, }, methods: { ...base.methods, @@ -126,6 +148,11 @@ export default { const lastCheckedTimestamp = this.getLastUpdatedTimestamp(); const propertyValues = this._getPropertyValues(); + if (lastCheckedTimestamp == null) { + // recently updated (deactivated / activated), skip execution + return; + } + const params = { ...this.lastUpdatedSortParam(), filter: { From 594fcec06d988ee4b4ede346222f936fd9b9564e Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 10:12:11 -0300 Subject: [PATCH 3/6] remove deploy hook --- .../sources/updated-page/updated-page.mjs | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index b3f7fb722432a..39db0ca33a482 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -40,32 +40,6 @@ export default { }, }, hooks: { - async deploy() { - const propertiesToCheck = await this._getPropertiesToCheck(); - const propertyValues = {}; - const params = this.lastUpdatedSortParam(); - const pagesStream = this.notion.getPages(this.databaseId, params); - let count = 0; - let lastUpdatedTimestamp = 0; - for await (const page of pagesStream) { - for (const propertyName of propertiesToCheck) { - const currentValue = this._maybeRemoveFileSubItems(page.properties[propertyName]); - propertyValues[page.id] = { - ...propertyValues[page.id], - [propertyName]: currentValue, - }; - } - lastUpdatedTimestamp = Math.max( - lastUpdatedTimestamp, - Date.parse(page.last_edited_time), - ); - if (count++ < 25) { - this._emitEvent(page); - } - } - this._setPropertyValues(propertyValues); - this.setLastUpdatedTimestamp(lastUpdatedTimestamp); - }, async activate() { console.log("Restarting -- fetching all pages and properties"); const now = new Date(); From f0b42a2749b9ecdde267f5713437da3d2625e9e7 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 10:17:01 -0300 Subject: [PATCH 4/6] add logs --- components/notion/sources/updated-page/updated-page.mjs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index 39db0ca33a482..f07412dff7586 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -41,10 +41,10 @@ export default { }, hooks: { async activate() { - console.log("Restarting -- fetching all pages and properties"); - const now = new Date(); - const propertiesToCheck = await this._getPropertiesToCheck(); + console.log("Activating: fetching pages and properties"); + this.setLastUpdatedTimestamp(new Date()); const propertyValues = {}; + const propertiesToCheck = await this._getPropertiesToCheck(); const params = this.lastUpdatedSortParam(); const pagesStream = this.notion.getPages(this.databaseId, params); for await (const page of pagesStream) { @@ -57,9 +57,9 @@ export default { } } this._setPropertyValues(propertyValues); - this.setLastUpdatedTimestamp(now); }, async deactivate() { + console.log("Deactivating: clearing states"); this.setLastUpdatedTimestamp(null); }, }, @@ -124,6 +124,7 @@ export default { if (lastCheckedTimestamp == null) { // recently updated (deactivated / activated), skip execution + console.log("Awaiting restart completion: skipping execution"); return; } From 12b5b5ee53352e1f7cb98d756a8cbdded6482256 Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 11:13:21 -0300 Subject: [PATCH 5/6] save as timestamp in db --- .../notion/sources/updated-page/updated-page.mjs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index f07412dff7586..e7cf0acc85893 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -42,7 +42,7 @@ export default { hooks: { async activate() { console.log("Activating: fetching pages and properties"); - this.setLastUpdatedTimestamp(new Date()); + this._setLastUpdatedTimestamp(Date.now()); const propertyValues = {}; const propertiesToCheck = await this._getPropertiesToCheck(); const params = this.lastUpdatedSortParam(); @@ -60,11 +60,17 @@ export default { }, async deactivate() { console.log("Deactivating: clearing states"); - this.setLastUpdatedTimestamp(null); + this._setLastUpdatedTimestamp(null); }, }, methods: { ...base.methods, + _getLastUpdatedTimestamp() { + return this.db.get(constants.timestamps.LAST_EDITED_TIME); + }, + _setLastUpdatedTimestamp(ts) { + this.db.set(constants.timestamps.LAST_EDITED_TIME, ts); + }, _getPropertyValues() { const compressed = this.db.get("propertyValues"); const buffer = Buffer.from(compressed, "base64"); @@ -119,10 +125,10 @@ export default { }, }, async run() { - const lastCheckedTimestamp = this.getLastUpdatedTimestamp(); + const lastCheckedTimestamp = this._getLastUpdatedTimestamp(); const propertyValues = this._getPropertyValues(); - if (lastCheckedTimestamp == null) { + if (!lastCheckedTimestamp) { // recently updated (deactivated / activated), skip execution console.log("Awaiting restart completion: skipping execution"); return; @@ -203,7 +209,7 @@ export default { } } - this.setLastUpdatedTimestamp(newLastUpdatedTimestamp); + this._setLastUpdatedTimestamp(newLastUpdatedTimestamp); this._setPropertyValues(propertyValues); }, sampleEmit, From bf98504450e8624a7452b6afb982cda0f268cb5a Mon Sep 17 00:00:00 2001 From: Andrew Chuang Date: Wed, 9 Oct 2024 11:15:34 -0300 Subject: [PATCH 6/6] bump versions --- components/notion/package.json | 2 +- components/notion/sources/updated-page/updated-page.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/notion/package.json b/components/notion/package.json index 8e70c22c2c7b2..79139e3951f78 100644 --- a/components/notion/package.json +++ b/components/notion/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/notion", - "version": "0.2.3", + "version": "0.2.4", "description": "Pipedream Notion Components", "main": "notion.app.mjs", "keywords": [ diff --git a/components/notion/sources/updated-page/updated-page.mjs b/components/notion/sources/updated-page/updated-page.mjs index e7cf0acc85893..2c7d16e6e7ed6 100644 --- a/components/notion/sources/updated-page/updated-page.mjs +++ b/components/notion/sources/updated-page/updated-page.mjs @@ -9,7 +9,7 @@ export default { key: "notion-updated-page", name: "Updated Page in Database", /* eslint-disable-line pipedream/source-name */ description: "Emit new event when a page in a database is updated. To select a specific page, use `Updated Page ID` instead", - version: "0.1.2", + version: "0.1.3", type: "source", dedupe: "unique", props: {