updates

Author: michelle0927

components/egnyte/egnyte.app.mjs

@@ -1,4 +1,10 @@
 import { axios } from "@pipedream/platform";
+import Bottleneck from "bottleneck";
+const limiter = new Bottleneck({
+  minTime: 500, // 2 requests per second
+  maxConcurrent: 1,
+});
+const axiosRateLimiter = limiter.wrap(axios);
 
 export default {
   type: "app",
@@ -13,28 +19,24 @@ export default {
       headers,
       ...otherOpts
     }) {
-      return axios($ || this, {
+      const config = {
         url: `${this._baseUrl()}${path}`,
         headers: {
           ...headers,
           Authorization: `Bearer ${this.$auth.oauth_access_token}`,
         },
         ...otherOpts,
-      });
+      };
+      return axiosRateLimiter($, config);
     },
-    createWebhook(opts = {}) {
+    getFolder({
+      folderPath, ...opts
+    }) {
       return this._makeRequest({
-        method: "POST",
-        path: "/webhooks",
+        path: `/fs/${folderPath}`,
         ...opts,
       });
     },
-    deleteWebhook({ hookId }) {
-      return this._makeRequest({
-        method: "DELETE",
-        path: `/webhooks/${hookId}`,
-      });
-    },
     createFolder({ folderPath }) {
       return this._makeRequest({
         method: "POST",

components/egnyte/sources/common/base.mjs

@@ -1,66 +1,80 @@
 import egnyte from "../../egnyte.app.mjs";
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 
 export default {
   props: {
     egnyte,
     db: "$.service.db",
-    http: {
-      type: "$.interface.http",
-      customResponse: true,
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
     },
-    folderPaths: {
-      type: "string[]",
-      label: "Folder Paths",
-      description: "An array of folder paths (example: `/Shared/Documents`) to watch for updates. You may specify up to 100 paths.",
-    },
-  },
-  hooks: {
-    async activate() {
-      const { webhookId } = await this.egnyte.createWebhook({
-        data: {
-          url: this.http.endpoint,
-          eventType: this.getEventType(),
-          path: this.folderPaths.join(","),
-        },
-      });
-      this._setHookId(webhookId);
-    },
-    async deactivate() {
-      const hookId = this._getHookId();
-      if (hookId) {
-        await this.egnyte.deleteWebhook({
-          hookId,
-        });
-      }
+    folderPath: {
+      type: "string",
+      label: "Folder Path",
+      description: "The folder path (example: `/Shared/Documents`) to watch for updates.",
     },
   },
   methods: {
-    _getHookId() {
-      return this.db.get("hookId");
+    _getLastTs() {
+      return this.db.get("lastTs") || 0;
+    },
+    _setLastTs(lastTs) {
+      this.db.set("lastTs", lastTs);
     },
-    _setHookId(hookId) {
-      this.db.set("hookId", hookId);
+    getResourceType() {
+      throw new Error("getResourceType is not implemented");
     },
     generateMeta() {
       throw new Error("generateMeta is not implemented");
     },
-    getEventType() {
-      throw new Error("getEventType is not implemented");
-    },
   },
-  async run(event) {
-    this.http.respond({
-      status: 200,
-    });
+  async run() {
+    const lastTs = this._getLastTs();
+    let maxTs = lastTs;
+    const resourceType = this.getResourceType();
+
+    // Recursively process folder and subfolders
+    const processFolder = async (folderPath) => {
+      const results = await this.egnyte.getFolder({
+        folderPath,
+        params: {
+          sort_by: "last_modified",
+          sort_direction: "descending",
+        },
+      });
+
+      const items = results[resourceType];
+      if (!items?.length) {
+        return;
+      }
+      const newItems = [];
+
+      for (const item of items) {
+        const ts = item.uploaded;
+        if (ts >= lastTs) {
+          newItems.push(item);
+          maxTs = Math.max(ts, maxTs);
+        }
+      }
+
+      const folders = results.folders;
+      if (folders?.length) {
+        for (const folder of folders) {
+          await processFolder(folder.path);
+        }
+      }
+
+      newItems.reverse().forEach((item) => {
+        const meta = this.generateMeta(item);
+        this.$emit(item, meta);
+      });
+    };
 
-    const { body } = event;
-    if (!body || !body?.length) {
-      return;
-    }
+    await processFolder(this.folderPath);
 
-    for (const item of body) {
-      const meta = this.generateMeta(item); console.log(meta);
-      this.$emit(item, meta);
-    }
+    this._setLastTs(maxTs);
   },
 };

components/egnyte/sources/new-file-in-folder/new-file-in-folder.mjs

@@ -3,21 +3,21 @@ import common from "../common/base.mjs";
 export default {
   ...common,
   key: "egnyte-new-file-in-folder",
-  name: "New File in Folder (Instant)",
-  description: "Emit new event when a file is added to the specified folder(s) in Egnyte. [See the documentation](https://storage.googleapis.com/pint-internal-static-hosting/connectWebhooks/index.html#operation/registerWebhook)",
+  name: "New File in Folder",
+  description: "Emit new event when a file is added within the specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/read/File_System_Management_API_Documentation#List-File-or-Folder)",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
   methods: {
     ...common.methods,
-    getEventType() {
-      return "fs:add_file";
+    getResourceType() {
+      return "files";
     },
-    generateMeta(event) {
+    generateMeta(file) {
       return {
-        id: event.id,
-        summary: "New File Added",
-        ts: event.timestamp,
+        id: file.entry_id,
+        summary: `New file: ${file.name}`,
+        ts: file.uploaded,
       };
     },
   },

components/egnyte/sources/new-folder-added/new-folder-added.mjs

@@ -3,21 +3,21 @@ import common from "../common/base.mjs";
 export default {
   ...common,
   key: "egnyte-new-folder-added",
-  name: "New Folder (Instant)",
-  description: "Emit new event when a folder is added to the specified folder(s) in Egnyte. [See the documentation](https://storage.googleapis.com/pint-internal-static-hosting/connectWebhooks/index.html#operation/registerWebhook).",
+  name: "New Folder",
+  description: "Emit new event when a folder is added within the specified folder in Egnyte. [See the documentation](https://developers.egnyte.com/docs/read/File_System_Management_API_Documentation#List-File-or-Folder).",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
   methods: {
     ...common.methods,
-    getEventType() {
-      return "fs:add_folder";
+    getResourceType() {
+      return "folders";
     },
-    generateMeta(event) {
+    generateMeta(folder) {
       return {
-        id: event.id,
-        summary: "New Folder Added",
-        ts: event.timestamp,
+        id: folder.folder_id,
+        summary: `New folder: ${folder.name}`,
+        ts: folder.uploaded,
       };
     },
   },