sftp_password_based_auth-watch-remote-directory and refactor (#14940)

Author: jcortes

components/sftp/actions/upload-file/upload-file.mjs

@@ -6,7 +6,7 @@ export default {
   key: "sftp-upload-file",
   name: "Upload File",
   description: "Uploads a file or string in UTF-8 format to the SFTP server. [See the documentation](https://github.com/theophilusx/ssh2-sftp-client#org99d1b64)",
-  version: "0.2.0",
+  version: "0.3.0",
   type: "action",
   props: {
     app,

components/sftp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/sftp",
-  "version": "0.3.8",
+  "version": "0.4.0",
   "description": "Pipedream SFTP Components",
   "main": "sftp.app.mjs",
   "keywords": [
@@ -10,8 +10,8 @@
   "homepage": "https://pipedream.com/apps/sftp",
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "dependencies": {
-    "@pipedream/platform": "^1.2.0",
-    "ssh2-sftp-client": "^8.1.0"
+    "@pipedream/platform": "^3.0.3",
+    "ssh2-sftp-client": "^11.0.0"
   },
   "gitHead": "e12480b94cc03bed4808ebc6b13e7fdb3a1ba535",
   "publishConfig": {

components/sftp/sftp.app.mjs

@@ -20,23 +20,27 @@ export default {
     },
   },
   methods: {
-    async connect() {
+    getConfig() {
       const {
         host,
         username,
         privateKey,
       } = this.$auth;
 
-      const config = {
+      return {
         host,
         username,
         privateKey,
       };
-
+    },
+    async connect() {
       const sftp = new Client();
-      await sftp.connect(config);
+      await sftp.connect(this.getConfig());
       return sftp;
     },
+    async disconnect(sftp) {
+      await sftp.end();
+    },
     async put({
       buffer,
       remotePath,

components/sftp/sources/common/base.mjs

@@ -1,15 +1,16 @@
-import sftpApp from "../../sftp.app.mjs";
+import { createHash } from "crypto";
+import app from "../../sftp.app.mjs";
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 
 export default {
   key: "sftp-watch-remote-directory",
   name: "New Remote Directory Watcher",
   description: "Emit new events when files get created, changed or deleted from a remote directory. [See the docs](https://github.com/theophilusx/ssh2-sftp-client#orgfac43d1)",
-  version: "0.0.4",
+  version: "0.1.0",
   type: "source",
   dedupe: "unique",
   props: {
-    sftpApp,
+    app,
     timer: {
       type: "$.interface.timer",
       default: {
@@ -121,9 +122,15 @@ export default {
       }
     },
     getEventId(event) {
-      return `${event.event}|${event.path}|${event.size}|${event.modifyTime}`;
+      const uniqueIdentifier =
+        createHash("md5")
+          .update(event.path + event.modifyTime)
+          .digest("hex");
+      return `${event.event}|${event.path}|${event.size}|${event.modifyTime}|${uniqueIdentifier}`;
     },
     emitEvents(events) {
+      // sort events by modification time in ascending order
+      events.sort((a, b) => a.modifyTime - b.modifyTime);
       for (const fileChangeEvent of events) {
         this.$emit(fileChangeEvent, {
           id: this.getEventId(fileChangeEvent),
@@ -132,17 +139,11 @@ export default {
         });
       }
     },
-    async connect() {
-      return await this.sftpApp.connect();
-    },
-    async disconnect(sftp) {
-      await sftp.end();
-    },
   },
   async run() {
     this.validateRootDirectory(this.rootDirectory);
 
-    const sftp = await this.connect();
+    const sftp = await this.app.connect();
 
     let directories = [
       {
@@ -159,6 +160,6 @@ export default {
     this.emitEvents(filesChangesWithEvent);
 
     this.setFiles(currentFiles);
-    await this.disconnect(sftp);
+    await this.app.disconnect(sftp);
   },
 };

components/sftp/sources/watch-remote-directory/watch-remote-directory.mjs

@@ -1,44 +1,14 @@
-// legacy_hash_id: a_YEikdQ
-import Client from "ssh2-sftp-client";
+import utils from "../../common/utils.mjs";
+import component from "../../../sftp/actions/upload-file/upload-file.mjs";
 
 export default {
+  ...component,
+  props: utils.buildAppProps({
+    component,
+  }),
   key: "sftp_password_based_auth-upload-file",
-  name: "Upload String as File",
-  description: "Uploads a UTF-8 string as a file on an SFTP server",
-  version: "0.1.1",
+  name: "Upload File (Password Auth)",
+  description: "Uploads a file or string in UTF-8 format to the SFTP server. [See the documentation](https://github.com/theophilusx/ssh2-sftp-client#org99d1b64)",
+  version: "0.2.0",
   type: "action",
-  props: {
-    sftp_password_based_auth: {
-      type: "app",
-      app: "sftp_password_based_auth",
-    },
-    data: {
-      type: "string",
-      description: "A UTF-8 string to upload as a file on the remote server.",
-    },
-    remotePath: {
-      type: "string",
-      label: "Remote Path",
-      description: "The path to the remote file to be created on the server.",
-    },
-  },
-  async run({ $ }) {
-    const {
-      host,
-      username,
-      password,
-    } = this.sftp_password_based_auth.$auth;
-
-    const config = {
-      host,
-      username,
-      password,
-    };
-
-    const sftp = new Client();
-
-    await sftp.connect(config);
-    $.export("putResponse", await sftp.put(Buffer.from(this.data), this.remotePath));
-    await sftp.end();
-  },
 };

components/sftp_password_based_auth/actions/upload-file/upload-file.mjs

@@ -0,0 +1,79 @@
+import app from "../sftp_password_based_auth.app.mjs";
+
+function buildPropDefinitions({
+  app = {}, props = {},
+}) {
+  return Object.entries(props)
+    .reduce((newProps, [
+      key,
+      prop,
+    ]) => {
+      if (!prop.propDefinition) {
+        return Object.assign(newProps, {
+          [key]: prop,
+        });
+      }
+
+      const [
+        , ...propDefinitionItems
+      ] = prop.propDefinition;
+
+      return Object.assign(newProps, {
+        [key]: Object.assign(prop, {
+          propDefinition: [
+            app,
+            ...propDefinitionItems,
+          ],
+        }),
+      });
+    }, {});
+}
+
+function getPropsOnly({
+  component, omitProps = [], appLabel, addedProps = {},
+} = {}) {
+  const {
+    // eslint-disable-next-line no-unused-vars
+    [appLabel]: appProp,
+    ...props
+  } = component.props;
+  const builtProps = Object.entries(props)
+    .reduce((reduction, [
+      key,
+      value,
+    ]) => {
+      if (omitProps.includes(key)) {
+        return reduction;
+      }
+      return Object.assign(reduction, {
+        [key]: value,
+      });
+    }, {});
+  return {
+    ...builtProps,
+    ...addedProps,
+  };
+}
+
+function buildAppProps({
+  component, omitProps = [], appLabel = "app", addedProps,
+} = {}) {
+  return {
+    [appLabel]: app,
+    ...buildPropDefinitions({
+      app,
+      props: getPropsOnly({
+        component,
+        omitProps,
+        appLabel,
+        addedProps,
+      }),
+    }),
+  };
+}
+
+export default {
+  buildPropDefinitions,
+  getPropsOnly,
+  buildAppProps,
+};