New Components - vectorshift

components/botsonic/botsonic.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/piped/piped.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/rocketadmin/rocketadmin.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/supadata/supadata.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/timelink/timelink.app.mjs

@@ -8,4 +8,4 @@ export default {
       console.log(Object.keys(this.$auth));
     },
   },
-};
+};

components/vectorshift/actions/add-data-to-knowledge-base/add-data-to-knowledge-base.mjs

@@ -0,0 +1,56 @@
+import { RESCRAPE_FREQUENCY_OPTIONS } from "../../common/constants.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-add-data-to-knowledge-base",
+  name: "Add Data to Knowledge Base",
+  description: "Adds data to a knowledge base in VectorShift. [See the documentation](https://docs.vectorshift.ai/api-reference/knowledge-bases/index).",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    knowledgeBaseId: {
+      propDefinition: [
+        vectorshift,
+        "knowledgeBaseId",
+      ],
+    },
+    url: {
+      type: "string",
+      label: "URL",
+      description: "URL to add to the knowledge base",
+    },
+    recursive: {
+      type: "boolean",
+      label: "Recursive",
+      description: "Whether the scrape is recursive or not",
+      default: false,
+    },
+    rescrapeFrequency: {
+      type: "string",
+      label: "Rescrape Frequency",
+      description: "The frequency to rescrape the URL",
+      options: RESCRAPE_FREQUENCY_OPTIONS,
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.vectorshift.addDataToKnowledgeBase({
+      $,
+      knowledgeBaseId: this.knowledgeBaseId,
+      data: {
+        url_data: {
+          request: {
+            url: this.url,
+            recursive: this.recursive,
+            return_type: "CONTENT",
+          },
+          rescrape_frequency: this.rescrapeFrequency,
+        },
+      },
+    });
+
+    $.export("$summary", `Added ${response.document_ids.length} document(s) to knowledge base ${this.knowledgeBaseId}. Document IDs: ${response.document_ids.join(", ")}`);
+    return response;
+  },
+};

components/vectorshift/actions/create-pipeline/create-pipeline.mjs

@@ -0,0 +1,48 @@
+import { ConfigurationError } from "@pipedream/platform";
+import { parseObject } from "../../common/utils.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-create-pipeline",
+  name: "Create Pipeline",
+  description: "Creates a new pipeline in VectorShift. [See the documentation](https://docs.vectorshift.ai)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    name: {
+      type: "string",
+      label: "Pipeline Name",
+      description: "Name of the new pipeline",
+    },
+    config: {
+      type: "object",
+      label: "Pipeline Config",
+      description: "Configuration for the new pipeline",
+    },
+    description: {
+      type: "string",
+      label: "Description",
+      description: "Optional description of the new pipeline",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    try {
+      const response = await this.vectorshift.createPipeline({
+        $,
+        data: {
+          name: this.name,
+          config: parseObject(this.config),
+          description: this.description,
+        },
+      });
+
+      $.export("$summary", `Created pipeline with ID ${response.id}`);
+      return response;
+    } catch ({ message }) {
+      const parsedError = JSON.parse(message).error;
+      throw new ConfigurationError(parsedError);
+    }
+  },
+};

components/vectorshift/actions/run-pipeline/run-pipeline.mjs

@@ -0,0 +1,42 @@
+import { ConfigurationError } from "@pipedream/platform";
+import { parseObject } from "../../common/utils.mjs";
+import vectorshift from "../../vectorshift.app.mjs";
+
+export default {
+  key: "vectorshift-run-pipeline",
+  name: "Run Pipeline",
+  description: "Executes a VectorShift pipeline with specified inputs. [See the documentation](https://docs.vectorshift.ai/api-reference/pipelines/run)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    vectorshift,
+    pipelineId: {
+      propDefinition: [
+        vectorshift,
+        "pipelineId",
+      ],
+    },
+    inputs: {
+      type: "object",
+      label: "Pipeline Inputs",
+      description: "Inputs for the pipeline execution. [See the documentation](https://docs.vectorshift.ai/platform/pipelines/general/input) for further details",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    try {
+      const response = await this.vectorshift.executePipeline({
+        $,
+        pipelineId: this.pipelineId,
+        data: {
+          inputs: parseObject(this.inputs),
+        },
+      });
+      $.export("$summary", `Pipeline executed successfully. Run ID: ${response.run_id}`);
+      return response;
+    } catch ({ message }) {
+      const parsedError = JSON.parse(message).error;
+      throw new ConfigurationError(parsedError);
+    }
+  },
+};

components/vectorshift/common/constants.mjs

@@ -0,0 +1,15 @@
+export const DATA_TYPE_OPTIONS = [
+  "url",
+  "wikipedia",
+  "youtube",
+  "arxiv",
+  "git",
+];
+
+export const RESCRAPE_FREQUENCY_OPTIONS = [
+  "Never",
+  "Hourly",
+  "Daily",
+  "Weekly",
+  "Monthly",
+];

components/vectorshift/common/utils.mjs

@@ -0,0 +1,31 @@
+export const checkTmp = (filename) => {
+  if (!filename.startsWith("/tmp")) {
+    return `/tmp/${filename}`;
+  }
+  return filename;
+};
+
+export const parseObject = (obj) => {
+  if (!obj) return undefined;
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "string") {
+        try {
+          return JSON.parse(item);
+        } catch (e) {
+          return item;
+        }
+      }
+      return item;
+    });
+  }
+  if (typeof obj === "string") {
+    try {
+      return JSON.parse(obj);
+    } catch (e) {
+      return obj;
+    }
+  }
+  return obj;
+};

components/vectorshift/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/vectorshift",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream VectorShift Components",
   "main": "vectorshift.app.mjs",
   "keywords": [
@@ -11,5 +11,9 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.3",
+    "fs": "^0.0.1-security"
   }
-}
+}

components/vectorshift/sources/common/base.mjs

@@ -0,0 +1,42 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import app from "../../vectorshift.app.mjs";
+
+export default {
+  props: {
+    app,
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+  },
+  methods: {
+    async emitEvent(maxResults = false) {
+      const fn = this.getFunction();
+      const { objects: response = [] } = await fn();
+
+      if (response.length) {
+        if (maxResults && (response.length > maxResults)) {
+          response.length = maxResults;
+        }
+      }
+
+      for (const item of response) {
+        this.$emit(item, {
+          id: item._id,
+          summary: this.getSummary(item),
+          ts: Date.parse(item.createdDate || new Date()),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};

components/vectorshift/sources/new-chatbot/new-chatbot.mjs

@@ -0,0 +1,22 @@
+import common from "../common/base.mjs";
+import sampleEmit from "./test-event.mjs";
+
+export default {
+  ...common,
+  key: "vectorshift-new-chatbot",
+  name: "New Chatbot Created",
+  description: "Emit new event when a chatbot is created.",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getFunction() {
+      return this.app.listChatbots;
+    },
+    getSummary(item) {
+      return `New Chatbot: ${item.name || item._id}`;
+    },
+  },
+  sampleEmit,
+};