New Components - ragie

components/ragie/actions/create-document/create-document.mjs

@@ -0,0 +1,71 @@
+import FormData from "form-data";
+import fs from "fs";
+import { checkTmp } from "../../common/utils.mjs";
+import ragie from "../../ragie.app.mjs";
+
+export default {
+  key: "ragie-create-document",
+  name: "Create Document",
+  description: "Creates a new document in Ragie. [See the documentation](https://docs.ragie.ai/reference/createdocument)",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    ragie,
+    file: {
+      propDefinition: [
+        ragie,
+        "createDocumentFile",
+      ],
+    },
+    mode: {
+      propDefinition: [
+        ragie,
+        "createDocumentMode",
+      ],
+      optional: true,
+    },
+    metadata: {
+      type: "object",
+      label: "Metadata",
+      description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.",
+      optional: true,
+    },
+    externalId: {
+      type: "string",
+      label: "External ID",
+      description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.",
+      optional: true,
+    },
+    name: {
+      type: "string",
+      label: "Name",
+      description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.",
+      optional: true,
+    },
+    partition: {
+      propDefinition: [
+        ragie,
+        "partition",
+      ],
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const data = new FormData();
+    data.append("file", fs.createReadStream(checkTmp(this.file)));
+    if (this.mode) data.append("mode", this.mode);
+    if (this.metadata) data.append("metadata", JSON.stringify(this.metadata));
+    if (this.externalId) data.append("external_id", this.externalId);
+    if (this.name) data.append("name", this.name);
+    if (this.partition) data.append("partition", this.partition);
+
+    const response = await this.ragie.createDocument({
+      $,
+      data,
+      headers: data.getHeaders(),
+    });
+
+    $.export("$summary", `Created document: ${response.name} (ID: ${response.id})`);
+    return response;
+  },
+};

components/ragie/actions/update-document-file/update-document-file.mjs

@@ -0,0 +1,48 @@
+import FormData from "form-data";
+import fs from "fs";
+import { checkTmp } from "../../common/utils.mjs";
+import ragie from "../../ragie.app.mjs";
+
+export default {
+  key: "ragie-update-document-file",
+  name: "Update Document File",
+  description: "Updates an existing document file in Ragie. [See the documentation](https://docs.ragie.ai/reference/updatedocumentfile).",
+  version: "0.0.1",
+  type: "action",
+  props: {
+    ragie,
+    documentId: {
+      propDefinition: [
+        ragie,
+        "documentId",
+      ],
+    },
+    mode: {
+      propDefinition: [
+        ragie,
+        "createDocumentMode",
+      ],
+      optional: true,
+    },
+    file: {
+      propDefinition: [
+        ragie,
+        "createDocumentFile",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const data = new FormData();
+    data.append("file", fs.createReadStream(checkTmp(this.file)));
+    if (this.mode) data.append("mode", this.mode);
+
+    const response = await this.ragie.updateDocumentFile({
+      $,
+      documentId: this.documentId,
+      data,
+      headers: data.getHeaders(),
+    });
+    $.export("$summary", `Successfully updated document file with ID: ${this.documentId}`);
+    return response;
+  },
+};

components/ragie/common/constants.mjs

@@ -0,0 +1,15 @@
+export const SCOPE_OPTIONS = [
+  {
+    label: "Document",
+    value: "document",
+  },
+  {
+    label: "Chunk",
+    value: "chunk",
+  },
+];
+
+export const DOCUMENT_MODE_OPTIONS = [
+  "hi_res",
+  "fast",
+];

components/ragie/common/utils.mjs

@@ -0,0 +1,31 @@
+export const checkTmp = (filename) => {
+  if (!filename.startsWith("/tmp")) {
+    return `/tmp/${filename}`;
+  }
+  return filename;
+};
+
+export const parseObject = (obj) => {
+  if (!obj) return undefined;
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "string") {
+        try {
+          return JSON.parse(item);
+        } catch (e) {
+          return item;
+        }
+      }
+      return item;
+    });
+  }
+  if (typeof obj === "string") {
+    try {
+      return JSON.parse(obj);
+    } catch (e) {
+      return obj;
+    }
+  }
+  return obj;
+};

components/ragie/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/ragie",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream Ragie Components",
   "main": "ragie.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <[email protected]> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.3"
   }
-}
+}

components/ragie/ragie.app.mjs

@@ -1,11 +1,127 @@
+import { axios } from "@pipedream/platform";
+import { DOCUMENT_MODE_OPTIONS } from "./common/constants.mjs";
+
 export default {
   type: "app",
   app: "ragie",
-  propDefinitions: {},
+  propDefinitions: {
+    documentFile: {
+      type: "string",
+      label: "File",
+      description: "The path to the file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory).",
+    },
+    documentMode: {
+      type: "string",
+      label: "Mode",
+      description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.",
+      optional: true,
+      options: DOCUMENT_MODE_OPTIONS,
+    },
+    partition: {
+      type: "string",
+      label: "Partition",
+      description: "An optional partition identifier. Partitions must be lowercase alphanumeric and may only include the special characters '_' and '-'.",
+    },
+    documentId: {
+      type: "string",
+      label: "Document ID",
+      description: "The ID of the document to update.",
+      async options({ prevContext }) {
+        const {
+          documents, pagination,
+        } = await this.listDocuments({
+          params: {
+            cursor: prevContext.cursor,
+          },
+        });
+        return {
+          options: documents.map(({
+            id: value, name: label,
+          }) => ({
+            label,
+            value,
+          })),
+          context: {
+            cursor: pagination.next_cursor,
+          },
+        };
+      },
+    },
+  },
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    _baseUrl() {
+      return "https://api.ragie.ai";
+    },
+    _headers(headers = {}) {
+      return {
+        ...headers,
+        "Authorization": `Bearer ${this.$auth.api_key}`,
+      };
+    },
+    _makeRequest({
+      $ = this, path, headers, ...opts
+    }) {
+      return axios($, {
+        url: this._baseUrl() + path,
+        headers: this._headers(headers),
+        ...opts,
+      });
+    },
+    createDocument(opts = {}) {
+      return this._makeRequest({
+        method: "POST",
+        path: "/documents",
+        ...opts,
+      });
+    },
+    listDocuments(opts = {}) {
+      return this._makeRequest({
+        path: "/documents",
+        params: opts,
+      });
+    },
+    updateDocumentFile({
+      documentId, ...opts
+    }) {
+      return this._makeRequest({
+        method: "PUT",
+        path: `/documents/${documentId}/file`,
+        ...opts,
+      });
+    },
+    listConnections(opts = {}) {
+      return this._makeRequest({
+        path: "/connections",
+        params: opts,
+      });
+    },
+    async *paginate({
+      fn, params = {}, fieldName, maxResults = null, ...opts
+    }) {
+      let count = 0;
+      let nextCursor;
+
+      do {
+        params.cursor = nextCursor;
+        const {
+          pagination,
+          ...data
+        } = await fn({
+          params,
+          ...opts,
+        });
+        const items = data[fieldName];
+
+        for (const d of items) {
+          yield d;
+
+          if (maxResults && ++count === maxResults) {
+            return count;
+          }
+        }
+
+        nextCursor = pagination?.next_cursor;
+      } while (nextCursor);
     },
   },
 };

components/ragie/sources/common/base.mjs

@@ -0,0 +1,60 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import ragie from "../../ragie.app.mjs";
+
+export default {
+  props: {
+    ragie,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+  },
+  methods: {
+    _getLastDate() {
+      return this.db.get("lastDate") || 0;
+    },
+    _setLastDate(lastDate) {
+      this.db.set("lastDate", lastDate);
+    },
+    async emitEvent(maxResults = false) {
+      const lastDate = this._getLastDate();
+
+      const response = this.ragie.paginate({
+        fn: this.getFunction(),
+        fieldName: this.getFieldName(),
+      });
+
+      let responseArray = [];
+      for await (const item of response) {
+        if (Date.parse(item.created_at) <= lastDate) break;
+        responseArray.push(item);
+      }
+
+      if (responseArray.length) {
+        if (maxResults && (responseArray.length > maxResults)) {
+          responseArray.length = maxResults;
+        }
+        this._setLastDate(Date.parse(responseArray[0].created_at));
+      }
+
+      for (const item of responseArray.reverse()) {
+        this.$emit(item, {
+          id: item.id,
+          summary: this.getSummary(item),
+          ts: Date.parse(item.created_at || new Date()),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};