diff --git a/components/ragie/actions/create-document/create-document.mjs b/components/ragie/actions/create-document/create-document.mjs new file mode 100644 index 0000000000000..70fada4cf4a38 --- /dev/null +++ b/components/ragie/actions/create-document/create-document.mjs @@ -0,0 +1,71 @@ +import FormData from "form-data"; +import fs from "fs"; +import { checkTmp } from "../../common/utils.mjs"; +import ragie from "../../ragie.app.mjs"; + +export default { + key: "ragie-create-document", + name: "Create Document", + description: "Creates a new document in Ragie. [See the documentation](https://docs.ragie.ai/reference/createdocument)", + version: "0.0.1", + type: "action", + props: { + ragie, + file: { + propDefinition: [ + ragie, + "documentFile", + ], + }, + mode: { + propDefinition: [ + ragie, + "documentMode", + ], + optional: true, + }, + metadata: { + type: "object", + label: "Metadata", + description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.", + optional: true, + }, + externalId: { + type: "string", + label: "External ID", + description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.", + optional: true, + }, + name: { + type: "string", + label: "Name", + description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.", + optional: true, + }, + partition: { + propDefinition: [ + ragie, + "partition", + ], + optional: true, + }, + }, + async run({ $ }) { + const data = new FormData(); + data.append("file", fs.createReadStream(checkTmp(this.file))); + if (this.mode) data.append("mode", this.mode); + if (this.metadata) data.append("metadata", JSON.stringify(this.metadata)); + if (this.externalId) data.append("external_id", this.externalId); + if (this.name) data.append("name", this.name); + if (this.partition) data.append("partition", this.partition); + + const response = await this.ragie.createDocument({ + $, + data, + headers: data.getHeaders(), + }); + + $.export("$summary", `Created document: ${response.name} (ID: ${response.id})`); + return response; + }, +}; diff --git a/components/ragie/actions/update-document-file/update-document-file.mjs b/components/ragie/actions/update-document-file/update-document-file.mjs new file mode 100644 index 0000000000000..e62730e0a6034 --- /dev/null +++ b/components/ragie/actions/update-document-file/update-document-file.mjs @@ -0,0 +1,48 @@ +import FormData from "form-data"; +import fs from "fs"; +import { checkTmp } from "../../common/utils.mjs"; +import ragie from "../../ragie.app.mjs"; + +export default { + key: "ragie-update-document-file", + name: "Update Document File", + description: "Updates an existing document file in Ragie. [See the documentation](https://docs.ragie.ai/reference/updatedocumentfile).", + version: "0.0.1", + type: "action", + props: { + ragie, + documentId: { + propDefinition: [ + ragie, + "documentId", + ], + }, + mode: { + propDefinition: [ + ragie, + "documentMode", + ], + optional: true, + }, + file: { + propDefinition: [ + ragie, + "documentFile", + ], + }, + }, + async run({ $ }) { + const data = new FormData(); + data.append("file", fs.createReadStream(checkTmp(this.file))); + if (this.mode) data.append("mode", this.mode); + + const response = await this.ragie.updateDocumentFile({ + $, + documentId: this.documentId, + data, + headers: data.getHeaders(), + }); + $.export("$summary", `Successfully updated document file with ID: ${this.documentId}`); + return response; + }, +}; diff --git a/components/ragie/common/constants.mjs b/components/ragie/common/constants.mjs new file mode 100644 index 0000000000000..d681c66f145c1 --- /dev/null +++ b/components/ragie/common/constants.mjs @@ -0,0 +1,15 @@ +export const SCOPE_OPTIONS = [ + { + label: "Document", + value: "document", + }, + { + label: "Chunk", + value: "chunk", + }, +]; + +export const DOCUMENT_MODE_OPTIONS = [ + "hi_res", + "fast", +]; diff --git a/components/ragie/common/utils.mjs b/components/ragie/common/utils.mjs new file mode 100644 index 0000000000000..0cd1a12b6a4ba --- /dev/null +++ b/components/ragie/common/utils.mjs @@ -0,0 +1,31 @@ +export const checkTmp = (filename) => { + if (!filename.startsWith("/tmp")) { + return `/tmp/${filename}`; + } + return filename; +}; + +export const parseObject = (obj) => { + if (!obj) return undefined; + + if (Array.isArray(obj)) { + return obj.map((item) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + return item; + } + } + return item; + }); + } + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + return obj; +}; diff --git a/components/ragie/package.json b/components/ragie/package.json index 6d34e5ccb42c8..b331423650937 100644 --- a/components/ragie/package.json +++ b/components/ragie/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/ragie", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Ragie Components", "main": "ragie.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/components/ragie/ragie.app.mjs b/components/ragie/ragie.app.mjs index 763d081ee89fe..ea3c95b014d66 100644 --- a/components/ragie/ragie.app.mjs +++ b/components/ragie/ragie.app.mjs @@ -1,11 +1,127 @@ +import { axios } from "@pipedream/platform"; +import { DOCUMENT_MODE_OPTIONS } from "./common/constants.mjs"; + export default { type: "app", app: "ragie", - propDefinitions: {}, + propDefinitions: { + documentFile: { + type: "string", + label: "File", + description: "The path to the file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory).", + }, + documentMode: { + type: "string", + label: "Mode", + description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.", + optional: true, + options: DOCUMENT_MODE_OPTIONS, + }, + partition: { + type: "string", + label: "Partition", + description: "An optional partition identifier. Partitions must be lowercase alphanumeric and may only include the special characters '_' and '-'.", + }, + documentId: { + type: "string", + label: "Document ID", + description: "The ID of the document to update.", + async options({ prevContext }) { + const { + documents, pagination, + } = await this.listDocuments({ + params: { + cursor: prevContext.cursor, + }, + }); + return { + options: documents.map(({ + id: value, name: label, + }) => ({ + label, + value, + })), + context: { + cursor: pagination.next_cursor, + }, + }; + }, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.ragie.ai"; + }, + _headers(headers = {}) { + return { + ...headers, + "Authorization": `Bearer ${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, headers, ...opts + }) { + return axios($, { + url: this._baseUrl() + path, + headers: this._headers(headers), + ...opts, + }); + }, + createDocument(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/documents", + ...opts, + }); + }, + listDocuments(opts = {}) { + return this._makeRequest({ + path: "/documents", + params: opts, + }); + }, + updateDocumentFile({ + documentId, ...opts + }) { + return this._makeRequest({ + method: "PUT", + path: `/documents/${documentId}/file`, + ...opts, + }); + }, + listConnections(opts = {}) { + return this._makeRequest({ + path: "/connections", + params: opts, + }); + }, + async *paginate({ + fn, params = {}, fieldName, maxResults = null, ...opts + }) { + let count = 0; + let nextCursor; + + do { + params.cursor = nextCursor; + const { + pagination, + ...data + } = await fn({ + params, + ...opts, + }); + const items = data[fieldName]; + + for (const d of items) { + yield d; + + if (maxResults && ++count === maxResults) { + return count; + } + } + + nextCursor = pagination?.next_cursor; + } while (nextCursor); }, }, }; diff --git a/components/ragie/sources/common/base.mjs b/components/ragie/sources/common/base.mjs new file mode 100644 index 0000000000000..d09ddc09671a2 --- /dev/null +++ b/components/ragie/sources/common/base.mjs @@ -0,0 +1,60 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import ragie from "../../ragie.app.mjs"; + +export default { + props: { + ragie, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getLastDate() { + return this.db.get("lastDate") || 0; + }, + _setLastDate(lastDate) { + this.db.set("lastDate", lastDate); + }, + async emitEvent(maxResults = false) { + const lastDate = this._getLastDate(); + + const response = this.ragie.paginate({ + fn: this.getFunction(), + fieldName: this.getFieldName(), + }); + + let responseArray = []; + for await (const item of response) { + if (Date.parse(item.created_at) <= lastDate) break; + responseArray.push(item); + } + + if (responseArray.length) { + if (maxResults && (responseArray.length > maxResults)) { + responseArray.length = maxResults; + } + this._setLastDate(Date.parse(responseArray[0].created_at)); + } + + for (const item of responseArray.reverse()) { + this.$emit(item, { + id: item.id, + summary: this.getSummary(item), + ts: Date.parse(item.created_at || new Date()), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, +}; diff --git a/components/ragie/sources/new-connection/new-connection.mjs b/components/ragie/sources/new-connection/new-connection.mjs new file mode 100644 index 0000000000000..43a70dc255e20 --- /dev/null +++ b/components/ragie/sources/new-connection/new-connection.mjs @@ -0,0 +1,27 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "ragie-new-connection", + name: "New Ragie Connection Created", + description: "Emit new event whenever a new connection is created in Ragie. [See the documentation](https://docs.ragie.ai/reference/list_connections_connections_get)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getFunction() { + return this.ragie.listConnections; + }, + getFieldName() { + return "connections"; + }, + getSummary({ + name, type, + }) { + return `New Ragie Connection: ${name} (${type})`; + }, + }, + sampleEmit, +}; diff --git a/components/ragie/sources/new-connection/test-event.mjs b/components/ragie/sources/new-connection/test-event.mjs new file mode 100644 index 0000000000000..27c452a389fe9 --- /dev/null +++ b/components/ragie/sources/new-connection/test-event.mjs @@ -0,0 +1,11 @@ +export default { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2025-01-16T18:44:13.901Z", + "updated_at": "2025-01-16T18:44:13.901Z", + "metadata": {}, + "type": "string", + "name": "string", + "enabled": true, + "last_synced_at": "2025-01-16T18:44:13.901Z", + "syncing": true +} \ No newline at end of file diff --git a/components/ragie/sources/new-document/new-document.mjs b/components/ragie/sources/new-document/new-document.mjs new file mode 100644 index 0000000000000..95995cfbc1df0 --- /dev/null +++ b/components/ragie/sources/new-document/new-document.mjs @@ -0,0 +1,25 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "ragie-new-document", + name: "New Ragie Document Created", + description: "Emit new event whenever a new document is created in Ragie. [See the documentation](https://docs.ragie.ai/reference/listdocuments)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getFunction() { + return this.ragie.listDocuments; + }, + getFieldName() { + return "documents"; + }, + getSummary(document) { + return `New Ragie Document: ${document.name || document.id}`; + }, + }, + sampleEmit, +}; diff --git a/components/ragie/sources/new-document/test-event.mjs b/components/ragie/sources/new-document/test-event.mjs new file mode 100644 index 0000000000000..e86b0ff192662 --- /dev/null +++ b/components/ragie/sources/new-document/test-event.mjs @@ -0,0 +1,13 @@ +export default { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2025-01-16T18:44:13.901Z", + "updated_at": "2025-01-16T18:44:13.901Z", + "status": "string", + "name": "string", + "metadata": { + "additionalProp": "string" + }, + "partition": "string", + "chunk_count": 0, + "external_id": "string" +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48907789d1c59..99e66cc8e1dc3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1554,8 +1554,7 @@ importers: components/calllerapi: {} - components/callminer: - specifiers: {} + components/callminer: {} components/callpage: dependencies: @@ -8407,7 +8406,11 @@ importers: specifier: ^1.1.1 version: 1.6.6 - components/ragie: {} + components/ragie: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 components/railsr: {}