From a148f753be7d4de0386af2338306296a40510a99 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Thu, 16 Jan 2025 16:45:59 -0300 Subject: [PATCH 1/4] ragie init --- .../create-document/create-document.mjs | 67 +++++ .../create-instruction/create-instruction.mjs | 69 +++++ .../update-document-file.mjs | 49 ++++ components/ragie/package.json | 2 +- components/ragie/ragie.app.mjs | 241 +++++++++++++++++- .../sources/new-connection/new-connection.mjs | 114 +++++++++ .../sources/new-document/new-document.mjs | 102 ++++++++ .../new-instruction/new-instruction.mjs | 91 +++++++ 8 files changed, 730 insertions(+), 5 deletions(-) create mode 100644 components/ragie/actions/create-document/create-document.mjs create mode 100644 components/ragie/actions/create-instruction/create-instruction.mjs create mode 100644 components/ragie/actions/update-document-file/update-document-file.mjs create mode 100644 components/ragie/sources/new-connection/new-connection.mjs create mode 100644 components/ragie/sources/new-document/new-document.mjs create mode 100644 components/ragie/sources/new-instruction/new-instruction.mjs diff --git a/components/ragie/actions/create-document/create-document.mjs b/components/ragie/actions/create-document/create-document.mjs new file mode 100644 index 0000000000000..a64ad46c46820 --- /dev/null +++ b/components/ragie/actions/create-document/create-document.mjs @@ -0,0 +1,67 @@ +import ragie from "../../ragie.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "ragie-create-document", + name: "Create Document", + description: "Creates a new document in Ragie. [See the documentation](https://docs.ragie.ai/reference/createdocument)", + version: "0.0.{{ts}}", + type: "action", + props: { + ragie, + file: { + propDefinition: [ + ragie, + "createDocumentFile", + ], + }, + mode: { + propDefinition: [ + ragie, + "createDocumentMode", + ], + optional: true, + }, + metadata: { + propDefinition: [ + ragie, + "createDocumentMetadata", + ], + optional: true, + }, + externalId: { + propDefinition: [ + ragie, + "createDocumentExternalId", + ], + optional: true, + }, + name: { + propDefinition: [ + ragie, + "createDocumentName", + ], + optional: true, + }, + partition: { + propDefinition: [ + ragie, + "createDocumentPartition", + ], + optional: true, + }, + }, + async run({ $ }) { + const response = await this.ragie.createDocument({ + createDocumentFile: this.file, + createDocumentMode: this.mode, + createDocumentMetadata: this.metadata, + createDocumentExternalId: this.externalId, + createDocumentName: this.name, + createDocumentPartition: this.partition, + }); + + $.export("$summary", `Created document: ${response.name} (ID: ${response.id})`); + return response; + }, +}; diff --git a/components/ragie/actions/create-instruction/create-instruction.mjs b/components/ragie/actions/create-instruction/create-instruction.mjs new file mode 100644 index 0000000000000..1b4dc23153d1e --- /dev/null +++ b/components/ragie/actions/create-instruction/create-instruction.mjs @@ -0,0 +1,69 @@ +import ragie from "../../ragie.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "ragie-create-instruction", + name: "Create Instruction", + description: "Creates a new instruction in Ragie. [See the documentation](https://docs.ragie.ai/reference/createinstruction)", + version: "0.0.{{ts}}", + type: "action", + props: { + ragie, + name: { + propDefinition: [ + ragie, + "createInstructionName", + ], + }, + prompt: { + propDefinition: [ + ragie, + "createInstructionPrompt", + ], + }, + active: { + propDefinition: [ + ragie, + "createInstructionActive", + ], + }, + scope: { + propDefinition: [ + ragie, + "createInstructionScope", + ], + }, + filter: { + propDefinition: [ + ragie, + "createInstructionFilter", + ], + }, + partition: { + propDefinition: [ + ragie, + "createInstructionPartition", + ], + }, + }, + async run({ $ }) { + const response = await this.ragie.createInstruction({ + createInstructionName: this.name, + createInstructionPrompt: this.prompt, + ...(this.active !== undefined && { + createInstructionActive: this.active, + }), + ...(this.scope !== undefined && { + createInstructionScope: this.scope, + }), + ...(this.filter !== undefined && { + createInstructionFilter: this.filter, + }), + ...(this.partition !== undefined && { + createInstructionPartition: this.partition, + }), + }); + $.export("$summary", `Created instruction ${this.name}`); + return response; + }, +}; diff --git a/components/ragie/actions/update-document-file/update-document-file.mjs b/components/ragie/actions/update-document-file/update-document-file.mjs new file mode 100644 index 0000000000000..b06b7da52f22a --- /dev/null +++ b/components/ragie/actions/update-document-file/update-document-file.mjs @@ -0,0 +1,49 @@ +import ragie from "../../ragie.app.mjs"; +import { axios } from "@pipedream/platform"; + +export default { + key: "ragie-update-document-file", + name: "Update Document File", + description: "Updates an existing document file in Ragie. [See the documentation]().", + version: "0.0.{{ts}}", + type: "action", + props: { + ragie, + updateDocumentId: { + propDefinition: [ + ragie, + "updateDocumentId", + ], + }, + updateDocumentFile: { + propDefinition: [ + ragie, + "updateDocumentFile", + ], + }, + updateDocumentMode: { + propDefinition: [ + ragie, + "updateDocumentMode", + ], + optional: true, + }, + updateDocumentPartition: { + propDefinition: [ + ragie, + "updateDocumentPartition", + ], + optional: true, + }, + }, + async run({ $ }) { + const response = await this.ragie.updateDocumentFile({ + updateDocumentId: this.updateDocumentId, + updateDocumentFile: this.updateDocumentFile, + updateDocumentMode: this.updateDocumentMode, + updateDocumentPartition: this.updateDocumentPartition, + }); + $.export("$summary", `Successfully updated document file with ID: ${this.updateDocumentId}`); + return response; + }, +}; diff --git a/components/ragie/package.json b/components/ragie/package.json index 6d34e5ccb42c8..19a13f2bf1c82 100644 --- a/components/ragie/package.json +++ b/components/ragie/package.json @@ -12,4 +12,4 @@ "publishConfig": { "access": "public" } -} \ No newline at end of file +} diff --git a/components/ragie/ragie.app.mjs b/components/ragie/ragie.app.mjs index 763d081ee89fe..ad0c8883f30da 100644 --- a/components/ragie/ragie.app.mjs +++ b/components/ragie/ragie.app.mjs @@ -1,11 +1,244 @@ +import { axios } from "@pipedream/platform"; + export default { type: "app", app: "ragie", - propDefinitions: {}, + propDefinitions: { + // Create Document Props + createDocumentFile: { + type: "string", + label: "File", + description: "The binary file to upload, extract, and index for retrieval.", + }, + createDocumentMode: { + type: "string", + label: "Mode", + description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.", + optional: true, + options: [ + { + label: "hi_res", + value: "hi_res", + }, + { + label: "fast", + value: "fast", + }, + ], + }, + createDocumentMetadata: { + type: "object", + label: "Metadata", + description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.", + optional: true, + }, + createDocumentExternalId: { + type: "string", + label: "External ID", + description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.", + optional: true, + }, + createDocumentName: { + type: "string", + label: "Name", + description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.", + optional: true, + }, + createDocumentPartition: { + type: "string", + label: "Partition", + description: "An optional partition identifier. Partitions must be lowercase alphanumeric and may only include the special characters '_' and '-'.", + optional: true, + }, + // Update Document File Props + updateDocumentId: { + type: "string", + label: "Document ID", + description: "The ID of the document to update.", + async options() { + const response = await this.listDocuments(); + return response.documents.map((doc) => ({ + label: doc.name || doc.id, + value: doc.id, + })); + }, + }, + updateDocumentFile: { + type: "string", + label: "File", + description: "The new binary file to upload, extract, and index for retrieval.", + }, + updateDocumentMode: { + type: "string", + label: "Mode", + description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.", + optional: true, + options: [ + { + label: "hi_res", + value: "hi_res", + }, + { + label: "fast", + value: "fast", + }, + ], + }, + updateDocumentPartition: { + type: "string", + label: "Partition", + description: "An optional partition identifier.", + optional: true, + }, + // Create Instruction Props + createInstructionName: { + type: "string", + label: "Name", + description: "The name of the instruction. Must be unique.", + }, + createInstructionPrompt: { + type: "string", + label: "Prompt", + description: "A natural language instruction which will be applied to documents as they are created and updated.", + }, + createInstructionActive: { + type: "boolean", + label: "Active", + description: "Whether the instruction is active. Active instructions are applied to documents when they're created or when their file is updated.", + optional: true, + default: true, + }, + createInstructionScope: { + type: "string", + label: "Scope", + description: "Determines whether the instruction is applied to the entire document or to each chunk of the document.", + optional: true, + options: [ + { + label: "Document", + value: "document", + }, + { + label: "Chunk", + value: "chunk", + }, + ], + }, + createInstructionFilter: { + type: "object", + label: "Filter", + description: "An optional metadata filter that is matched against document metadata during update and creation.", + optional: true, + }, + createInstructionPartition: { + type: "string", + label: "Partition", + description: "An optional partition identifier. Instructions can be scoped to a partition.", + optional: true, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.ragie.ai"; + }, + async _makeRequest(opts = {}) { + const { + $, method = "GET", path = "/", headers = {}, ...otherOpts + } = opts; + return axios($, { + method, + url: this._baseUrl() + path, + headers: { + ...headers, + "Authorization": `Bearer ${this.$auth.api_key}`, + }, + ...otherOpts, + }); + }, + // List Documents + async listDocuments(opts = {}) { + return this._makeRequest({ + method: "GET", + path: "/documents", + params: opts, + }); + }, + // Create Document + async createDocument(opts = {}) { + const { + createDocumentFile, createDocumentMode, createDocumentMetadata, createDocumentExternalId, createDocumentName, createDocumentPartition, + } = opts; + const formData = new FormData(); + formData.append("file", createDocumentFile); + if (createDocumentMode) formData.append("mode", createDocumentMode); + if (createDocumentMetadata) formData.append("metadata", JSON.stringify(createDocumentMetadata)); + if (createDocumentExternalId) formData.append("external_id", createDocumentExternalId); + if (createDocumentName) formData.append("name", createDocumentName); + if (createDocumentPartition) formData.append("partition", createDocumentPartition); + + return this._makeRequest({ + method: "POST", + path: "/documents", + headers: { + "Content-Type": "multipart/form-data", + }, + data: formData, + }); + }, + // Update Document File + async updateDocumentFile(opts = {}) { + const { + updateDocumentId, updateDocumentFile, updateDocumentMode, updateDocumentPartition, + } = opts; + const formData = new FormData(); + formData.append("file", updateDocumentFile); + if (updateDocumentMode) formData.append("mode", updateDocumentMode); + if (updateDocumentPartition) formData.append("partition", updateDocumentPartition); + + return this._makeRequest({ + method: "PUT", + path: `/documents/${updateDocumentId}/file`, + headers: { + "Content-Type": "multipart/form-data", + }, + data: formData, + }); + }, + // Create Instruction + async createInstruction(opts = {}) { + const { + createInstructionName, createInstructionPrompt, createInstructionActive, createInstructionScope, createInstructionFilter, createInstructionPartition, + } = opts; + return this._makeRequest({ + method: "POST", + path: "/instructions", + data: { + name: createInstructionName, + prompt: createInstructionPrompt, + active: createInstructionActive, + scope: createInstructionScope, + filter: createInstructionFilter, + partition: createInstructionPartition, + }, + }); + }, + // List Instructions (for possible future use) + async listInstructions(opts = {}) { + return this._makeRequest({ + method: "GET", + path: "/instructions", + params: opts, + }); + }, + // List Connections + async listConnections(opts = {}) { + return this._makeRequest({ + method: "GET", + path: "/connections", + params: opts, + }); }, + // Additional methods to emit events can be added here if necessary }, + version: "0.0.{{ts}}", }; diff --git a/components/ragie/sources/new-connection/new-connection.mjs b/components/ragie/sources/new-connection/new-connection.mjs new file mode 100644 index 0000000000000..89184097ac2ad --- /dev/null +++ b/components/ragie/sources/new-connection/new-connection.mjs @@ -0,0 +1,114 @@ +import ragie from "../../ragie.app.mjs"; +import { + axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; + +export default { + key: "ragie-new-connection", + name: "New Ragie Connection Created", + description: "Emits a new event whenever a new connection is created in Ragie. [See the documentation](https://docs.ragie.ai)", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + ragie: { + type: "app", + app: "ragie", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + async getConnections(params = {}) { + return await this.ragie.listConnections(params); + }, + getLastCreatedAt() { + return this.db.get("lastCreatedAt") || 0; + }, + setLastCreatedAt(timestamp) { + this.db.set("lastCreatedAt", timestamp); + }, + }, + hooks: { + async deploy() { + const connections = await this.getConnections({ + page_size: 50, + }); + const sortedConnections = connections.connections.sort( + (a, b) => new Date(b.created_at) - new Date(a.created_at), + ); + + for (const connection of sortedConnections) { + this.$emit(connection, { + id: connection.id, + summary: `New Ragie Connection: ${connection.name || connection.id}`, + ts: Date.parse(connection.created_at), + }); + } + + if (sortedConnections.length > 0) { + const latestCreatedAt = sortedConnections[0].created_at; + this.setLastCreatedAt(Date.parse(latestCreatedAt)); + } + }, + async activate() { + // No webhook subscription needed for polling + }, + async deactivate() { + // No cleanup needed for polling + }, + }, + async run() { + let cursor = null; + let hasMore = true; + let latestCreatedAt = this.getLastCreatedAt(); + const newConnections = []; + + while (hasMore) { + const params = { + page_size: 100, + }; + if (cursor) { + params.cursor = cursor; + } + + const response = await this.getConnections(params); + const connections = response.connections; + + for (const connection of connections) { + const connectionCreatedAt = Date.parse(connection.created_at); + if (connectionCreatedAt > latestCreatedAt) { + newConnections.push(connection); + } else { + hasMore = false; + break; + } + } + + cursor = response.pagination?.next_cursor; + if (!cursor) { + hasMore = false; + } + } + + if (newConnections.length > 0) { + newConnections.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); + + for (const connection of newConnections) { + this.$emit(connection, { + id: connection.id, + summary: `New Ragie Connection: ${connection.name || connection.id}`, + ts: Date.parse(connection.created_at), + }); + } + + const mostRecentConnection = newConnections[newConnections.length - 1]; + this.setLastCreatedAt(Date.parse(mostRecentConnection.created_at)); + } + }, +}; diff --git a/components/ragie/sources/new-document/new-document.mjs b/components/ragie/sources/new-document/new-document.mjs new file mode 100644 index 0000000000000..0a2f4e0af18f4 --- /dev/null +++ b/components/ragie/sources/new-document/new-document.mjs @@ -0,0 +1,102 @@ +import { + axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import ragie from "../../ragie.app.mjs"; + +export default { + key: "ragie-new-document", + name: "New Document Created", + description: "Emits a new event whenever a new document is created in Ragie. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + ragie: { + type: "app", + app: "ragie", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getLastCreatedAt() { + return this.db.get("lastCreatedAt") ?? "1970-01-01T00:00:00Z"; + }, + _setLastCreatedAt(ts) { + return this.db.set("lastCreatedAt", ts); + }, + }, + hooks: { + async deploy() { + const pageSize = 50; + let cursor = null; + const response = await this.ragie.listDocuments({ + page_size: pageSize, + cursor, + }); + const documents = response.documents.reverse(); // Emit oldest first + + for (const doc of documents) { + this.$emit(doc, { + id: doc.id, + summary: `New Ragie Document: ${doc.name || doc.id}`, + ts: Date.parse(doc.created_at), + }); + } + + if (documents.length > 0) { + const latestCreatedAt = documents[documents.length - 1].created_at; + await this._setLastCreatedAt(latestCreatedAt); + } + }, + async activate() { + // No action needed on activate for a polling source + }, + async deactivate() { + // No action needed on deactivate for a polling source + }, + }, + async run() { + const lastCreatedAt = await this._getLastCreatedAt(); + let cursor = null; + let hasMore = true; + let newLastCreatedAt = lastCreatedAt; + + while (hasMore) { + const response = await this.ragie.listDocuments({ + page_size: 100, + cursor, + }); + const newDocuments = response.documents + .filter((doc) => new Date(doc.created_at) > new Date(lastCreatedAt)) + .reverse(); // Emit oldest first + + for (const doc of newDocuments) { + this.$emit(doc, { + id: doc.id, + summary: `New Ragie Document: ${doc.name || doc.id}`, + ts: Date.parse(doc.created_at), + }); + + if (doc.created_at > newLastCreatedAt) { + newLastCreatedAt = doc.created_at; + } + } + + if (response.documents.length < 100 || !response.pagination?.next_cursor) { + hasMore = false; + } else { + cursor = response.pagination.next_cursor; + } + } + + if (newLastCreatedAt > lastCreatedAt) { + await this._setLastCreatedAt(newLastCreatedAt); + } + }, +}; diff --git a/components/ragie/sources/new-instruction/new-instruction.mjs b/components/ragie/sources/new-instruction/new-instruction.mjs new file mode 100644 index 0000000000000..10103560d6765 --- /dev/null +++ b/components/ragie/sources/new-instruction/new-instruction.mjs @@ -0,0 +1,91 @@ +import { + axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, +} from "@pipedream/platform"; +import ragie from "../../ragie.app.mjs"; + +export default { + key: "ragie-new-instruction", + name: "New Instruction", + description: "Emit new events whenever an instruction is added to a task in Ragie. [See the documentation]()", + version: "0.0.{{ts}}", + type: "source", + dedupe: "unique", + props: { + ragie: { + type: "app", + app: "ragie", + }, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + hooks: { + async deploy() { + // Fetch the latest instructions, up to 50 + const instructions = await this.ragie.listInstructions({ + perpage: 50, + }); + + // Sort instructions by created_at descending (most recent first) + instructions.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + + // Take the first 50 and reverse to emit oldest first + const latestInstructions = instructions.slice(0, 50).reverse(); + + for (const instruction of latestInstructions) { + this.$emit(instruction, { + id: instruction.id, + summary: `New Instruction: ${instruction.name}`, + ts: new Date(instruction.created_at).getTime(), + }); + } + + // Set the lastCreatedAt to the latest instruction's created_at + if (instructions.length > 0) { + const latestCreatedAt = instructions[0].created_at; + await this.db.set("lastCreatedAt", latestCreatedAt); + } + }, + async activate() { + // No-op + }, + async deactivate() { + // No-op + }, + }, + async run() { + // Get the last created_at timestamp + const lastCreatedAt = await this.db.get("lastCreatedAt") || new Date(0).toISOString(); + + // Fetch all instructions + const instructions = await this.ragie.listInstructions({ + perpage: 50, + }); + + // Filter instructions newer than lastCreatedAt + const newInstructions = instructions.filter( + (instruction) => new Date(instruction.created_at) > new Date(lastCreatedAt), + ); + + // Sort newInstructions by created_at ascending + newInstructions.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); + + for (const instruction of newInstructions) { + this.$emit(instruction, { + id: instruction.id, + summary: `New Instruction: ${instruction.name}`, + ts: new Date(instruction.created_at).getTime(), + }); + } + + if (newInstructions.length > 0) { + // Update lastCreatedAt to the latest instruction's created_at + const latestCreatedAt = newInstructions[newInstructions.length - 1].created_at; + await this.db.set("lastCreatedAt", latestCreatedAt); + } + }, +}; From 5d5d9555bc12ba5b19df04c8b578e466f91b99c2 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Fri, 17 Jan 2025 11:00:19 -0300 Subject: [PATCH 2/4] [Components] ragie #15183 Sources - New Document - New Connection Actions - Create Document - Update Document File --- .../create-document/create-document.mjs | 46 +-- .../create-instruction/create-instruction.mjs | 69 ----- .../update-document-file.mjs | 41 ++- components/ragie/common/constants.mjs | 15 + components/ragie/common/utils.mjs | 31 ++ components/ragie/package.json | 5 +- components/ragie/ragie.app.mjs | 269 +++++------------- components/ragie/sources/common/base.mjs | 60 ++++ .../sources/new-connection/new-connection.mjs | 117 +------- .../sources/new-connection/test-event.mjs | 11 + .../sources/new-document/new-document.mjs | 105 +------ .../ragie/sources/new-document/test-event.mjs | 13 + .../new-instruction/new-instruction.mjs | 91 ------ 13 files changed, 284 insertions(+), 589 deletions(-) delete mode 100644 components/ragie/actions/create-instruction/create-instruction.mjs create mode 100644 components/ragie/common/constants.mjs create mode 100644 components/ragie/common/utils.mjs create mode 100644 components/ragie/sources/common/base.mjs create mode 100644 components/ragie/sources/new-connection/test-event.mjs create mode 100644 components/ragie/sources/new-document/test-event.mjs delete mode 100644 components/ragie/sources/new-instruction/new-instruction.mjs diff --git a/components/ragie/actions/create-document/create-document.mjs b/components/ragie/actions/create-document/create-document.mjs index a64ad46c46820..e7066c2e5cf6b 100644 --- a/components/ragie/actions/create-document/create-document.mjs +++ b/components/ragie/actions/create-document/create-document.mjs @@ -1,11 +1,13 @@ +import FormData from "form-data"; +import fs from "fs"; +import { checkTmp } from "../../common/utils.mjs"; import ragie from "../../ragie.app.mjs"; -import { axios } from "@pipedream/platform"; export default { key: "ragie-create-document", name: "Create Document", description: "Creates a new document in Ragie. [See the documentation](https://docs.ragie.ai/reference/createdocument)", - version: "0.0.{{ts}}", + version: "0.0.1", type: "action", props: { ragie, @@ -23,42 +25,44 @@ export default { optional: true, }, metadata: { - propDefinition: [ - ragie, - "createDocumentMetadata", - ], + type: "object", + label: "Metadata", + description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.", optional: true, }, externalId: { - propDefinition: [ - ragie, - "createDocumentExternalId", - ], + type: "string", + label: "External ID", + description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.", optional: true, }, name: { - propDefinition: [ - ragie, - "createDocumentName", - ], + type: "string", + label: "Name", + description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.", optional: true, }, partition: { propDefinition: [ ragie, - "createDocumentPartition", + "partition", ], optional: true, }, }, async run({ $ }) { + const data = new FormData(); + data.append("file", fs.createReadStream(checkTmp(this.file))); + if (this.mode) data.append("mode", this.mode); + if (this.metadata) data.append("metadata", JSON.stringify(this.metadata)); + if (this.externalId) data.append("external_id", this.externalId); + if (this.name) data.append("name", this.name); + if (this.partition) data.append("partition", this.partition); + const response = await this.ragie.createDocument({ - createDocumentFile: this.file, - createDocumentMode: this.mode, - createDocumentMetadata: this.metadata, - createDocumentExternalId: this.externalId, - createDocumentName: this.name, - createDocumentPartition: this.partition, + $, + data, + headers: data.getHeaders(), }); $.export("$summary", `Created document: ${response.name} (ID: ${response.id})`); diff --git a/components/ragie/actions/create-instruction/create-instruction.mjs b/components/ragie/actions/create-instruction/create-instruction.mjs deleted file mode 100644 index 1b4dc23153d1e..0000000000000 --- a/components/ragie/actions/create-instruction/create-instruction.mjs +++ /dev/null @@ -1,69 +0,0 @@ -import ragie from "../../ragie.app.mjs"; -import { axios } from "@pipedream/platform"; - -export default { - key: "ragie-create-instruction", - name: "Create Instruction", - description: "Creates a new instruction in Ragie. [See the documentation](https://docs.ragie.ai/reference/createinstruction)", - version: "0.0.{{ts}}", - type: "action", - props: { - ragie, - name: { - propDefinition: [ - ragie, - "createInstructionName", - ], - }, - prompt: { - propDefinition: [ - ragie, - "createInstructionPrompt", - ], - }, - active: { - propDefinition: [ - ragie, - "createInstructionActive", - ], - }, - scope: { - propDefinition: [ - ragie, - "createInstructionScope", - ], - }, - filter: { - propDefinition: [ - ragie, - "createInstructionFilter", - ], - }, - partition: { - propDefinition: [ - ragie, - "createInstructionPartition", - ], - }, - }, - async run({ $ }) { - const response = await this.ragie.createInstruction({ - createInstructionName: this.name, - createInstructionPrompt: this.prompt, - ...(this.active !== undefined && { - createInstructionActive: this.active, - }), - ...(this.scope !== undefined && { - createInstructionScope: this.scope, - }), - ...(this.filter !== undefined && { - createInstructionFilter: this.filter, - }), - ...(this.partition !== undefined && { - createInstructionPartition: this.partition, - }), - }); - $.export("$summary", `Created instruction ${this.name}`); - return response; - }, -}; diff --git a/components/ragie/actions/update-document-file/update-document-file.mjs b/components/ragie/actions/update-document-file/update-document-file.mjs index b06b7da52f22a..b650c150acd60 100644 --- a/components/ragie/actions/update-document-file/update-document-file.mjs +++ b/components/ragie/actions/update-document-file/update-document-file.mjs @@ -1,49 +1,48 @@ +import FormData from "form-data"; +import fs from "fs"; +import { checkTmp } from "../../common/utils.mjs"; import ragie from "../../ragie.app.mjs"; -import { axios } from "@pipedream/platform"; export default { key: "ragie-update-document-file", name: "Update Document File", - description: "Updates an existing document file in Ragie. [See the documentation]().", - version: "0.0.{{ts}}", + description: "Updates an existing document file in Ragie. [See the documentation](https://docs.ragie.ai/reference/updatedocumentfile).", + version: "0.0.1", type: "action", props: { ragie, - updateDocumentId: { + documentId: { propDefinition: [ ragie, - "updateDocumentId", + "documentId", ], }, - updateDocumentFile: { + mode: { propDefinition: [ ragie, - "updateDocumentFile", - ], - }, - updateDocumentMode: { - propDefinition: [ - ragie, - "updateDocumentMode", + "createDocumentMode", ], optional: true, }, - updateDocumentPartition: { + file: { propDefinition: [ ragie, - "updateDocumentPartition", + "createDocumentFile", ], - optional: true, }, }, async run({ $ }) { + const data = new FormData(); + data.append("file", fs.createReadStream(checkTmp(this.file))); + if (this.mode) data.append("mode", this.mode); + const response = await this.ragie.updateDocumentFile({ - updateDocumentId: this.updateDocumentId, - updateDocumentFile: this.updateDocumentFile, - updateDocumentMode: this.updateDocumentMode, - updateDocumentPartition: this.updateDocumentPartition, + $, + documentId: this.documentId, + data, + headers: data.getHeaders(), }); - $.export("$summary", `Successfully updated document file with ID: ${this.updateDocumentId}`); + $.export("$summary", `Successfully updated document file with ID: ${this.documentId}`); return response; }, }; diff --git a/components/ragie/common/constants.mjs b/components/ragie/common/constants.mjs new file mode 100644 index 0000000000000..d681c66f145c1 --- /dev/null +++ b/components/ragie/common/constants.mjs @@ -0,0 +1,15 @@ +export const SCOPE_OPTIONS = [ + { + label: "Document", + value: "document", + }, + { + label: "Chunk", + value: "chunk", + }, +]; + +export const DOCUMENT_MODE_OPTIONS = [ + "hi_res", + "fast", +]; diff --git a/components/ragie/common/utils.mjs b/components/ragie/common/utils.mjs new file mode 100644 index 0000000000000..0cd1a12b6a4ba --- /dev/null +++ b/components/ragie/common/utils.mjs @@ -0,0 +1,31 @@ +export const checkTmp = (filename) => { + if (!filename.startsWith("/tmp")) { + return `/tmp/${filename}`; + } + return filename; +}; + +export const parseObject = (obj) => { + if (!obj) return undefined; + + if (Array.isArray(obj)) { + return obj.map((item) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + return item; + } + } + return item; + }); + } + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + return obj; +}; diff --git a/components/ragie/package.json b/components/ragie/package.json index 19a13f2bf1c82..b331423650937 100644 --- a/components/ragie/package.json +++ b/components/ragie/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/ragie", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Ragie Components", "main": "ragie.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } } diff --git a/components/ragie/ragie.app.mjs b/components/ragie/ragie.app.mjs index ad0c8883f30da..ea3c95b014d66 100644 --- a/components/ragie/ragie.app.mjs +++ b/components/ragie/ragie.app.mjs @@ -1,244 +1,127 @@ import { axios } from "@pipedream/platform"; +import { DOCUMENT_MODE_OPTIONS } from "./common/constants.mjs"; export default { type: "app", app: "ragie", propDefinitions: { - // Create Document Props - createDocumentFile: { + documentFile: { type: "string", label: "File", - description: "The binary file to upload, extract, and index for retrieval.", + description: "The path to the file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory).", }, - createDocumentMode: { + documentMode: { type: "string", label: "Mode", description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.", optional: true, - options: [ - { - label: "hi_res", - value: "hi_res", - }, - { - label: "fast", - value: "fast", - }, - ], + options: DOCUMENT_MODE_OPTIONS, }, - createDocumentMetadata: { - type: "object", - label: "Metadata", - description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.", - optional: true, - }, - createDocumentExternalId: { - type: "string", - label: "External ID", - description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.", - optional: true, - }, - createDocumentName: { - type: "string", - label: "Name", - description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.", - optional: true, - }, - createDocumentPartition: { + partition: { type: "string", label: "Partition", description: "An optional partition identifier. Partitions must be lowercase alphanumeric and may only include the special characters '_' and '-'.", - optional: true, }, - // Update Document File Props - updateDocumentId: { + documentId: { type: "string", label: "Document ID", description: "The ID of the document to update.", - async options() { - const response = await this.listDocuments(); - return response.documents.map((doc) => ({ - label: doc.name || doc.id, - value: doc.id, - })); + async options({ prevContext }) { + const { + documents, pagination, + } = await this.listDocuments({ + params: { + cursor: prevContext.cursor, + }, + }); + return { + options: documents.map(({ + id: value, name: label, + }) => ({ + label, + value, + })), + context: { + cursor: pagination.next_cursor, + }, + }; }, }, - updateDocumentFile: { - type: "string", - label: "File", - description: "The new binary file to upload, extract, and index for retrieval.", - }, - updateDocumentMode: { - type: "string", - label: "Mode", - description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.", - optional: true, - options: [ - { - label: "hi_res", - value: "hi_res", - }, - { - label: "fast", - value: "fast", - }, - ], - }, - updateDocumentPartition: { - type: "string", - label: "Partition", - description: "An optional partition identifier.", - optional: true, - }, - // Create Instruction Props - createInstructionName: { - type: "string", - label: "Name", - description: "The name of the instruction. Must be unique.", - }, - createInstructionPrompt: { - type: "string", - label: "Prompt", - description: "A natural language instruction which will be applied to documents as they are created and updated.", - }, - createInstructionActive: { - type: "boolean", - label: "Active", - description: "Whether the instruction is active. Active instructions are applied to documents when they're created or when their file is updated.", - optional: true, - default: true, - }, - createInstructionScope: { - type: "string", - label: "Scope", - description: "Determines whether the instruction is applied to the entire document or to each chunk of the document.", - optional: true, - options: [ - { - label: "Document", - value: "document", - }, - { - label: "Chunk", - value: "chunk", - }, - ], - }, - createInstructionFilter: { - type: "object", - label: "Filter", - description: "An optional metadata filter that is matched against document metadata during update and creation.", - optional: true, - }, - createInstructionPartition: { - type: "string", - label: "Partition", - description: "An optional partition identifier. Instructions can be scoped to a partition.", - optional: true, - }, }, methods: { _baseUrl() { return "https://api.ragie.ai"; }, - async _makeRequest(opts = {}) { - const { - $, method = "GET", path = "/", headers = {}, ...otherOpts - } = opts; + _headers(headers = {}) { + return { + ...headers, + "Authorization": `Bearer ${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, headers, ...opts + }) { return axios($, { - method, url: this._baseUrl() + path, - headers: { - ...headers, - "Authorization": `Bearer ${this.$auth.api_key}`, - }, - ...otherOpts, + headers: this._headers(headers), + ...opts, }); }, - // List Documents - async listDocuments(opts = {}) { + createDocument(opts = {}) { return this._makeRequest({ - method: "GET", + method: "POST", path: "/documents", - params: opts, + ...opts, }); }, - // Create Document - async createDocument(opts = {}) { - const { - createDocumentFile, createDocumentMode, createDocumentMetadata, createDocumentExternalId, createDocumentName, createDocumentPartition, - } = opts; - const formData = new FormData(); - formData.append("file", createDocumentFile); - if (createDocumentMode) formData.append("mode", createDocumentMode); - if (createDocumentMetadata) formData.append("metadata", JSON.stringify(createDocumentMetadata)); - if (createDocumentExternalId) formData.append("external_id", createDocumentExternalId); - if (createDocumentName) formData.append("name", createDocumentName); - if (createDocumentPartition) formData.append("partition", createDocumentPartition); - + listDocuments(opts = {}) { return this._makeRequest({ - method: "POST", path: "/documents", - headers: { - "Content-Type": "multipart/form-data", - }, - data: formData, + params: opts, }); }, - // Update Document File - async updateDocumentFile(opts = {}) { - const { - updateDocumentId, updateDocumentFile, updateDocumentMode, updateDocumentPartition, - } = opts; - const formData = new FormData(); - formData.append("file", updateDocumentFile); - if (updateDocumentMode) formData.append("mode", updateDocumentMode); - if (updateDocumentPartition) formData.append("partition", updateDocumentPartition); - + updateDocumentFile({ + documentId, ...opts + }) { return this._makeRequest({ method: "PUT", - path: `/documents/${updateDocumentId}/file`, - headers: { - "Content-Type": "multipart/form-data", - }, - data: formData, - }); - }, - // Create Instruction - async createInstruction(opts = {}) { - const { - createInstructionName, createInstructionPrompt, createInstructionActive, createInstructionScope, createInstructionFilter, createInstructionPartition, - } = opts; - return this._makeRequest({ - method: "POST", - path: "/instructions", - data: { - name: createInstructionName, - prompt: createInstructionPrompt, - active: createInstructionActive, - scope: createInstructionScope, - filter: createInstructionFilter, - partition: createInstructionPartition, - }, - }); - }, - // List Instructions (for possible future use) - async listInstructions(opts = {}) { - return this._makeRequest({ - method: "GET", - path: "/instructions", - params: opts, + path: `/documents/${documentId}/file`, + ...opts, }); }, - // List Connections - async listConnections(opts = {}) { + listConnections(opts = {}) { return this._makeRequest({ - method: "GET", path: "/connections", params: opts, }); }, - // Additional methods to emit events can be added here if necessary + async *paginate({ + fn, params = {}, fieldName, maxResults = null, ...opts + }) { + let count = 0; + let nextCursor; + + do { + params.cursor = nextCursor; + const { + pagination, + ...data + } = await fn({ + params, + ...opts, + }); + const items = data[fieldName]; + + for (const d of items) { + yield d; + + if (maxResults && ++count === maxResults) { + return count; + } + } + + nextCursor = pagination?.next_cursor; + } while (nextCursor); + }, }, - version: "0.0.{{ts}}", }; diff --git a/components/ragie/sources/common/base.mjs b/components/ragie/sources/common/base.mjs new file mode 100644 index 0000000000000..d09ddc09671a2 --- /dev/null +++ b/components/ragie/sources/common/base.mjs @@ -0,0 +1,60 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import ragie from "../../ragie.app.mjs"; + +export default { + props: { + ragie, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + }, + methods: { + _getLastDate() { + return this.db.get("lastDate") || 0; + }, + _setLastDate(lastDate) { + this.db.set("lastDate", lastDate); + }, + async emitEvent(maxResults = false) { + const lastDate = this._getLastDate(); + + const response = this.ragie.paginate({ + fn: this.getFunction(), + fieldName: this.getFieldName(), + }); + + let responseArray = []; + for await (const item of response) { + if (Date.parse(item.created_at) <= lastDate) break; + responseArray.push(item); + } + + if (responseArray.length) { + if (maxResults && (responseArray.length > maxResults)) { + responseArray.length = maxResults; + } + this._setLastDate(Date.parse(responseArray[0].created_at)); + } + + for (const item of responseArray.reverse()) { + this.$emit(item, { + id: item.id, + summary: this.getSummary(item), + ts: Date.parse(item.created_at || new Date()), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, +}; diff --git a/components/ragie/sources/new-connection/new-connection.mjs b/components/ragie/sources/new-connection/new-connection.mjs index 89184097ac2ad..43a70dc255e20 100644 --- a/components/ragie/sources/new-connection/new-connection.mjs +++ b/components/ragie/sources/new-connection/new-connection.mjs @@ -1,114 +1,27 @@ -import ragie from "../../ragie.app.mjs"; -import { - axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, -} from "@pipedream/platform"; +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; export default { + ...common, key: "ragie-new-connection", name: "New Ragie Connection Created", - description: "Emits a new event whenever a new connection is created in Ragie. [See the documentation](https://docs.ragie.ai)", - version: "0.0.{{ts}}", + description: "Emit new event whenever a new connection is created in Ragie. [See the documentation](https://docs.ragie.ai/reference/list_connections_connections_get)", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - ragie: { - type: "app", - app: "ragie", - }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - }, methods: { - async getConnections(params = {}) { - return await this.ragie.listConnections(params); - }, - getLastCreatedAt() { - return this.db.get("lastCreatedAt") || 0; - }, - setLastCreatedAt(timestamp) { - this.db.set("lastCreatedAt", timestamp); - }, - }, - hooks: { - async deploy() { - const connections = await this.getConnections({ - page_size: 50, - }); - const sortedConnections = connections.connections.sort( - (a, b) => new Date(b.created_at) - new Date(a.created_at), - ); - - for (const connection of sortedConnections) { - this.$emit(connection, { - id: connection.id, - summary: `New Ragie Connection: ${connection.name || connection.id}`, - ts: Date.parse(connection.created_at), - }); - } - - if (sortedConnections.length > 0) { - const latestCreatedAt = sortedConnections[0].created_at; - this.setLastCreatedAt(Date.parse(latestCreatedAt)); - } + ...common.methods, + getFunction() { + return this.ragie.listConnections; }, - async activate() { - // No webhook subscription needed for polling + getFieldName() { + return "connections"; }, - async deactivate() { - // No cleanup needed for polling + getSummary({ + name, type, + }) { + return `New Ragie Connection: ${name} (${type})`; }, }, - async run() { - let cursor = null; - let hasMore = true; - let latestCreatedAt = this.getLastCreatedAt(); - const newConnections = []; - - while (hasMore) { - const params = { - page_size: 100, - }; - if (cursor) { - params.cursor = cursor; - } - - const response = await this.getConnections(params); - const connections = response.connections; - - for (const connection of connections) { - const connectionCreatedAt = Date.parse(connection.created_at); - if (connectionCreatedAt > latestCreatedAt) { - newConnections.push(connection); - } else { - hasMore = false; - break; - } - } - - cursor = response.pagination?.next_cursor; - if (!cursor) { - hasMore = false; - } - } - - if (newConnections.length > 0) { - newConnections.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); - - for (const connection of newConnections) { - this.$emit(connection, { - id: connection.id, - summary: `New Ragie Connection: ${connection.name || connection.id}`, - ts: Date.parse(connection.created_at), - }); - } - - const mostRecentConnection = newConnections[newConnections.length - 1]; - this.setLastCreatedAt(Date.parse(mostRecentConnection.created_at)); - } - }, + sampleEmit, }; diff --git a/components/ragie/sources/new-connection/test-event.mjs b/components/ragie/sources/new-connection/test-event.mjs new file mode 100644 index 0000000000000..27c452a389fe9 --- /dev/null +++ b/components/ragie/sources/new-connection/test-event.mjs @@ -0,0 +1,11 @@ +export default { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2025-01-16T18:44:13.901Z", + "updated_at": "2025-01-16T18:44:13.901Z", + "metadata": {}, + "type": "string", + "name": "string", + "enabled": true, + "last_synced_at": "2025-01-16T18:44:13.901Z", + "syncing": true +} \ No newline at end of file diff --git a/components/ragie/sources/new-document/new-document.mjs b/components/ragie/sources/new-document/new-document.mjs index 0a2f4e0af18f4..95995cfbc1df0 100644 --- a/components/ragie/sources/new-document/new-document.mjs +++ b/components/ragie/sources/new-document/new-document.mjs @@ -1,102 +1,25 @@ -import { - axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, -} from "@pipedream/platform"; -import ragie from "../../ragie.app.mjs"; +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; export default { + ...common, key: "ragie-new-document", - name: "New Document Created", - description: "Emits a new event whenever a new document is created in Ragie. [See the documentation]()", - version: "0.0.{{ts}}", + name: "New Ragie Document Created", + description: "Emit new event whenever a new document is created in Ragie. [See the documentation](https://docs.ragie.ai/reference/listdocuments)", + version: "0.0.1", type: "source", dedupe: "unique", - props: { - ragie: { - type: "app", - app: "ragie", - }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - }, methods: { - _getLastCreatedAt() { - return this.db.get("lastCreatedAt") ?? "1970-01-01T00:00:00Z"; - }, - _setLastCreatedAt(ts) { - return this.db.set("lastCreatedAt", ts); - }, - }, - hooks: { - async deploy() { - const pageSize = 50; - let cursor = null; - const response = await this.ragie.listDocuments({ - page_size: pageSize, - cursor, - }); - const documents = response.documents.reverse(); // Emit oldest first - - for (const doc of documents) { - this.$emit(doc, { - id: doc.id, - summary: `New Ragie Document: ${doc.name || doc.id}`, - ts: Date.parse(doc.created_at), - }); - } - - if (documents.length > 0) { - const latestCreatedAt = documents[documents.length - 1].created_at; - await this._setLastCreatedAt(latestCreatedAt); - } + ...common.methods, + getFunction() { + return this.ragie.listDocuments; }, - async activate() { - // No action needed on activate for a polling source + getFieldName() { + return "documents"; }, - async deactivate() { - // No action needed on deactivate for a polling source + getSummary(document) { + return `New Ragie Document: ${document.name || document.id}`; }, }, - async run() { - const lastCreatedAt = await this._getLastCreatedAt(); - let cursor = null; - let hasMore = true; - let newLastCreatedAt = lastCreatedAt; - - while (hasMore) { - const response = await this.ragie.listDocuments({ - page_size: 100, - cursor, - }); - const newDocuments = response.documents - .filter((doc) => new Date(doc.created_at) > new Date(lastCreatedAt)) - .reverse(); // Emit oldest first - - for (const doc of newDocuments) { - this.$emit(doc, { - id: doc.id, - summary: `New Ragie Document: ${doc.name || doc.id}`, - ts: Date.parse(doc.created_at), - }); - - if (doc.created_at > newLastCreatedAt) { - newLastCreatedAt = doc.created_at; - } - } - - if (response.documents.length < 100 || !response.pagination?.next_cursor) { - hasMore = false; - } else { - cursor = response.pagination.next_cursor; - } - } - - if (newLastCreatedAt > lastCreatedAt) { - await this._setLastCreatedAt(newLastCreatedAt); - } - }, + sampleEmit, }; diff --git a/components/ragie/sources/new-document/test-event.mjs b/components/ragie/sources/new-document/test-event.mjs new file mode 100644 index 0000000000000..e86b0ff192662 --- /dev/null +++ b/components/ragie/sources/new-document/test-event.mjs @@ -0,0 +1,13 @@ +export default { + "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "created_at": "2025-01-16T18:44:13.901Z", + "updated_at": "2025-01-16T18:44:13.901Z", + "status": "string", + "name": "string", + "metadata": { + "additionalProp": "string" + }, + "partition": "string", + "chunk_count": 0, + "external_id": "string" +} \ No newline at end of file diff --git a/components/ragie/sources/new-instruction/new-instruction.mjs b/components/ragie/sources/new-instruction/new-instruction.mjs deleted file mode 100644 index 10103560d6765..0000000000000 --- a/components/ragie/sources/new-instruction/new-instruction.mjs +++ /dev/null @@ -1,91 +0,0 @@ -import { - axios, DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, -} from "@pipedream/platform"; -import ragie from "../../ragie.app.mjs"; - -export default { - key: "ragie-new-instruction", - name: "New Instruction", - description: "Emit new events whenever an instruction is added to a task in Ragie. [See the documentation]()", - version: "0.0.{{ts}}", - type: "source", - dedupe: "unique", - props: { - ragie: { - type: "app", - app: "ragie", - }, - db: "$.service.db", - timer: { - type: "$.interface.timer", - default: { - intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, - }, - }, - }, - hooks: { - async deploy() { - // Fetch the latest instructions, up to 50 - const instructions = await this.ragie.listInstructions({ - perpage: 50, - }); - - // Sort instructions by created_at descending (most recent first) - instructions.sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); - - // Take the first 50 and reverse to emit oldest first - const latestInstructions = instructions.slice(0, 50).reverse(); - - for (const instruction of latestInstructions) { - this.$emit(instruction, { - id: instruction.id, - summary: `New Instruction: ${instruction.name}`, - ts: new Date(instruction.created_at).getTime(), - }); - } - - // Set the lastCreatedAt to the latest instruction's created_at - if (instructions.length > 0) { - const latestCreatedAt = instructions[0].created_at; - await this.db.set("lastCreatedAt", latestCreatedAt); - } - }, - async activate() { - // No-op - }, - async deactivate() { - // No-op - }, - }, - async run() { - // Get the last created_at timestamp - const lastCreatedAt = await this.db.get("lastCreatedAt") || new Date(0).toISOString(); - - // Fetch all instructions - const instructions = await this.ragie.listInstructions({ - perpage: 50, - }); - - // Filter instructions newer than lastCreatedAt - const newInstructions = instructions.filter( - (instruction) => new Date(instruction.created_at) > new Date(lastCreatedAt), - ); - - // Sort newInstructions by created_at ascending - newInstructions.sort((a, b) => new Date(a.created_at) - new Date(b.created_at)); - - for (const instruction of newInstructions) { - this.$emit(instruction, { - id: instruction.id, - summary: `New Instruction: ${instruction.name}`, - ts: new Date(instruction.created_at).getTime(), - }); - } - - if (newInstructions.length > 0) { - // Update lastCreatedAt to the latest instruction's created_at - const latestCreatedAt = newInstructions[newInstructions.length - 1].created_at; - await this.db.set("lastCreatedAt", latestCreatedAt); - } - }, -}; From e0f71976e5a0c4feea102bced976f97095b5256f Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Fri, 17 Jan 2025 11:01:44 -0300 Subject: [PATCH 3/4] pnpm update --- pnpm-lock.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48907789d1c59..99e66cc8e1dc3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1554,8 +1554,7 @@ importers: components/calllerapi: {} - components/callminer: - specifiers: {} + components/callminer: {} components/callpage: dependencies: @@ -8407,7 +8406,11 @@ importers: specifier: ^1.1.1 version: 1.6.6 - components/ragie: {} + components/ragie: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 components/railsr: {} From 20d01ced091a19949f981401b84c6cbe803997a7 Mon Sep 17 00:00:00 2001 From: Luan Cazarine Date: Mon, 20 Jan 2025 17:47:37 -0300 Subject: [PATCH 4/4] fix props reference names --- components/ragie/actions/create-document/create-document.mjs | 4 ++-- .../actions/update-document-file/update-document-file.mjs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/ragie/actions/create-document/create-document.mjs b/components/ragie/actions/create-document/create-document.mjs index e7066c2e5cf6b..70fada4cf4a38 100644 --- a/components/ragie/actions/create-document/create-document.mjs +++ b/components/ragie/actions/create-document/create-document.mjs @@ -14,13 +14,13 @@ export default { file: { propDefinition: [ ragie, - "createDocumentFile", + "documentFile", ], }, mode: { propDefinition: [ ragie, - "createDocumentMode", + "documentMode", ], optional: true, }, diff --git a/components/ragie/actions/update-document-file/update-document-file.mjs b/components/ragie/actions/update-document-file/update-document-file.mjs index b650c150acd60..e62730e0a6034 100644 --- a/components/ragie/actions/update-document-file/update-document-file.mjs +++ b/components/ragie/actions/update-document-file/update-document-file.mjs @@ -20,14 +20,14 @@ export default { mode: { propDefinition: [ ragie, - "createDocumentMode", + "documentMode", ], optional: true, }, file: { propDefinition: [ ragie, - "createDocumentFile", + "documentFile", ], }, },