diff --git a/components/agentset/actions/create-ingest-job/create-ingest-job.mjs b/components/agentset/actions/create-ingest-job/create-ingest-job.mjs new file mode 100644 index 0000000000000..b63d4614e3e06 --- /dev/null +++ b/components/agentset/actions/create-ingest-job/create-ingest-job.mjs @@ -0,0 +1,100 @@ +import agentset from "../../agentset.app.mjs"; +import { PAYLOAD_TYPE_OPTIONS } from "../../common/constants.mjs"; +import { parseObject } from "../../common/utils.mjs"; + +export default { + key: "agentset-create-ingest-job", + name: "Create Ingest Job", + description: "Create an ingest job for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/create)", + version: "0.0.1", + type: "action", + props: { + agentset, + namespaceId: { + propDefinition: [ + agentset, + "namespaceId", + ], + }, + payloadType: { + type: "string", + label: "Payload Type", + description: "Type of payload for the ingest job", + options: PAYLOAD_TYPE_OPTIONS, + reloadProps: true, + }, + text: { + type: "string", + label: "Text", + description: "The text to ingest", + hidden: true, + }, + fileUrl: { + type: "string", + label: "File URL", + description: "The URL of the file to ingest", + hidden: true, + }, + urls: { + type: "string[]", + label: "URLs", + description: "The URLs to ingest", + hidden: true, + }, + name: { + type: "string", + label: "Name", + description: "The name of the ingest job", + optional: true, + hidden: true, + }, + }, + async additionalProps(props) { + props.text.hidden = true; + props.name.hidden = true; + props.fileUrl.hidden = true; + props.urls.hidden = true; + + switch (this.payloadType) { + case "TEXT": + props.text.hidden = false; + props.name.hidden = false; + break; + case "FILE": + props.fileUrl.hidden = false; + props.name.hidden = false; + break; + case "URLS": + props.urls.hidden = false; + break; + } + return {}; + }, + async run({ $ }) { + const payload = { + type: this.payloadType, + }; + switch (this.payloadType) { + case "TEXT": + payload.text = this.text; + payload.name = this.name; + break; + case "FILE": + payload.fileUrl = this.fileUrl; + payload.name = this.name; + break; + case "URLS": + payload.urls = parseObject(this.urls); + break; + } + const response = await this.agentset.createIngestJob({ + $, + namespaceId: this.namespaceId, + data: { + payload, + }, + }); + $.export("$summary", `Ingest job created successfully: ID ${response.data.id}`); + return response; + }, +}; diff --git a/components/agentset/actions/create-namespace/create-namespace.mjs b/components/agentset/actions/create-namespace/create-namespace.mjs new file mode 100644 index 0000000000000..62f3d9dcd3fc3 --- /dev/null +++ b/components/agentset/actions/create-namespace/create-namespace.mjs @@ -0,0 +1,30 @@ +import agentset from "../../agentset.app.mjs"; +import { slugify } from "../../common/utils.mjs"; + +export default { + key: "agentset-create-namespace", + name: "Create Namespace", + description: "Creates a namespace for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/namespaces/create)", + version: "0.0.1", + type: "action", + props: { + agentset, + name: { + type: "string", + label: "Name", + description: "The name of the namespace to create", + }, + }, + async run({ $ }) { + const response = await this.agentset.createNamespace({ + $, + data: { + name: this.name, + slug: slugify(this.name), + }, + }); + + $.export("$summary", `Successfully created namespace with ID: ${response.data.id}`); + return response; + }, +}; diff --git a/components/agentset/actions/search-namespace/search-namespace.mjs b/components/agentset/actions/search-namespace/search-namespace.mjs new file mode 100644 index 0000000000000..594debdc24d81 --- /dev/null +++ b/components/agentset/actions/search-namespace/search-namespace.mjs @@ -0,0 +1,89 @@ +import agentset from "../../agentset.app.mjs"; + +export default { + key: "agentset-search-namespace", + name: "Agentset Search Namespace", + description: "Complete retrieval pipeline for RAG with semantic search, filtering, and reranking. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/search)", + version: "0.0.1", + type: "action", + props: { + agentset, + namespaceId: { + propDefinition: [ + agentset, + "namespaceId", + ], + }, + query: { + type: "string", + label: "Query", + description: "The query for semantic search", + }, + topK: { + type: "integer", + label: "Top K", + description: "Number of top documents to return", + min: 1, + max: 100, + optional: true, + }, + rerank: { + type: "boolean", + label: "Rerank", + description: "Whether to rerank the results", + optional: true, + }, + rerankLimit: { + type: "integer", + label: "Rerank Limit", + description: "The number of results to return after reranking", + min: 1, + max: 100, + optional: true, + }, + filter: { + type: "object", + label: "Filter", + description: "Filter to apply to search results", + optional: true, + }, + minScore: { + type: "string", + label: "Minimum Score", + description: "Minimum score to return. Range from 0 to 1", + optional: true, + }, + includeRelationship: { + type: "boolean", + label: "Include Relationship", + description: "Whether to include relationships in the results", + optional: true, + }, + includeMetadata: { + type: "boolean", + label: "Include Metadata", + description: "Whether to include metadata in the results", + optional: true, + }, + }, + + async run({ $ }) { + const response = await this.agentset.searchNamespace({ + $, + namespaceId: this.namespaceId, + data: { + query: this.query, + topK: this.topK, + rerank: this.rerank, + rerankLimit: this.rerankLimit, + filter: this.filter, + minScore: this.minScore && parseFloat(this.minScore), + includeRelationships: this.includeRelationship, + includeMetadata: this.includeMetadata, + }, + }); + + $.export("$summary", `Successfully completed the search for query: "${this.query}"`); + return response; + }, +}; diff --git a/components/agentset/agentset.app.mjs b/components/agentset/agentset.app.mjs index 29ee120e290ea..7a72aaef1f133 100644 --- a/components/agentset/agentset.app.mjs +++ b/components/agentset/agentset.app.mjs @@ -1,11 +1,123 @@ +import { axios } from "@pipedream/platform"; +import { STATUSES_OPTIONS } from "./common/constants.mjs"; + export default { type: "app", app: "agentset", - propDefinitions: {}, + propDefinitions: { + namespaceId: { + type: "string", + label: "Namespace ID", + description: "The ID of the namespace", + async options() { + const { data } = await this.listNamespaces(); + return data.map(({ + id: value, name: label, + }) => ({ + label, + value, + })); + }, + }, + statuses: { + type: "string[]", + label: "Statuses", + description: "Filter by status", + options: STATUSES_OPTIONS, + }, + }, methods: { - // this.$auth contains connected account data - authKeys() { - console.log(Object.keys(this.$auth)); + _baseUrl() { + return "https://api.agentset.ai/v1"; + }, + _headers() { + return { + Authorization: `Bearer ${this.$auth.api_key}`, + }; + }, + _makeRequest({ + $ = this, path, ...opts + }) { + return axios($, { + url: this._baseUrl() + path, + headers: this._headers(), + ...opts, + }); + }, + listNamespaces(opts = {}) { + return this._makeRequest({ + path: "/namespace", + ...opts, + }); + }, + createNamespace(opts = {}) { + return this._makeRequest({ + method: "POST", + path: "/namespace", + ...opts, + }); + }, + createIngestJob({ + namespaceId, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/namespace/${namespaceId}/ingest-jobs`, + ...opts, + }); + }, + listIngestJobs({ + namespaceId, ...opts + }) { + return this._makeRequest({ + path: `/namespace/${namespaceId}/ingest-jobs`, + ...opts, + }); + }, + listDocuments({ + namespaceId, ...opts + }) { + return this._makeRequest({ + path: `/namespace/${namespaceId}/documents`, + ...opts, + }); + }, + searchNamespace({ + namespaceId, ...opts + }) { + return this._makeRequest({ + method: "POST", + path: `/namespace/${namespaceId}/search`, + ...opts, + }); + }, + async *paginate({ + fn, params = {}, maxResults = null, ...opts + }) { + let count = 0; + let cursor; + + do { + params.cursor = cursor; + const { + data, + pagination: { nextCursor }, + } = await fn({ + params, + ...opts, + }); + for (const d of data) { + yield d; + + if (maxResults && ++count === maxResults) { + return count; + } + } + + cursor = nextCursor; + + } while (cursor); }, }, }; + diff --git a/components/agentset/common/constants.mjs b/components/agentset/common/constants.mjs new file mode 100644 index 0000000000000..b2772d07d7cf7 --- /dev/null +++ b/components/agentset/common/constants.mjs @@ -0,0 +1,19 @@ +export const PAYLOAD_TYPE_OPTIONS = [ + "TEXT", + "FILE", + "URLS", +]; + +export const STATUSES_OPTIONS = [ + "BACKLOG", + "QUEUED", + "QUEUED_FOR_RESYNC", + "QUEUED_FOR_DELETE", + "PRE_PROCESSING", + "PROCESSING", + "DELETING", + "CANCELLING", + "COMPLETED", + "FAILED", + "CANCELLED", +]; diff --git a/components/agentset/common/utils.mjs b/components/agentset/common/utils.mjs new file mode 100644 index 0000000000000..f691cd3e147ee --- /dev/null +++ b/components/agentset/common/utils.mjs @@ -0,0 +1,33 @@ +export const slugify = (str) => { + str = str.replace(/^\s+|\s+$/g, ""); + str = str.toLowerCase(); + str = str.replace(/[^a-z0-9 -]/g, "") + .replace(/\s+/g, "-") + .replace(/-+/g, "-"); + return str; +}; + +export const parseObject = (obj) => { + if (!obj) return undefined; + + if (Array.isArray(obj)) { + return obj.map((item) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + return item; + } + } + return item; + }); + } + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + return obj; +}; diff --git a/components/agentset/package.json b/components/agentset/package.json index 720c4d8649e2a..9ddb2a04d0aa3 100644 --- a/components/agentset/package.json +++ b/components/agentset/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/agentset", - "version": "0.0.1", + "version": "0.1.0", "description": "Pipedream Agentset Components", "main": "agentset.app.mjs", "keywords": [ @@ -11,5 +11,8 @@ "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" + }, + "dependencies": { + "@pipedream/platform": "^3.0.3" } -} \ No newline at end of file +} diff --git a/components/agentset/sources/common/base.mjs b/components/agentset/sources/common/base.mjs new file mode 100644 index 0000000000000..c86ac62c97cbb --- /dev/null +++ b/components/agentset/sources/common/base.mjs @@ -0,0 +1,69 @@ +import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import agentset from "../../agentset.app.mjs"; + +export default { + props: { + agentset, + db: "$.service.db", + timer: { + type: "$.interface.timer", + default: { + intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, + }, + }, + namespaceId: { + propDefinition: [ + agentset, + "namespaceId", + ], + }, + }, + methods: { + _getLastData() { + return this.db.get("lastData") || 0; + }, + _setLastData(lastData) { + this.db.set("lastData", lastData); + }, + async emitEvent(maxResults = false) { + const lastData = this._getLastData(); + + const response = this.agentset.paginate({ + fn: this.getFunction(), + namespaceId: this.namespaceId, + params: { + orderBy: "createdAt", + order: "desc", + pageSize: 100, + maxResults, + }, + }); + + let responseArray = []; + for await (const item of response) { + if (Date.parse(item.createdAt) <= lastData) break; + responseArray.push(item); + } + + if (responseArray.length) { + this._setLastData(Date.parse(responseArray[0].createdAt)); + } + + for (const item of responseArray.reverse()) { + this.$emit(item, { + id: `${item.id}`, + summary: this.getSummary(item), + ts: Date.parse(item.createdAt), + }); + } + }, + }, + hooks: { + async deploy() { + await this.emitEvent(25); + }, + }, + async run() { + await this.emitEvent(); + }, +}; diff --git a/components/agentset/sources/new-document-created/new-document-created.mjs b/components/agentset/sources/new-document-created/new-document-created.mjs new file mode 100644 index 0000000000000..88a7ce2ae02a8 --- /dev/null +++ b/components/agentset/sources/new-document-created/new-document-created.mjs @@ -0,0 +1,22 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "agentset-new-document-created", + name: "New Document Created", + description: "Emit new event when a new document is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getFunction() { + return this.agentset.listDocuments; + }, + getSummary(item) { + return `New document created: ${item.name || item.id}`; + }, + }, + sampleEmit, +}; diff --git a/components/agentset/sources/new-document-created/test-event.mjs b/components/agentset/sources/new-document-created/test-event.mjs new file mode 100644 index 0000000000000..9ff5948b31967 --- /dev/null +++ b/components/agentset/sources/new-document-created/test-event.mjs @@ -0,0 +1,24 @@ +export default { + "id": "", + "ingestJobId": "", + "externalId": null, + "name": null, + "tenantId": null, + "status": "BACKLOG", + "error": null, + "source": { + "type": "TEXT", + "text": "" + }, + "properties": null, + "totalChunks": 123, + "totalTokens": 123, + "totalCharacters": 123, + "totalPages": 123, + "createdAt": "", + "queuedAt": null, + "preProcessingAt": null, + "processingAt": null, + "completedAt": null, + "failedAt": null +} \ No newline at end of file diff --git a/components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs b/components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs new file mode 100644 index 0000000000000..9b3bbf43ac287 --- /dev/null +++ b/components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs @@ -0,0 +1,22 @@ +import common from "../common/base.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "agentset-new-ingest-job-created", + name: "New Ingest Job Created", + description: "Emit new event when a ingest job is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getFunction() { + return this.agentset.listIngestJobs; + }, + getSummary(item) { + return `New ingest job created (${item.id})`; + }, + }, + sampleEmit, +}; diff --git a/components/agentset/sources/new-ingest-job-created/test-event.mjs b/components/agentset/sources/new-ingest-job-created/test-event.mjs new file mode 100644 index 0000000000000..4060cf1298f25 --- /dev/null +++ b/components/agentset/sources/new-ingest-job-created/test-event.mjs @@ -0,0 +1,19 @@ +export default { + "id": "", + "namespaceId": "", + "tenantId": null, + "status": "BACKLOG", + "error": null, + "payload": { + "type": "TEXT", + "text": "", + "name": null + }, + "config": null, + "createdAt": "", + "queuedAt": null, + "preProcessingAt": null, + "processingAt": null, + "completedAt": null, + "failedAt": null +} \ No newline at end of file diff --git a/components/facebook_graph_api/facebook_graph_api.app.mjs b/components/facebook_graph_api/facebook_graph_api.app.mjs index e43e6f28f6bc8..a897a39bcf741 100644 --- a/components/facebook_graph_api/facebook_graph_api.app.mjs +++ b/components/facebook_graph_api/facebook_graph_api.app.mjs @@ -8,4 +8,4 @@ export default { console.log(Object.keys(this.$auth)); }, }, -}; \ No newline at end of file +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88c0fcbfb9798..ee925f62e9016 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -425,7 +425,11 @@ importers: components/agentql: {} - components/agentset: {} + components/agentset: + dependencies: + '@pipedream/platform': + specifier: ^3.0.3 + version: 3.0.3 components/agenty: dependencies: @@ -4308,8 +4312,7 @@ importers: components/facebook_conversions: {} - components/facebook_graph_api: - specifiers: {} + components/facebook_graph_api: {} components/facebook_groups: dependencies: @@ -4929,8 +4932,7 @@ importers: components/gatekeeper: {} - components/gather: - specifiers: {} + components/gather: {} components/gatherup: dependencies: