From 952648aa6d900b71c9859b9f2543b368625cd9ab Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Wed, 9 Apr 2025 15:40:28 -0300
Subject: [PATCH 1/8] agentset init

---
 .../create-injest-job/create-injest-job.mjs   |  36 +++
 .../create-namespace/create-namespace.mjs     |  88 ++++++++
 .../search-namespace/search-namespace.mjs     |  80 +++++++
 components/agentset/agentset.app.mjs          | 210 +++++++++++++++++-
 components/agentset/package.json              |   2 +-
 .../sources/new-document/new-document.mjs     |  89 ++++++++
 .../new-ingest-jopb/new-ingest-jopb.mjs       |  68 ++++++
 7 files changed, 567 insertions(+), 6 deletions(-)
 create mode 100644 components/agentset/actions/create-injest-job/create-injest-job.mjs
 create mode 100644 components/agentset/actions/create-namespace/create-namespace.mjs
 create mode 100644 components/agentset/actions/search-namespace/search-namespace.mjs
 create mode 100644 components/agentset/sources/new-document/new-document.mjs
 create mode 100644 components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs

diff --git a/components/agentset/actions/create-injest-job/create-injest-job.mjs b/components/agentset/actions/create-injest-job/create-injest-job.mjs
new file mode 100644
index 0000000000000..4e005b057597c
--- /dev/null
+++ b/components/agentset/actions/create-injest-job/create-injest-job.mjs
@@ -0,0 +1,36 @@
+import agentset from "../../agentset.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "agentset-create-ingest-job",
+  name: "Create Ingest Job",
+  description: "Create an ingest job for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/create)",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    agentset,
+    namespaceId: {
+      propDefinition: [
+        agentset,
+        "namespaceId",
+      ],
+    },
+    payloadType: {
+      propDefinition: [
+        agentset,
+        "payloadType",
+      ],
+    },
+    payload: {
+      propDefinition: [
+        agentset,
+        "payload",
+      ],
+    },
+  },
+  async run({ $ }) {
+    const response = await this.agentset.createIngestJob(this.namespaceId, this.payloadType, this.payload);
+    $.export("$summary", `Ingest job created successfully: ID ${response.id}`);
+    return response;
+  },
+};
diff --git a/components/agentset/actions/create-namespace/create-namespace.mjs b/components/agentset/actions/create-namespace/create-namespace.mjs
new file mode 100644
index 0000000000000..9eac9c78d4055
--- /dev/null
+++ b/components/agentset/actions/create-namespace/create-namespace.mjs
@@ -0,0 +1,88 @@
+import agentset from "../../agentset.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "agentset-create-namespace",
+  name: "Create Namespace",
+  description: "Creates a namespace for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/namespaces/create)",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    agentset,
+    name: {
+      type: "string",
+      label: "Name",
+      description: "The name of the namespace to create",
+    },
+    slug: {
+      type: "string",
+      label: "Slug",
+      description: "A unique slug for the namespace",
+    },
+    embeddingConfigProvider: {
+      type: "string",
+      label: "Embedding Config Provider",
+      description: "Provider for the embedding config",
+      options: [
+        "OPENAI",
+      ],
+    },
+    embeddingConfigModel: {
+      type: "string",
+      label: "Embedding Config Model",
+      description: "The model for the embedding config",
+      options: [
+        "text-embedding-3-small",
+        "text-embedding-3-large",
+      ],
+    },
+    embeddingConfigApiKey: {
+      type: "string",
+      label: "Embedding Config API Key",
+      description: "API key for the embedding config provider",
+      secret: true,
+    },
+    vectorStoreConfigProvider: {
+      type: "string",
+      label: "Vector Store Config Provider",
+      description: "Provider for the vector store config",
+      options: [
+        "PINECONE",
+      ],
+    },
+    vectorStoreConfigApiKey: {
+      type: "string",
+      label: "Vector Store Config API Key",
+      description: "API key for the vector store",
+      secret: true,
+    },
+    vectorStoreConfigIndexHost: {
+      type: "string",
+      label: "Vector Store Config Index Host",
+      description: "URL of the Pinecone index host",
+    },
+  },
+  async run({ $ }) {
+    const response = await this.agentset._makeRequest({
+      method: "POST",
+      path: "/namespace",
+      data: {
+        name: this.name,
+        slug: this.slug,
+        embeddingConfig: {
+          provider: this.embeddingConfigProvider,
+          model: this.embeddingConfigModel,
+          apiKey: this.embeddingConfigApiKey,
+        },
+        vectorStoreConfig: {
+          provider: this.vectorStoreConfigProvider,
+          apiKey: this.vectorStoreConfigApiKey,
+          indexHost: this.vectorStoreConfigIndexHost,
+        },
+      },
+    });
+
+    $.export("$summary", `Successfully created namespace ${response.name}`);
+    return response;
+  },
+};
diff --git a/components/agentset/actions/search-namespace/search-namespace.mjs b/components/agentset/actions/search-namespace/search-namespace.mjs
new file mode 100644
index 0000000000000..4b974df3598e6
--- /dev/null
+++ b/components/agentset/actions/search-namespace/search-namespace.mjs
@@ -0,0 +1,80 @@
+import agentset from "../../agentset.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "agentset-search-namespace",
+  name: "Agentset Search Namespace",
+  description: "Complete retrieval pipeline for RAG with semantic search, filtering, and reranking. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/search)",
+  version: "0.0.{{ts}}",
+  type: "action",
+  props: {
+    agentset,
+    namespaceId: {
+      propDefinition: [
+        agentset,
+        "namespaceId",
+      ],
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "The query for semantic search",
+    },
+    topK: {
+      type: "integer",
+      label: "Top K",
+      description: "Number of top documents to return",
+      optional: true,
+    },
+    rerank: {
+      type: "boolean",
+      label: "Rerank",
+      description: "Rerank documents based on query",
+      optional: true,
+    },
+    rerankLimit: {
+      type: "integer",
+      label: "Rerank Limit",
+      description: "Limit for reranking documents",
+      optional: true,
+    },
+    filter: {
+      type: "string",
+      label: "Filter",
+      description: "Filter to apply to search results",
+      optional: true,
+    },
+    minScore: {
+      type: "number",
+      label: "Minimum Score",
+      description: "Minimum score threshold for results",
+      optional: true,
+    },
+    includeRelationship: {
+      type: "boolean",
+      label: "Include Relationship",
+      description: "Include relationship data in results",
+      optional: true,
+    },
+    includeMetadata: {
+      type: "boolean",
+      label: "Include Metadata",
+      description: "Include metadata in results",
+      optional: true,
+    },
+  },
+  async run({ $ }) {
+    const response = await this.agentset.searchNamespace(this.namespaceId, this.query, {
+      topK: this.topK,
+      rerank: this.rerank,
+      rerankLimit: this.rerankLimit,
+      filter: this.filter,
+      minScore: this.minScore,
+      includeRelationships: this.includeRelationship,
+      includeMetadata: this.includeMetadata,
+    });
+
+    $.export("$summary", `Successfully completed the search for query: "${this.query}"`);
+    return response;
+  },
+};
diff --git a/components/agentset/agentset.app.mjs b/components/agentset/agentset.app.mjs
index fbf090bcc6d46..bb519ce939ab1 100644
--- a/components/agentset/agentset.app.mjs
+++ b/components/agentset/agentset.app.mjs
@@ -1,11 +1,211 @@
+import { axios } from "@pipedream/platform";
+
 export default {
   type: "app",
   app: "agentset",
-  propDefinitions: {},
+  propDefinitions: {
+    namespaceId: {
+      type: "string",
+      label: "Namespace ID",
+      description: "The ID of the namespace",
+      async options() {
+        const namespaces = await this.listNamespaces();
+        return namespaces.map((ns) => ({
+          label: ns.name,
+          value: ns.id,
+        }));
+      },
+    },
+    documentStatuses: {
+      type: "string[]",
+      label: "Document Statuses",
+      description: "Filter documents by status",
+      optional: true,
+      options: [
+        {
+          label: "BACKLOG",
+          value: "BACKLOG",
+        },
+        {
+          label: "QUEUED",
+          value: "QUEUED",
+        },
+        {
+          label: "QUEUED_FOR_RESYNC",
+          value: "QUEUED_FOR_RESYNC",
+        },
+        {
+          label: "QUEUED_FOR_DELETE",
+          value: "QUEUED_FOR_DELETE",
+        },
+        {
+          label: "PRE_PROCESSING",
+          value: "PRE_PROCESSING",
+        },
+        {
+          label: "PROCESSING",
+          value: "PROCESSING",
+        },
+        {
+          label: "DELETING",
+          value: "DELETING",
+        },
+        {
+          label: "CANCELLING",
+          value: "CANCELLING",
+        },
+        {
+          label: "COMPLETED",
+          value: "COMPLETED",
+        },
+        {
+          label: "FAILED",
+          value: "FAILED",
+        },
+        {
+          label: "CANCELLED",
+          value: "CANCELLED",
+        },
+      ],
+    },
+    payloadType: {
+      type: "string",
+      label: "Payload Type",
+      description: "Type of payload for the ingest job",
+    },
+    payload: {
+      type: "string",
+      label: "Payload",
+      description: "The data payload for the ingest job",
+    },
+    query: {
+      type: "string",
+      label: "Query",
+      description: "The query for semantic search",
+    },
+    topK: {
+      type: "integer",
+      label: "Top K",
+      description: "Number of top documents to return",
+      optional: true,
+    },
+    rerank: {
+      type: "boolean",
+      label: "Rerank",
+      description: "Rerank documents based on query",
+      optional: true,
+    },
+    rerankLimit: {
+      type: "integer",
+      label: "Rerank Limit",
+      description: "Limit for reranking documents",
+      optional: true,
+    },
+    filter: {
+      type: "string",
+      label: "Filter",
+      description: "Filter to apply to search results",
+      optional: true,
+    },
+    minScore: {
+      type: "number",
+      label: "Minimum Score",
+      description: "Minimum score threshold for results",
+      optional: true,
+    },
+    includeRelationship: {
+      type: "boolean",
+      label: "Include Relationship",
+      description: "Include relationship data in results",
+      optional: true,
+    },
+    includeMetadata: {
+      type: "boolean",
+      label: "Include Metadata",
+      description: "Include metadata in results",
+      optional: true,
+    },
+  },
   methods: {
-    // this.$auth contains connected account data
-    authKeys() {
-      console.log(Object.keys(this.$auth));
+    _baseUrl() {
+      return "https://api.agentset.ai/v1";
+    },
+    async _makeRequest(opts = {}) {
+      const {
+        $ = this, method = "GET", path = "/", headers, ...otherOpts
+      } = opts;
+      return axios($, {
+        ...otherOpts,
+        method,
+        url: this._baseUrl() + path,
+        headers: {
+          ...headers,
+          Authorization: `Bearer ${this.$auth.api_key}`,
+        },
+      });
+    },
+    async listNamespaces(opts = {}) {
+      return this._makeRequest({
+        path: "/namespace",
+        ...opts,
+      });
+    },
+    async createNamespace(name) {
+      return this._makeRequest({
+        method: "POST",
+        path: "/namespace",
+        data: {
+          name,
+        },
+      });
+    },
+    async createIngestJob(namespaceId, payloadType, payload) {
+      return this._makeRequest({
+        method: "POST",
+        path: `/namespace/${namespaceId}/ingest-jobs`,
+        data: {
+          payloadType,
+          payload,
+        },
+      });
+    },
+    async listIngestJobs(namespaceId, opts = {}) {
+      return this._makeRequest({
+        path: `/namespace/${namespaceId}/ingest-jobs`,
+        ...opts,
+      });
     },
+    async listDocuments(namespaceId, opts = {}) {
+      return this._makeRequest({
+        path: `/namespace/${namespaceId}/documents`,
+        ...opts,
+      });
+    },
+    async searchNamespace(namespaceId, query, opts = {}) {
+      return this._makeRequest({
+        method: "POST",
+        path: `/namespace/${namespaceId}/search`,
+        data: {
+          query,
+          ...opts,
+        },
+      });
+    },
+  },
+  async createNamespaceAndEmitEvent($, name) {
+    const namespace = await this.createNamespace(name);
+    $.export("namespace", namespace);
+  },
+  async createIngestJobAndEmitEvent($, namespaceId, payloadType, payload) {
+    const ingestJob = await this.createIngestJob(namespaceId, payloadType, payload);
+    $.export("ingestJob", ingestJob);
+  },
+  async listDocumentsAndEmit($, namespaceId, statuses) {
+    const documents = await this.listDocuments(namespaceId, {
+      params: {
+        statuses,
+      },
+    });
+    $.export("documents", documents);
   },
-};
\ No newline at end of file
+};
diff --git a/components/agentset/package.json b/components/agentset/package.json
index 720c4d8649e2a..fede995bd1502 100644
--- a/components/agentset/package.json
+++ b/components/agentset/package.json
@@ -12,4 +12,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
\ No newline at end of file
+}
diff --git a/components/agentset/sources/new-document/new-document.mjs b/components/agentset/sources/new-document/new-document.mjs
new file mode 100644
index 0000000000000..6ebe5aedc0429
--- /dev/null
+++ b/components/agentset/sources/new-document/new-document.mjs
@@ -0,0 +1,89 @@
+import agentset from "../../agentset.app.mjs";
+import { axios } from "@pipedream/platform";
+
+export default {
+  key: "agentset-new-document",
+  name: "New Document Created",
+  description: "Emit a new event when a new document is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
+  version: "0.0.{{ts}}",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    agentset,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 900,
+      },
+    },
+    namespaceId: {
+      propDefinition: [
+        agentset,
+        "namespaceId",
+      ],
+    },
+    documentStatuses: {
+      propDefinition: [
+        agentset,
+        "documentStatuses",
+      ],
+    },
+  },
+  methods: {
+    async _getDocuments(cursor) {
+      return this.agentset.listDocuments(this.namespaceId, {
+        params: {
+          statuses: this.documentStatuses,
+          cursor,
+          orderBy: "createdAt",
+          order: "desc",
+        },
+      });
+    },
+    _getStoredCursor() {
+      return this.db.get("cursor") || null;
+    },
+    _setStoredCursor(cursor) {
+      this.db.set("cursor", cursor);
+    },
+  },
+  hooks: {
+    async deploy() {
+      const {
+        data, pagination,
+      } = await this._getDocuments();
+      for (const doc of data.slice(0, 50).reverse()) {
+        this.$emit(doc, {
+          id: doc.id,
+          summary: `New Document: ${doc.name || doc.id}`,
+          ts: new Date(doc.createdAt).getTime(),
+        });
+      }
+      if (pagination.nextCursor) {
+        this._setStoredCursor(pagination.nextCursor);
+      }
+    },
+  },
+  async run() {
+    let cursor = this._getStoredCursor();
+    while (true) {
+      const {
+        data, pagination,
+      } = await this._getDocuments(cursor);
+      for (const doc of data.reverse()) {
+        this.$emit(doc, {
+          id: doc.id,
+          summary: `New Document: ${doc.name || doc.id}`,
+          ts: new Date(doc.createdAt).getTime(),
+        });
+      }
+
+      if (!pagination.nextCursor || data.length === 0) break;
+      cursor = pagination.nextCursor;
+    }
+
+    // Update cursor for next run
+    this._setStoredCursor(cursor);
+  },
+};
diff --git a/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs b/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs
new file mode 100644
index 0000000000000..caade4c9fb608
--- /dev/null
+++ b/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs
@@ -0,0 +1,68 @@
+import { axios } from "@pipedream/platform";
+import agentset from "../../agentset.app.mjs";
+
+export default {
+  key: "agentset-new-ingest-job",
+  name: "New Ingest Job Created",
+  description: "Emit new event when a new ingest job is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
+  version: "0.0.{{ts}}",
+  type: "source",
+  dedupe: "unique",
+  props: {
+    agentset,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: 15 * 60,
+      },
+    },
+    namespaceId: {
+      propDefinition: [
+        agentset,
+        "namespaceId",
+      ],
+    },
+  },
+  methods: {
+    _getLastIngestJobId() {
+      return this.db.get("lastIngestJobId");
+    },
+    _setLastIngestJobId(lastIngestJobId) {
+      this.db.set("lastIngestJobId", lastIngestJobId);
+    },
+    async listIngestJobs(namespaceId, params = {}) {
+      return this.agentset.listIngestJobs(namespaceId, {
+        params,
+      });
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.processIngestJobs();
+    },
+  },
+  async run() {
+    await this.processIngestJobs();
+  },
+  async processIngestJobs() {
+    const namespaceId = this.namespaceId;
+    const lastIngestJobId = this._getLastIngestJobId();
+
+    const ingestJobs = await this.listIngestJobs(namespaceId);
+
+    for (const job of ingestJobs.reverse()) {
+      if (job.id === lastIngestJobId) break;
+
+      this.$emit(job, {
+        id: job.id,
+        summary: `New Ingest Job: ${job.payload.type}`,
+        ts: Date.parse(job.createdAt),
+      });
+    }
+
+    if (ingestJobs[0]) {
+      this._setLastIngestJobId(ingestJobs[0].id);
+    }
+  },
+};

From 94f2901d7ae12324f50345a21591e69b1c76116a Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Wed, 9 Apr 2025 17:49:12 -0300
Subject: [PATCH 2/8] [Components] agentset #16193 Sources  - New Ingest Job  -
 New Document

Actions
 - Create Namespace
 - Create Ingest Job
 - Search Namespace
---
 .../create-injest-job/create-injest-job.mjs   |  90 +++++--
 .../create-namespace/create-namespace.mjs     |  70 +-----
 .../search-namespace/search-namespace.mjs     |  43 ++--
 components/agentset/agentset.app.mjs          | 219 ++++++------------
 components/agentset/common/constants.mjs      |  19 ++
 components/agentset/common/utils.mjs          |  33 +++
 components/agentset/package.json              |   5 +-
 components/agentset/sources/common/base.mjs   |  77 ++++++
 .../sources/new-document/new-document.mjs     |  91 +-------
 .../sources/new-document/test-event.mjs       |  24 ++
 .../sources/new-ingest-job/new-ingest-job.mjs |  22 ++
 .../sources/new-ingest-job/test-event.mjs     |  19 ++
 .../new-ingest-jopb/new-ingest-jopb.mjs       |  68 ------
 13 files changed, 384 insertions(+), 396 deletions(-)
 create mode 100644 components/agentset/common/constants.mjs
 create mode 100644 components/agentset/common/utils.mjs
 create mode 100644 components/agentset/sources/common/base.mjs
 create mode 100644 components/agentset/sources/new-document/test-event.mjs
 create mode 100644 components/agentset/sources/new-ingest-job/new-ingest-job.mjs
 create mode 100644 components/agentset/sources/new-ingest-job/test-event.mjs
 delete mode 100644 components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs

diff --git a/components/agentset/actions/create-injest-job/create-injest-job.mjs b/components/agentset/actions/create-injest-job/create-injest-job.mjs
index 4e005b057597c..b63d4614e3e06 100644
--- a/components/agentset/actions/create-injest-job/create-injest-job.mjs
+++ b/components/agentset/actions/create-injest-job/create-injest-job.mjs
@@ -1,11 +1,12 @@
 import agentset from "../../agentset.app.mjs";
-import { axios } from "@pipedream/platform";
+import { PAYLOAD_TYPE_OPTIONS } from "../../common/constants.mjs";
+import { parseObject } from "../../common/utils.mjs";
 
 export default {
   key: "agentset-create-ingest-job",
   name: "Create Ingest Job",
   description: "Create an ingest job for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/create)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     agentset,
@@ -16,21 +17,84 @@ export default {
       ],
     },
     payloadType: {
-      propDefinition: [
-        agentset,
-        "payloadType",
-      ],
+      type: "string",
+      label: "Payload Type",
+      description: "Type of payload for the ingest job",
+      options: PAYLOAD_TYPE_OPTIONS,
+      reloadProps: true,
     },
-    payload: {
-      propDefinition: [
-        agentset,
-        "payload",
-      ],
+    text: {
+      type: "string",
+      label: "Text",
+      description: "The text to ingest",
+      hidden: true,
     },
+    fileUrl: {
+      type: "string",
+      label: "File URL",
+      description: "The URL of the file to ingest",
+      hidden: true,
+    },
+    urls: {
+      type: "string[]",
+      label: "URLs",
+      description: "The URLs to ingest",
+      hidden: true,
+    },
+    name: {
+      type: "string",
+      label: "Name",
+      description: "The name of the ingest job",
+      optional: true,
+      hidden: true,
+    },
+  },
+  async additionalProps(props) {
+    props.text.hidden = true;
+    props.name.hidden = true;
+    props.fileUrl.hidden = true;
+    props.urls.hidden = true;
+
+    switch (this.payloadType) {
+    case "TEXT":
+      props.text.hidden = false;
+      props.name.hidden = false;
+      break;
+    case "FILE":
+      props.fileUrl.hidden = false;
+      props.name.hidden = false;
+      break;
+    case "URLS":
+      props.urls.hidden = false;
+      break;
+    }
+    return {};
   },
   async run({ $ }) {
-    const response = await this.agentset.createIngestJob(this.namespaceId, this.payloadType, this.payload);
-    $.export("$summary", `Ingest job created successfully: ID ${response.id}`);
+    const payload = {
+      type: this.payloadType,
+    };
+    switch (this.payloadType) {
+    case "TEXT":
+      payload.text = this.text;
+      payload.name = this.name;
+      break;
+    case "FILE":
+      payload.fileUrl = this.fileUrl;
+      payload.name = this.name;
+      break;
+    case "URLS":
+      payload.urls = parseObject(this.urls);
+      break;
+    }
+    const response = await this.agentset.createIngestJob({
+      $,
+      namespaceId: this.namespaceId,
+      data: {
+        payload,
+      },
+    });
+    $.export("$summary", `Ingest job created successfully: ID ${response.data.id}`);
     return response;
   },
 };
diff --git a/components/agentset/actions/create-namespace/create-namespace.mjs b/components/agentset/actions/create-namespace/create-namespace.mjs
index 9eac9c78d4055..62f3d9dcd3fc3 100644
--- a/components/agentset/actions/create-namespace/create-namespace.mjs
+++ b/components/agentset/actions/create-namespace/create-namespace.mjs
@@ -1,11 +1,11 @@
 import agentset from "../../agentset.app.mjs";
-import { axios } from "@pipedream/platform";
+import { slugify } from "../../common/utils.mjs";
 
 export default {
   key: "agentset-create-namespace",
   name: "Create Namespace",
   description: "Creates a namespace for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/namespaces/create)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     agentset,
@@ -14,75 +14,17 @@ export default {
       label: "Name",
       description: "The name of the namespace to create",
     },
-    slug: {
-      type: "string",
-      label: "Slug",
-      description: "A unique slug for the namespace",
-    },
-    embeddingConfigProvider: {
-      type: "string",
-      label: "Embedding Config Provider",
-      description: "Provider for the embedding config",
-      options: [
-        "OPENAI",
-      ],
-    },
-    embeddingConfigModel: {
-      type: "string",
-      label: "Embedding Config Model",
-      description: "The model for the embedding config",
-      options: [
-        "text-embedding-3-small",
-        "text-embedding-3-large",
-      ],
-    },
-    embeddingConfigApiKey: {
-      type: "string",
-      label: "Embedding Config API Key",
-      description: "API key for the embedding config provider",
-      secret: true,
-    },
-    vectorStoreConfigProvider: {
-      type: "string",
-      label: "Vector Store Config Provider",
-      description: "Provider for the vector store config",
-      options: [
-        "PINECONE",
-      ],
-    },
-    vectorStoreConfigApiKey: {
-      type: "string",
-      label: "Vector Store Config API Key",
-      description: "API key for the vector store",
-      secret: true,
-    },
-    vectorStoreConfigIndexHost: {
-      type: "string",
-      label: "Vector Store Config Index Host",
-      description: "URL of the Pinecone index host",
-    },
   },
   async run({ $ }) {
-    const response = await this.agentset._makeRequest({
-      method: "POST",
-      path: "/namespace",
+    const response = await this.agentset.createNamespace({
+      $,
       data: {
         name: this.name,
-        slug: this.slug,
-        embeddingConfig: {
-          provider: this.embeddingConfigProvider,
-          model: this.embeddingConfigModel,
-          apiKey: this.embeddingConfigApiKey,
-        },
-        vectorStoreConfig: {
-          provider: this.vectorStoreConfigProvider,
-          apiKey: this.vectorStoreConfigApiKey,
-          indexHost: this.vectorStoreConfigIndexHost,
-        },
+        slug: slugify(this.name),
       },
     });
 
-    $.export("$summary", `Successfully created namespace ${response.name}`);
+    $.export("$summary", `Successfully created namespace with ID: ${response.data.id}`);
     return response;
   },
 };
diff --git a/components/agentset/actions/search-namespace/search-namespace.mjs b/components/agentset/actions/search-namespace/search-namespace.mjs
index 4b974df3598e6..594debdc24d81 100644
--- a/components/agentset/actions/search-namespace/search-namespace.mjs
+++ b/components/agentset/actions/search-namespace/search-namespace.mjs
@@ -1,11 +1,10 @@
 import agentset from "../../agentset.app.mjs";
-import { axios } from "@pipedream/platform";
 
 export default {
   key: "agentset-search-namespace",
   name: "Agentset Search Namespace",
   description: "Complete retrieval pipeline for RAG with semantic search, filtering, and reranking. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/search)",
-  version: "0.0.{{ts}}",
+  version: "0.0.1",
   type: "action",
   props: {
     agentset,
@@ -24,54 +23,64 @@ export default {
       type: "integer",
       label: "Top K",
       description: "Number of top documents to return",
+      min: 1,
+      max: 100,
       optional: true,
     },
     rerank: {
       type: "boolean",
       label: "Rerank",
-      description: "Rerank documents based on query",
+      description: "Whether to rerank the results",
       optional: true,
     },
     rerankLimit: {
       type: "integer",
       label: "Rerank Limit",
-      description: "Limit for reranking documents",
+      description: "The number of results to return after reranking",
+      min: 1,
+      max: 100,
       optional: true,
     },
     filter: {
-      type: "string",
+      type: "object",
       label: "Filter",
       description: "Filter to apply to search results",
       optional: true,
     },
     minScore: {
-      type: "number",
+      type: "string",
       label: "Minimum Score",
-      description: "Minimum score threshold for results",
+      description: "Minimum score to return. Range from 0 to 1",
       optional: true,
     },
     includeRelationship: {
       type: "boolean",
       label: "Include Relationship",
-      description: "Include relationship data in results",
+      description: "Whether to include relationships in the results",
       optional: true,
     },
     includeMetadata: {
       type: "boolean",
       label: "Include Metadata",
-      description: "Include metadata in results",
+      description: "Whether to include metadata in the results",
       optional: true,
     },
   },
+
   async run({ $ }) {
-    const response = await this.agentset.searchNamespace(this.namespaceId, this.query, {
-      topK: this.topK,
-      rerank: this.rerank,
-      rerankLimit: this.rerankLimit,
-      filter: this.filter,
-      minScore: this.minScore,
-      includeRelationships: this.includeRelationship,
-      includeMetadata: this.includeMetadata,
+    const response = await this.agentset.searchNamespace({
+      $,
+      namespaceId: this.namespaceId,
+      data: {
+        query: this.query,
+        topK: this.topK,
+        rerank: this.rerank,
+        rerankLimit: this.rerankLimit,
+        filter: this.filter,
+        minScore: this.minScore && parseFloat(this.minScore),
+        includeRelationships: this.includeRelationship,
+        includeMetadata: this.includeMetadata,
+      },
     });
 
     $.export("$summary", `Successfully completed the search for query: "${this.query}"`);
diff --git a/components/agentset/agentset.app.mjs b/components/agentset/agentset.app.mjs
index bb519ce939ab1..ae62b54d703fe 100644
--- a/components/agentset/agentset.app.mjs
+++ b/components/agentset/agentset.app.mjs
@@ -1,4 +1,5 @@
 import { axios } from "@pipedream/platform";
+import { STATUSES_OPTIONS } from "./common/constants.mjs";
 
 export default {
   type: "app",
@@ -9,203 +10,113 @@ export default {
       label: "Namespace ID",
       description: "The ID of the namespace",
       async options() {
-        const namespaces = await this.listNamespaces();
-        return namespaces.map((ns) => ({
-          label: ns.name,
-          value: ns.id,
+        const { data } = await this.listNamespaces();
+        return data.map(({
+          id: value, name: label,
+        }) => ({
+          label,
+          value,
         }));
       },
     },
-    documentStatuses: {
+    statuses: {
       type: "string[]",
-      label: "Document Statuses",
-      description: "Filter documents by status",
-      optional: true,
-      options: [
-        {
-          label: "BACKLOG",
-          value: "BACKLOG",
-        },
-        {
-          label: "QUEUED",
-          value: "QUEUED",
-        },
-        {
-          label: "QUEUED_FOR_RESYNC",
-          value: "QUEUED_FOR_RESYNC",
-        },
-        {
-          label: "QUEUED_FOR_DELETE",
-          value: "QUEUED_FOR_DELETE",
-        },
-        {
-          label: "PRE_PROCESSING",
-          value: "PRE_PROCESSING",
-        },
-        {
-          label: "PROCESSING",
-          value: "PROCESSING",
-        },
-        {
-          label: "DELETING",
-          value: "DELETING",
-        },
-        {
-          label: "CANCELLING",
-          value: "CANCELLING",
-        },
-        {
-          label: "COMPLETED",
-          value: "COMPLETED",
-        },
-        {
-          label: "FAILED",
-          value: "FAILED",
-        },
-        {
-          label: "CANCELLED",
-          value: "CANCELLED",
-        },
-      ],
-    },
-    payloadType: {
-      type: "string",
-      label: "Payload Type",
-      description: "Type of payload for the ingest job",
-    },
-    payload: {
-      type: "string",
-      label: "Payload",
-      description: "The data payload for the ingest job",
-    },
-    query: {
-      type: "string",
-      label: "Query",
-      description: "The query for semantic search",
-    },
-    topK: {
-      type: "integer",
-      label: "Top K",
-      description: "Number of top documents to return",
-      optional: true,
-    },
-    rerank: {
-      type: "boolean",
-      label: "Rerank",
-      description: "Rerank documents based on query",
-      optional: true,
-    },
-    rerankLimit: {
-      type: "integer",
-      label: "Rerank Limit",
-      description: "Limit for reranking documents",
-      optional: true,
-    },
-    filter: {
-      type: "string",
-      label: "Filter",
-      description: "Filter to apply to search results",
-      optional: true,
-    },
-    minScore: {
-      type: "number",
-      label: "Minimum Score",
-      description: "Minimum score threshold for results",
-      optional: true,
-    },
-    includeRelationship: {
-      type: "boolean",
-      label: "Include Relationship",
-      description: "Include relationship data in results",
-      optional: true,
-    },
-    includeMetadata: {
-      type: "boolean",
-      label: "Include Metadata",
-      description: "Include metadata in results",
-      optional: true,
+      label: "Statuses",
+      description: "Filter by status",
+      options: STATUSES_OPTIONS,
     },
   },
   methods: {
     _baseUrl() {
       return "https://api.agentset.ai/v1";
     },
-    async _makeRequest(opts = {}) {
-      const {
-        $ = this, method = "GET", path = "/", headers, ...otherOpts
-      } = opts;
+    _headers() {
+      return {
+        Authorization: `Bearer ${this.$auth.api_key}`,
+      };
+    },
+    _makeRequest({
+      $ = this, path, ...opts
+    }) {
       return axios($, {
-        ...otherOpts,
-        method,
         url: this._baseUrl() + path,
-        headers: {
-          ...headers,
-          Authorization: `Bearer ${this.$auth.api_key}`,
-        },
+        headers: this._headers(),
+        ...opts,
       });
     },
-    async listNamespaces(opts = {}) {
+    listNamespaces(opts = {}) {
       return this._makeRequest({
         path: "/namespace",
         ...opts,
       });
     },
-    async createNamespace(name) {
+    createNamespace(opts = {}) {
       return this._makeRequest({
         method: "POST",
         path: "/namespace",
-        data: {
-          name,
-        },
+        ...opts,
       });
     },
-    async createIngestJob(namespaceId, payloadType, payload) {
+    createIngestJob({
+      namespaceId, ...opts
+    }) {
       return this._makeRequest({
         method: "POST",
         path: `/namespace/${namespaceId}/ingest-jobs`,
-        data: {
-          payloadType,
-          payload,
-        },
+        ...opts,
       });
     },
-    async listIngestJobs(namespaceId, opts = {}) {
+    listIngestJobs({
+      namespaceId, ...opts
+    }) {
       return this._makeRequest({
         path: `/namespace/${namespaceId}/ingest-jobs`,
         ...opts,
       });
     },
-    async listDocuments(namespaceId, opts = {}) {
+    listDocuments({
+      namespaceId, ...opts
+    }) {
       return this._makeRequest({
         path: `/namespace/${namespaceId}/documents`,
         ...opts,
       });
     },
-    async searchNamespace(namespaceId, query, opts = {}) {
+    searchNamespace({
+      namespaceId, ...opts
+    }) {
       return this._makeRequest({
         method: "POST",
         path: `/namespace/${namespaceId}/search`,
-        data: {
-          query,
-          ...opts,
-        },
+        ...opts,
       });
     },
-  },
-  async createNamespaceAndEmitEvent($, name) {
-    const namespace = await this.createNamespace(name);
-    $.export("namespace", namespace);
-  },
-  async createIngestJobAndEmitEvent($, namespaceId, payloadType, payload) {
-    const ingestJob = await this.createIngestJob(namespaceId, payloadType, payload);
-    $.export("ingestJob", ingestJob);
-  },
-  async listDocumentsAndEmit($, namespaceId, statuses) {
-    const documents = await this.listDocuments(namespaceId, {
-      params: {
-        statuses,
-      },
-    });
-    $.export("documents", documents);
+    async *paginate({
+      fn, params = {}, maxResults = null, ...opts
+    }) {
+      let count = 0;
+      let cursor;
+
+      do {
+        params.cursor = cursor;
+        const {
+          data,
+          pagination: { nextCursor },
+        } = await fn({
+          params,
+          ...opts,
+        });
+        for (const d of data) {
+          yield d;
+
+          if (maxResults && ++count === maxResults) {
+            return count;
+          }
+        }
+
+        cursor = nextCursor;
+
+      } while (cursor);
+    },
   },
 };
diff --git a/components/agentset/common/constants.mjs b/components/agentset/common/constants.mjs
new file mode 100644
index 0000000000000..b2772d07d7cf7
--- /dev/null
+++ b/components/agentset/common/constants.mjs
@@ -0,0 +1,19 @@
+export const PAYLOAD_TYPE_OPTIONS = [
+  "TEXT",
+  "FILE",
+  "URLS",
+];
+
+export const STATUSES_OPTIONS = [
+  "BACKLOG",
+  "QUEUED",
+  "QUEUED_FOR_RESYNC",
+  "QUEUED_FOR_DELETE",
+  "PRE_PROCESSING",
+  "PROCESSING",
+  "DELETING",
+  "CANCELLING",
+  "COMPLETED",
+  "FAILED",
+  "CANCELLED",
+];
diff --git a/components/agentset/common/utils.mjs b/components/agentset/common/utils.mjs
new file mode 100644
index 0000000000000..f691cd3e147ee
--- /dev/null
+++ b/components/agentset/common/utils.mjs
@@ -0,0 +1,33 @@
+export const slugify = (str) => {
+  str = str.replace(/^\s+|\s+$/g, "");
+  str = str.toLowerCase();
+  str = str.replace(/[^a-z0-9 -]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-");
+  return str;
+};
+
+export const parseObject = (obj) => {
+  if (!obj) return undefined;
+
+  if (Array.isArray(obj)) {
+    return obj.map((item) => {
+      if (typeof item === "string") {
+        try {
+          return JSON.parse(item);
+        } catch (e) {
+          return item;
+        }
+      }
+      return item;
+    });
+  }
+  if (typeof obj === "string") {
+    try {
+      return JSON.parse(obj);
+    } catch (e) {
+      return obj;
+    }
+  }
+  return obj;
+};
diff --git a/components/agentset/package.json b/components/agentset/package.json
index fede995bd1502..9ddb2a04d0aa3 100644
--- a/components/agentset/package.json
+++ b/components/agentset/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@pipedream/agentset",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "description": "Pipedream Agentset Components",
   "main": "agentset.app.mjs",
   "keywords": [
@@ -11,5 +11,8 @@
   "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
   "publishConfig": {
     "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.0.3"
   }
 }
diff --git a/components/agentset/sources/common/base.mjs b/components/agentset/sources/common/base.mjs
new file mode 100644
index 0000000000000..bd56dcefe4181
--- /dev/null
+++ b/components/agentset/sources/common/base.mjs
@@ -0,0 +1,77 @@
+import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
+import agentset from "../../agentset.app.mjs";
+
+export default {
+  props: {
+    agentset,
+    db: "$.service.db",
+    timer: {
+      type: "$.interface.timer",
+      default: {
+        intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
+      },
+    },
+    namespaceId: {
+      propDefinition: [
+        agentset,
+        "namespaceId",
+      ],
+    },
+    statuses: {
+      propDefinition: [
+        agentset,
+        "statuses",
+      ],
+      optional: true,
+    },
+  },
+  methods: {
+    _getLastData() {
+      return this.db.get("lastData") || 0;
+    },
+    _setLastData(lastData) {
+      this.db.set("lastData", lastData);
+    },
+    async emitEvent(maxResults = false) {
+      const lastData = this._getLastData();
+
+      const response = this.agentset.paginate({
+        fn: this.getFunction(),
+        namespaceId: this.namespaceId,
+        params: {
+          statuses: this.statuses,
+          orderBy: "createdAt",
+          order: "desc",
+          pageSize: 100,
+          maxResults,
+        },
+      });
+
+      let responseArray = [];
+      for await (const item of response) {
+        if (Date.parse(item.createdAt) <= lastData) break;
+        responseArray.push(item);
+      }
+
+      if (responseArray.length) {
+        this._setLastData(Date.parse(responseArray[0].createdAt));
+      }
+
+      for (const item of responseArray.reverse()) {
+        this.$emit(item, {
+          id: `${item.id}-${item.status}`,
+          summary: this.getSummary(item),
+          ts: Date.parse(item.createdAt),
+        });
+      }
+    },
+  },
+  hooks: {
+    async deploy() {
+      await this.emitEvent(25);
+    },
+  },
+  async run() {
+    await this.emitEvent();
+  },
+};
diff --git a/components/agentset/sources/new-document/new-document.mjs b/components/agentset/sources/new-document/new-document.mjs
index 6ebe5aedc0429..105fc95b74727 100644
--- a/components/agentset/sources/new-document/new-document.mjs
+++ b/components/agentset/sources/new-document/new-document.mjs
@@ -1,89 +1,22 @@
-import agentset from "../../agentset.app.mjs";
-import { axios } from "@pipedream/platform";
+import common from "../common/base.mjs";
+import sampleEmit from "./test-event.mjs";
 
 export default {
+  ...common,
   key: "agentset-new-document",
-  name: "New Document Created",
-  description: "Emit a new event when a new document is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
-  version: "0.0.{{ts}}",
+  name: "New Document Status",
+  description: "Emit new event when a new document status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
+  version: "0.0.1",
   type: "source",
   dedupe: "unique",
-  props: {
-    agentset,
-    db: "$.service.db",
-    timer: {
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: 900,
-      },
-    },
-    namespaceId: {
-      propDefinition: [
-        agentset,
-        "namespaceId",
-      ],
-    },
-    documentStatuses: {
-      propDefinition: [
-        agentset,
-        "documentStatuses",
-      ],
-    },
-  },
   methods: {
-    async _getDocuments(cursor) {
-      return this.agentset.listDocuments(this.namespaceId, {
-        params: {
-          statuses: this.documentStatuses,
-          cursor,
-          orderBy: "createdAt",
-          order: "desc",
-        },
-      });
-    },
-    _getStoredCursor() {
-      return this.db.get("cursor") || null;
+    ...common.methods,
+    getFunction() {
+      return this.agentset.listDocuments;
     },
-    _setStoredCursor(cursor) {
-      this.db.set("cursor", cursor);
+    getSummary(item) {
+      return `New Document: ${item.name || item.id}`;
     },
   },
-  hooks: {
-    async deploy() {
-      const {
-        data, pagination,
-      } = await this._getDocuments();
-      for (const doc of data.slice(0, 50).reverse()) {
-        this.$emit(doc, {
-          id: doc.id,
-          summary: `New Document: ${doc.name || doc.id}`,
-          ts: new Date(doc.createdAt).getTime(),
-        });
-      }
-      if (pagination.nextCursor) {
-        this._setStoredCursor(pagination.nextCursor);
-      }
-    },
-  },
-  async run() {
-    let cursor = this._getStoredCursor();
-    while (true) {
-      const {
-        data, pagination,
-      } = await this._getDocuments(cursor);
-      for (const doc of data.reverse()) {
-        this.$emit(doc, {
-          id: doc.id,
-          summary: `New Document: ${doc.name || doc.id}`,
-          ts: new Date(doc.createdAt).getTime(),
-        });
-      }
-
-      if (!pagination.nextCursor || data.length === 0) break;
-      cursor = pagination.nextCursor;
-    }
-
-    // Update cursor for next run
-    this._setStoredCursor(cursor);
-  },
+  sampleEmit,
 };
diff --git a/components/agentset/sources/new-document/test-event.mjs b/components/agentset/sources/new-document/test-event.mjs
new file mode 100644
index 0000000000000..9ff5948b31967
--- /dev/null
+++ b/components/agentset/sources/new-document/test-event.mjs
@@ -0,0 +1,24 @@
+export default {
+  "id": "<string>",
+  "ingestJobId": "<string>",
+  "externalId": null,
+  "name": null,
+  "tenantId": null,
+  "status": "BACKLOG",
+  "error": null,
+  "source": {
+    "type": "TEXT",
+    "text": "<string>"
+  },
+  "properties": null,
+  "totalChunks": 123,
+  "totalTokens": 123,
+  "totalCharacters": 123,
+  "totalPages": 123,
+  "createdAt": "<string>",
+  "queuedAt": null,
+  "preProcessingAt": null,
+  "processingAt": null,
+  "completedAt": null,
+  "failedAt": null
+}
\ No newline at end of file
diff --git a/components/agentset/sources/new-ingest-job/new-ingest-job.mjs b/components/agentset/sources/new-ingest-job/new-ingest-job.mjs
new file mode 100644
index 0000000000000..c3110992e4bf8
--- /dev/null
+++ b/components/agentset/sources/new-ingest-job/new-ingest-job.mjs
@@ -0,0 +1,22 @@
+import common from "../common/base.mjs";
+import sampleEmit from "./test-event.mjs";
+
+export default {
+  ...common,
+  key: "agentset-new-ingest-job",
+  name: "New Ingest Job Status",
+  description: "Emit new event when a new ingest job status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
+  version: "0.0.1",
+  type: "source",
+  dedupe: "unique",
+  methods: {
+    ...common.methods,
+    getFunction() {
+      return this.agentset.listIngestJobs;
+    },
+    getSummary(item) {
+      return `New Ingest Job: ${item.id}`;
+    },
+  },
+  sampleEmit,
+};
diff --git a/components/agentset/sources/new-ingest-job/test-event.mjs b/components/agentset/sources/new-ingest-job/test-event.mjs
new file mode 100644
index 0000000000000..4060cf1298f25
--- /dev/null
+++ b/components/agentset/sources/new-ingest-job/test-event.mjs
@@ -0,0 +1,19 @@
+export default {
+  "id": "<string>",
+  "namespaceId": "<string>",
+  "tenantId": null,
+  "status": "BACKLOG",
+  "error": null,
+  "payload": {
+    "type": "TEXT",
+    "text": "<string>",
+    "name": null
+  },
+  "config": null,
+  "createdAt": "<string>",
+  "queuedAt": null,
+  "preProcessingAt": null,
+  "processingAt": null,
+  "completedAt": null,
+  "failedAt": null
+}
\ No newline at end of file
diff --git a/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs b/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs
deleted file mode 100644
index caade4c9fb608..0000000000000
--- a/components/agentset/sources/new-ingest-jopb/new-ingest-jopb.mjs
+++ /dev/null
@@ -1,68 +0,0 @@
-import { axios } from "@pipedream/platform";
-import agentset from "../../agentset.app.mjs";
-
-export default {
-  key: "agentset-new-ingest-job",
-  name: "New Ingest Job Created",
-  description: "Emit new event when a new ingest job is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
-  version: "0.0.{{ts}}",
-  type: "source",
-  dedupe: "unique",
-  props: {
-    agentset,
-    db: "$.service.db",
-    timer: {
-      type: "$.interface.timer",
-      default: {
-        intervalSeconds: 15 * 60,
-      },
-    },
-    namespaceId: {
-      propDefinition: [
-        agentset,
-        "namespaceId",
-      ],
-    },
-  },
-  methods: {
-    _getLastIngestJobId() {
-      return this.db.get("lastIngestJobId");
-    },
-    _setLastIngestJobId(lastIngestJobId) {
-      this.db.set("lastIngestJobId", lastIngestJobId);
-    },
-    async listIngestJobs(namespaceId, params = {}) {
-      return this.agentset.listIngestJobs(namespaceId, {
-        params,
-      });
-    },
-  },
-  hooks: {
-    async deploy() {
-      await this.processIngestJobs();
-    },
-  },
-  async run() {
-    await this.processIngestJobs();
-  },
-  async processIngestJobs() {
-    const namespaceId = this.namespaceId;
-    const lastIngestJobId = this._getLastIngestJobId();
-
-    const ingestJobs = await this.listIngestJobs(namespaceId);
-
-    for (const job of ingestJobs.reverse()) {
-      if (job.id === lastIngestJobId) break;
-
-      this.$emit(job, {
-        id: job.id,
-        summary: `New Ingest Job: ${job.payload.type}`,
-        ts: Date.parse(job.createdAt),
-      });
-    }
-
-    if (ingestJobs[0]) {
-      this._setLastIngestJobId(ingestJobs[0].id);
-    }
-  },
-};

From d64d0ddb1e921161401114a0852f0f9e7c1dc86c Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Thu, 10 Apr 2025 16:26:07 -0300
Subject: [PATCH 3/8] pnpm update

---
 pnpm-lock.yaml | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7e0dd3ec1296f..4c304ba3044ec 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -425,7 +425,11 @@ importers:
 
   components/agentql: {}
 
-  components/agentset: {}
+  components/agentset:
+    dependencies:
+      '@pipedream/platform':
+        specifier: ^3.0.3
+        version: 3.0.3
 
   components/agenty:
     dependencies:
@@ -4922,8 +4926,7 @@ importers:
 
   components/gatekeeper: {}
 
-  components/gather:
-    specifiers: {}
+  components/gather: {}
 
   components/gatherup:
     dependencies:
@@ -8822,8 +8825,7 @@ importers:
         specifier: ^3.0.0
         version: 3.0.3
 
-  components/onehash:
-    specifiers: {}
+  components/onehash: {}
 
   components/onelogin: {}
 
@@ -10162,8 +10164,7 @@ importers:
         specifier: ^1.5.1
         version: 1.6.6
 
-  components/public_record:
-    specifiers: {}
+  components/public_record: {}
 
   components/publisherkit:
     dependencies:
@@ -10213,8 +10214,7 @@ importers:
         specifier: ^1.5.1
         version: 1.6.6
 
-  components/pushengage:
-    specifiers: {}
+  components/pushengage: {}
 
   components/pusher:
     dependencies:
@@ -27894,22 +27894,22 @@ packages:
   superagent@3.8.1:
     resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==}
     engines: {node: '>= 4.0'}
-    deprecated: Please upgrade to v7.0.2+ of superagent.  We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing.  See the releases tab for more information at <https://github.com/visionmedia/superagent/releases>.
+    deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net
 
   superagent@4.1.0:
     resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==}
     engines: {node: '>= 6.0'}
-    deprecated: Please upgrade to v7.0.2+ of superagent.  We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing.  See the releases tab for more information at <https://github.com/visionmedia/superagent/releases>.
+    deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net
 
   superagent@5.3.1:
     resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==}
     engines: {node: '>= 7.0.0'}
-    deprecated: Please upgrade to v7.0.2+ of superagent.  We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing.  See the releases tab for more information at <https://github.com/visionmedia/superagent/releases>.
+    deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net
 
   superagent@7.1.6:
     resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==}
     engines: {node: '>=6.4.0 <13 || >=14'}
-    deprecated: Please downgrade to v7.1.5 if you need IE/ActiveXObject support OR upgrade to v8.0.0 as we no longer support IE and published an incorrect patch version (see https://github.com/visionmedia/superagent/issues/1731)
+    deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net
 
   supports-color@2.0.0:
     resolution: {integrity: sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==}
@@ -34502,6 +34502,8 @@ snapshots:
       '@putout/operator-filesystem': 5.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3))
       '@putout/operator-json': 2.2.0
       putout: 36.13.1(eslint@8.57.1)(typescript@5.6.3)
+    transitivePeerDependencies:
+      - supports-color
 
   '@putout/operator-regexp@1.0.0(putout@36.13.1(eslint@8.57.1)(typescript@5.6.3))':
     dependencies:

From 3643f65be1b0fa776f131f2d349ce2c97c29dd83 Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Thu, 10 Apr 2025 16:29:20 -0300
Subject: [PATCH 4/8] pnpm update

---
 pnpm-lock.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d01ce1c75b496..ee925f62e9016 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -4312,8 +4312,7 @@ importers:
 
   components/facebook_conversions: {}
 
-  components/facebook_graph_api:
-    specifiers: {}
+  components/facebook_graph_api: {}
 
   components/facebook_groups:
     dependencies:

From 95ff67f42c1ac3b086e5ed758a1706f892b5e264 Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Thu, 10 Apr 2025 16:56:55 -0300
Subject: [PATCH 5/8] fix action name

---
 .../create-ingest-job.mjs}                                        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename components/agentset/actions/{create-injest-job/create-injest-job.mjs => create-ingest-job/create-ingest-job.mjs} (100%)

diff --git a/components/agentset/actions/create-injest-job/create-injest-job.mjs b/components/agentset/actions/create-ingest-job/create-ingest-job.mjs
similarity index 100%
rename from components/agentset/actions/create-injest-job/create-injest-job.mjs
rename to components/agentset/actions/create-ingest-job/create-ingest-job.mjs

From 7fc24f99040f9bd443d7db6a8496189bbb1072ca Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Thu, 10 Apr 2025 19:43:09 -0300
Subject: [PATCH 6/8] some adjusts

---
 components/agentset/common/utils.mjs                     | 9 +++++++++
 components/agentset/sources/common/base.mjs              | 7 +++++--
 .../document-status-updated.mjs}                         | 8 ++++----
 .../test-event.mjs                                       | 0
 .../ingest-job-status-updated.mjs}                       | 8 ++++----
 .../test-event.mjs                                       | 0
 6 files changed, 22 insertions(+), 10 deletions(-)
 rename components/agentset/sources/{new-document/new-document.mjs => document-status-updated/document-status-updated.mjs} (51%)
 rename components/agentset/sources/{new-document => document-status-updated}/test-event.mjs (100%)
 rename components/agentset/sources/{new-ingest-job/new-ingest-job.mjs => ingest-job-status-updated/ingest-job-status-updated.mjs} (51%)
 rename components/agentset/sources/{new-ingest-job => ingest-job-status-updated}/test-event.mjs (100%)

diff --git a/components/agentset/common/utils.mjs b/components/agentset/common/utils.mjs
index f691cd3e147ee..df48807e7cad7 100644
--- a/components/agentset/common/utils.mjs
+++ b/components/agentset/common/utils.mjs
@@ -31,3 +31,12 @@ export const parseObject = (obj) => {
   }
   return obj;
 };
+
+export const prepareDateTime = (data) => {
+  return data.completedAt ||
+  data.failedAt ||
+  data.processingAt ||
+  data.preProcessingAt ||
+  data.queuedAt ||
+  data.createdAt;
+};
diff --git a/components/agentset/sources/common/base.mjs b/components/agentset/sources/common/base.mjs
index bd56dcefe4181..7d61ebf48eec2 100644
--- a/components/agentset/sources/common/base.mjs
+++ b/components/agentset/sources/common/base.mjs
@@ -1,5 +1,6 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 import agentset from "../../agentset.app.mjs";
+import { prepareDateTime } from "../../common/utils.mjs";
 
 export default {
   props: {
@@ -49,7 +50,8 @@ export default {
 
       let responseArray = [];
       for await (const item of response) {
-        if (Date.parse(item.createdAt) <= lastData) break;
+        const dateTime = prepareDateTime(item);
+        if (Date.parse(dateTime) <= lastData) break;
         responseArray.push(item);
       }
 
@@ -58,10 +60,11 @@ export default {
       }
 
       for (const item of responseArray.reverse()) {
+        const dateTime = prepareDateTime(item);
         this.$emit(item, {
           id: `${item.id}-${item.status}`,
           summary: this.getSummary(item),
-          ts: Date.parse(item.createdAt),
+          ts: Date.parse(dateTime),
         });
       }
     },
diff --git a/components/agentset/sources/new-document/new-document.mjs b/components/agentset/sources/document-status-updated/document-status-updated.mjs
similarity index 51%
rename from components/agentset/sources/new-document/new-document.mjs
rename to components/agentset/sources/document-status-updated/document-status-updated.mjs
index 105fc95b74727..38dc196299922 100644
--- a/components/agentset/sources/new-document/new-document.mjs
+++ b/components/agentset/sources/document-status-updated/document-status-updated.mjs
@@ -3,9 +3,9 @@ import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
-  key: "agentset-new-document",
-  name: "New Document Status",
-  description: "Emit new event when a new document status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
+  key: "agentset-document-status-updated",
+  name: "Document Status Updated",
+  description: "Emit new event when a document status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
@@ -15,7 +15,7 @@ export default {
       return this.agentset.listDocuments;
     },
     getSummary(item) {
-      return `New Document: ${item.name || item.id}`;
+      return `Document ${item.name || item.id} has a new status: ${item.status}`;
     },
   },
   sampleEmit,
diff --git a/components/agentset/sources/new-document/test-event.mjs b/components/agentset/sources/document-status-updated/test-event.mjs
similarity index 100%
rename from components/agentset/sources/new-document/test-event.mjs
rename to components/agentset/sources/document-status-updated/test-event.mjs
diff --git a/components/agentset/sources/new-ingest-job/new-ingest-job.mjs b/components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs
similarity index 51%
rename from components/agentset/sources/new-ingest-job/new-ingest-job.mjs
rename to components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs
index c3110992e4bf8..e4c60456dfa7d 100644
--- a/components/agentset/sources/new-ingest-job/new-ingest-job.mjs
+++ b/components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs
@@ -3,9 +3,9 @@ import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
-  key: "agentset-new-ingest-job",
-  name: "New Ingest Job Status",
-  description: "Emit new event when a new ingest job status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
+  key: "agentset-ingest-job-status-updated",
+  name: "Ingest Job Status Updated",
+  description: "Emit new event when a ingest job status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
@@ -15,7 +15,7 @@ export default {
       return this.agentset.listIngestJobs;
     },
     getSummary(item) {
-      return `New Ingest Job: ${item.id}`;
+      return `Ingest Job (${item.id}) has a new status: ${item.status}`;
     },
   },
   sampleEmit,
diff --git a/components/agentset/sources/new-ingest-job/test-event.mjs b/components/agentset/sources/ingest-job-status-updated/test-event.mjs
similarity index 100%
rename from components/agentset/sources/new-ingest-job/test-event.mjs
rename to components/agentset/sources/ingest-job-status-updated/test-event.mjs

From b4eeb2c4f756063ae5ff2d75adaf6e604315f71d Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Fri, 11 Apr 2025 12:30:07 -0300
Subject: [PATCH 7/8] some adjusts

---
 components/agentset/common/utils.mjs            |  9 ---------
 components/agentset/sources/common/base.mjs     | 17 +++--------------
 .../new-document-created.mjs}                   |  8 ++++----
 .../test-event.mjs                              |  0
 .../new-ingest-job-created.mjs}                 |  8 ++++----
 .../test-event.mjs                              |  0
 6 files changed, 11 insertions(+), 31 deletions(-)
 rename components/agentset/sources/{document-status-updated/document-status-updated.mjs => new-document-created/new-document-created.mjs} (51%)
 rename components/agentset/sources/{document-status-updated => new-document-created}/test-event.mjs (100%)
 rename components/agentset/sources/{ingest-job-status-updated/ingest-job-status-updated.mjs => new-ingest-job-created/new-ingest-job-created.mjs} (51%)
 rename components/agentset/sources/{ingest-job-status-updated => new-ingest-job-created}/test-event.mjs (100%)

diff --git a/components/agentset/common/utils.mjs b/components/agentset/common/utils.mjs
index df48807e7cad7..f691cd3e147ee 100644
--- a/components/agentset/common/utils.mjs
+++ b/components/agentset/common/utils.mjs
@@ -31,12 +31,3 @@ export const parseObject = (obj) => {
   }
   return obj;
 };
-
-export const prepareDateTime = (data) => {
-  return data.completedAt ||
-  data.failedAt ||
-  data.processingAt ||
-  data.preProcessingAt ||
-  data.queuedAt ||
-  data.createdAt;
-};
diff --git a/components/agentset/sources/common/base.mjs b/components/agentset/sources/common/base.mjs
index 7d61ebf48eec2..c86ac62c97cbb 100644
--- a/components/agentset/sources/common/base.mjs
+++ b/components/agentset/sources/common/base.mjs
@@ -1,6 +1,5 @@
 import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
 import agentset from "../../agentset.app.mjs";
-import { prepareDateTime } from "../../common/utils.mjs";
 
 export default {
   props: {
@@ -18,13 +17,6 @@ export default {
         "namespaceId",
       ],
     },
-    statuses: {
-      propDefinition: [
-        agentset,
-        "statuses",
-      ],
-      optional: true,
-    },
   },
   methods: {
     _getLastData() {
@@ -40,7 +32,6 @@ export default {
         fn: this.getFunction(),
         namespaceId: this.namespaceId,
         params: {
-          statuses: this.statuses,
           orderBy: "createdAt",
           order: "desc",
           pageSize: 100,
@@ -50,8 +41,7 @@ export default {
 
       let responseArray = [];
       for await (const item of response) {
-        const dateTime = prepareDateTime(item);
-        if (Date.parse(dateTime) <= lastData) break;
+        if (Date.parse(item.createdAt) <= lastData) break;
         responseArray.push(item);
       }
 
@@ -60,11 +50,10 @@ export default {
       }
 
       for (const item of responseArray.reverse()) {
-        const dateTime = prepareDateTime(item);
         this.$emit(item, {
-          id: `${item.id}-${item.status}`,
+          id: `${item.id}`,
           summary: this.getSummary(item),
-          ts: Date.parse(dateTime),
+          ts: Date.parse(item.createdAt),
         });
       }
     },
diff --git a/components/agentset/sources/document-status-updated/document-status-updated.mjs b/components/agentset/sources/new-document-created/new-document-created.mjs
similarity index 51%
rename from components/agentset/sources/document-status-updated/document-status-updated.mjs
rename to components/agentset/sources/new-document-created/new-document-created.mjs
index 38dc196299922..9261f8fc19bea 100644
--- a/components/agentset/sources/document-status-updated/document-status-updated.mjs
+++ b/components/agentset/sources/new-document-created/new-document-created.mjs
@@ -3,9 +3,9 @@ import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
-  key: "agentset-document-status-updated",
-  name: "Document Status Updated",
-  description: "Emit new event when a document status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
+  key: "agentset-new-document-created",
+  name: "New Document Created",
+  description: "Emit new event when a new document is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/documents/list)",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
@@ -15,7 +15,7 @@ export default {
       return this.agentset.listDocuments;
     },
     getSummary(item) {
-      return `Document ${item.name || item.id} has a new status: ${item.status}`;
+      return `New document cerated: ${item.name || item.id}`;
     },
   },
   sampleEmit,
diff --git a/components/agentset/sources/document-status-updated/test-event.mjs b/components/agentset/sources/new-document-created/test-event.mjs
similarity index 100%
rename from components/agentset/sources/document-status-updated/test-event.mjs
rename to components/agentset/sources/new-document-created/test-event.mjs
diff --git a/components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs b/components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs
similarity index 51%
rename from components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs
rename to components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs
index e4c60456dfa7d..9b3bbf43ac287 100644
--- a/components/agentset/sources/ingest-job-status-updated/ingest-job-status-updated.mjs
+++ b/components/agentset/sources/new-ingest-job-created/new-ingest-job-created.mjs
@@ -3,9 +3,9 @@ import sampleEmit from "./test-event.mjs";
 
 export default {
   ...common,
-  key: "agentset-ingest-job-status-updated",
-  name: "Ingest Job Status Updated",
-  description: "Emit new event when a ingest job status is updated. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
+  key: "agentset-new-ingest-job-created",
+  name: "New Ingest Job Created",
+  description: "Emit new event when a ingest job is created. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/list)",
   version: "0.0.1",
   type: "source",
   dedupe: "unique",
@@ -15,7 +15,7 @@ export default {
       return this.agentset.listIngestJobs;
     },
     getSummary(item) {
-      return `Ingest Job (${item.id}) has a new status: ${item.status}`;
+      return `New ingest job created (${item.id})`;
     },
   },
   sampleEmit,
diff --git a/components/agentset/sources/ingest-job-status-updated/test-event.mjs b/components/agentset/sources/new-ingest-job-created/test-event.mjs
similarity index 100%
rename from components/agentset/sources/ingest-job-status-updated/test-event.mjs
rename to components/agentset/sources/new-ingest-job-created/test-event.mjs

From 1ec00544674bc1e675aeec71f964dac58312814d Mon Sep 17 00:00:00 2001
From: Luan Cazarine <luanhc@gmail.com>
Date: Mon, 14 Apr 2025 11:10:58 -0300
Subject: [PATCH 8/8] Update
 components/agentset/sources/new-document-created/new-document-created.mjs

Co-authored-by: Leo Vu <vunguyenhung@outlook.com>
---
 .../sources/new-document-created/new-document-created.mjs       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/components/agentset/sources/new-document-created/new-document-created.mjs b/components/agentset/sources/new-document-created/new-document-created.mjs
index 9261f8fc19bea..88a7ce2ae02a8 100644
--- a/components/agentset/sources/new-document-created/new-document-created.mjs
+++ b/components/agentset/sources/new-document-created/new-document-created.mjs
@@ -15,7 +15,7 @@ export default {
       return this.agentset.listDocuments;
     },
     getSummary(item) {
-      return `New document cerated: ${item.name || item.id}`;
+      return `New document created: ${item.name || item.id}`;
     },
   },
   sampleEmit,