Skip to content

Commit 943fec5

Browse files
luancazarinevunguyenhung
luancazarine
and
vunguyenhung
authored
New Components - agentset (#16232)
* agentset init * [Components] agentset #16193 Sources - New Ingest Job - New Document Actions - Create Namespace - Create Ingest Job - Search Namespace * pnpm update * pnpm update * fix action name * some adjusts * some adjusts * Update components/agentset/sources/new-document-created/new-document-created.mjs Co-authored-by: Leo Vu <[email protected]> --------- Co-authored-by: Leo Vu <[email protected]>
1 parent 241a764 commit 943fec5

File tree

13 files changed

+553
-7
lines changed

13 files changed

+553
-7
lines changed

‎components/agentset/actions/create-ingest-job/create-ingest-job.mjs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import agentset from "../../agentset.app.mjs";
2+
import { PAYLOAD_TYPE_OPTIONS } from "../../common/constants.mjs";
3+
import { parseObject } from "../../common/utils.mjs";
4+
5+
export default {
6+
key: "agentset-create-ingest-job",
7+
name: "Create Ingest Job",
8+
description: "Create an ingest job for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/ingest-jobs/create)",
9+
version: "0.0.1",
10+
type: "action",
11+
props: {
12+
agentset,
13+
namespaceId: {
14+
propDefinition: [
15+
agentset,
16+
"namespaceId",
17+
],
18+
},
19+
payloadType: {
20+
type: "string",
21+
label: "Payload Type",
22+
description: "Type of payload for the ingest job",
23+
options: PAYLOAD_TYPE_OPTIONS,
24+
reloadProps: true,
25+
},
26+
text: {
27+
type: "string",
28+
label: "Text",
29+
description: "The text to ingest",
30+
hidden: true,
31+
},
32+
fileUrl: {
33+
type: "string",
34+
label: "File URL",
35+
description: "The URL of the file to ingest",
36+
hidden: true,
37+
},
38+
urls: {
39+
type: "string[]",
40+
label: "URLs",
41+
description: "The URLs to ingest",
42+
hidden: true,
43+
},
44+
name: {
45+
type: "string",
46+
label: "Name",
47+
description: "The name of the ingest job",
48+
optional: true,
49+
hidden: true,
50+
},
51+
},
52+
async additionalProps(props) {
53+
props.text.hidden = true;
54+
props.name.hidden = true;
55+
props.fileUrl.hidden = true;
56+
props.urls.hidden = true;
57+
58+
switch (this.payloadType) {
59+
case "TEXT":
60+
props.text.hidden = false;
61+
props.name.hidden = false;
62+
break;
63+
case "FILE":
64+
props.fileUrl.hidden = false;
65+
props.name.hidden = false;
66+
break;
67+
case "URLS":
68+
props.urls.hidden = false;
69+
break;
70+
}
71+
return {};
72+
},
73+
async run({ $ }) {
74+
const payload = {
75+
type: this.payloadType,
76+
};
77+
switch (this.payloadType) {
78+
case "TEXT":
79+
payload.text = this.text;
80+
payload.name = this.name;
81+
break;
82+
case "FILE":
83+
payload.fileUrl = this.fileUrl;
84+
payload.name = this.name;
85+
break;
86+
case "URLS":
87+
payload.urls = parseObject(this.urls);
88+
break;
89+
}
90+
const response = await this.agentset.createIngestJob({
91+
$,
92+
namespaceId: this.namespaceId,
93+
data: {
94+
payload,
95+
},
96+
});
97+
$.export("$summary", `Ingest job created successfully: ID ${response.data.id}`);
98+
return response;
99+
},
100+
};

‎components/agentset/actions/create-namespace/create-namespace.mjs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import agentset from "../../agentset.app.mjs";
2+
import { slugify } from "../../common/utils.mjs";
3+
4+
export default {
5+
key: "agentset-create-namespace",
6+
name: "Create Namespace",
7+
description: "Creates a namespace for the authenticated organization. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/namespaces/create)",
8+
version: "0.0.1",
9+
type: "action",
10+
props: {
11+
agentset,
12+
name: {
13+
type: "string",
14+
label: "Name",
15+
description: "The name of the namespace to create",
16+
},
17+
},
18+
async run({ $ }) {
19+
const response = await this.agentset.createNamespace({
20+
$,
21+
data: {
22+
name: this.name,
23+
slug: slugify(this.name),
24+
},
25+
});
26+
27+
$.export("$summary", `Successfully created namespace with ID: ${response.data.id}`);
28+
return response;
29+
},
30+
};

‎components/agentset/actions/search-namespace/search-namespace.mjs‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import agentset from "../../agentset.app.mjs";
2+
3+
export default {
4+
key: "agentset-search-namespace",
5+
name: "Agentset Search Namespace",
6+
description: "Complete retrieval pipeline for RAG with semantic search, filtering, and reranking. [See the documentation](https://docs.agentset.ai/api-reference/endpoint/search)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
agentset,
11+
namespaceId: {
12+
propDefinition: [
13+
agentset,
14+
"namespaceId",
15+
],
16+
},
17+
query: {
18+
type: "string",
19+
label: "Query",
20+
description: "The query for semantic search",
21+
},
22+
topK: {
23+
type: "integer",
24+
label: "Top K",
25+
description: "Number of top documents to return",
26+
min: 1,
27+
max: 100,
28+
optional: true,
29+
},
30+
rerank: {
31+
type: "boolean",
32+
label: "Rerank",
33+
description: "Whether to rerank the results",
34+
optional: true,
35+
},
36+
rerankLimit: {
37+
type: "integer",
38+
label: "Rerank Limit",
39+
description: "The number of results to return after reranking",
40+
min: 1,
41+
max: 100,
42+
optional: true,
43+
},
44+
filter: {
45+
type: "object",
46+
label: "Filter",
47+
description: "Filter to apply to search results",
48+
optional: true,
49+
},
50+
minScore: {
51+
type: "string",
52+
label: "Minimum Score",
53+
description: "Minimum score to return. Range from 0 to 1",
54+
optional: true,
55+
},
56+
includeRelationship: {
57+
type: "boolean",
58+
label: "Include Relationship",
59+
description: "Whether to include relationships in the results",
60+
optional: true,
61+
},
62+
includeMetadata: {
63+
type: "boolean",
64+
label: "Include Metadata",
65+
description: "Whether to include metadata in the results",
66+
optional: true,
67+
},
68+
},
69+
70+
async run({ $ }) {
71+
const response = await this.agentset.searchNamespace({
72+
$,
73+
namespaceId: this.namespaceId,
74+
data: {
75+
query: this.query,
76+
topK: this.topK,
77+
rerank: this.rerank,
78+
rerankLimit: this.rerankLimit,
79+
filter: this.filter,
80+
minScore: this.minScore && parseFloat(this.minScore),
81+
includeRelationships: this.includeRelationship,
82+
includeMetadata: this.includeMetadata,
83+
},
84+
});
85+
86+
$.export("$summary", `Successfully completed the search for query: "${this.query}"`);
87+
return response;
88+
},
89+
};

‎components/agentset/agentset.app.mjs‎

Lines changed: 116 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,123 @@
1+
import { axios } from "@pipedream/platform";
2+
import { STATUSES_OPTIONS } from "./common/constants.mjs";
3+
14
export default {
25
type: "app",
36
app: "agentset",
4-
propDefinitions: {},
7+
propDefinitions: {
8+
namespaceId: {
9+
type: "string",
10+
label: "Namespace ID",
11+
description: "The ID of the namespace",
12+
async options() {
13+
const { data } = await this.listNamespaces();
14+
return data.map(({
15+
id: value, name: label,
16+
}) => ({
17+
label,
18+
value,
19+
}));
20+
},
21+
},
22+
statuses: {
23+
type: "string[]",
24+
label: "Statuses",
25+
description: "Filter by status",
26+
options: STATUSES_OPTIONS,
27+
},
28+
},
529
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
30+
_baseUrl() {
31+
return "https://api.agentset.ai/v1";
32+
},
33+
_headers() {
34+
return {
35+
Authorization: `Bearer ${this.$auth.api_key}`,
36+
};
37+
},
38+
_makeRequest({
39+
$ = this, path, ...opts
40+
}) {
41+
return axios($, {
42+
url: this._baseUrl() + path,
43+
headers: this._headers(),
44+
...opts,
45+
});
46+
},
47+
listNamespaces(opts = {}) {
48+
return this._makeRequest({
49+
path: "/namespace",
50+
...opts,
51+
});
52+
},
53+
createNamespace(opts = {}) {
54+
return this._makeRequest({
55+
method: "POST",
56+
path: "/namespace",
57+
...opts,
58+
});
59+
},
60+
createIngestJob({
61+
namespaceId, ...opts
62+
}) {
63+
return this._makeRequest({
64+
method: "POST",
65+
path: `/namespace/${namespaceId}/ingest-jobs`,
66+
...opts,
67+
});
68+
},
69+
listIngestJobs({
70+
namespaceId, ...opts
71+
}) {
72+
return this._makeRequest({
73+
path: `/namespace/${namespaceId}/ingest-jobs`,
74+
...opts,
75+
});
76+
},
77+
listDocuments({
78+
namespaceId, ...opts
79+
}) {
80+
return this._makeRequest({
81+
path: `/namespace/${namespaceId}/documents`,
82+
...opts,
83+
});
84+
},
85+
searchNamespace({
86+
namespaceId, ...opts
87+
}) {
88+
return this._makeRequest({
89+
method: "POST",
90+
path: `/namespace/${namespaceId}/search`,
91+
...opts,
92+
});
93+
},
94+
async *paginate({
95+
fn, params = {}, maxResults = null, ...opts
96+
}) {
97+
let count = 0;
98+
let cursor;
99+
100+
do {
101+
params.cursor = cursor;
102+
const {
103+
data,
104+
pagination: { nextCursor },
105+
} = await fn({
106+
params,
107+
...opts,
108+
});
109+
for (const d of data) {
110+
yield d;
111+
112+
if (maxResults && ++count === maxResults) {
113+
return count;
114+
}
115+
}
116+
117+
cursor = nextCursor;
118+
119+
} while (cursor);
9120
},
10121
},
11122
};
123+

‎components/agentset/common/constants.mjs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
export const PAYLOAD_TYPE_OPTIONS = [
2+
"TEXT",
3+
"FILE",
4+
"URLS",
5+
];
6+
7+
export const STATUSES_OPTIONS = [
8+
"BACKLOG",
9+
"QUEUED",
10+
"QUEUED_FOR_RESYNC",
11+
"QUEUED_FOR_DELETE",
12+
"PRE_PROCESSING",
13+
"PROCESSING",
14+
"DELETING",
15+
"CANCELLING",
16+
"COMPLETED",
17+
"FAILED",
18+
"CANCELLED",
19+
];

0 commit comments

Comments
 (0)