Skip to content

Commit 7dd5427

Browse files
luancazarineluancazarine
authored
New Components - ragie (#15322)
* ragie init * [Components] ragie #15183 Sources - New Document - New Connection Actions - Create Document - Update Document File * pnpm update * fix props reference names
1 parent cf980fd commit 7dd5427

File tree

12 files changed

+431
-7
lines changed

12 files changed

+431
-7
lines changed

‎components/ragie/actions/create-document/create-document.mjs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import FormData from "form-data";
2+
import fs from "fs";
3+
import { checkTmp } from "../../common/utils.mjs";
4+
import ragie from "../../ragie.app.mjs";
5+
6+
export default {
7+
key: "ragie-create-document",
8+
name: "Create Document",
9+
description: "Creates a new document in Ragie. [See the documentation](https://docs.ragie.ai/reference/createdocument)",
10+
version: "0.0.1",
11+
type: "action",
12+
props: {
13+
ragie,
14+
file: {
15+
propDefinition: [
16+
ragie,
17+
"documentFile",
18+
],
19+
},
20+
mode: {
21+
propDefinition: [
22+
ragie,
23+
"documentMode",
24+
],
25+
optional: true,
26+
},
27+
metadata: {
28+
type: "object",
29+
label: "Metadata",
30+
description: "Metadata for the document. Keys must be strings. Values may be strings, numbers, booleans, or lists of strings.",
31+
optional: true,
32+
},
33+
externalId: {
34+
type: "string",
35+
label: "External ID",
36+
description: "An optional identifier for the document. A common value might be an ID in an external system or the URL where the source file may be found.",
37+
optional: true,
38+
},
39+
name: {
40+
type: "string",
41+
label: "Name",
42+
description: "An optional name for the document. If set, the document will have this name. Otherwise, it will default to the file's name.",
43+
optional: true,
44+
},
45+
partition: {
46+
propDefinition: [
47+
ragie,
48+
"partition",
49+
],
50+
optional: true,
51+
},
52+
},
53+
async run({ $ }) {
54+
const data = new FormData();
55+
data.append("file", fs.createReadStream(checkTmp(this.file)));
56+
if (this.mode) data.append("mode", this.mode);
57+
if (this.metadata) data.append("metadata", JSON.stringify(this.metadata));
58+
if (this.externalId) data.append("external_id", this.externalId);
59+
if (this.name) data.append("name", this.name);
60+
if (this.partition) data.append("partition", this.partition);
61+
62+
const response = await this.ragie.createDocument({
63+
$,
64+
data,
65+
headers: data.getHeaders(),
66+
});
67+
68+
$.export("$summary", `Created document: ${response.name} (ID: ${response.id})`);
69+
return response;
70+
},
71+
};

‎components/ragie/actions/update-document-file/update-document-file.mjs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import FormData from "form-data";
2+
import fs from "fs";
3+
import { checkTmp } from "../../common/utils.mjs";
4+
import ragie from "../../ragie.app.mjs";
5+
6+
export default {
7+
key: "ragie-update-document-file",
8+
name: "Update Document File",
9+
description: "Updates an existing document file in Ragie. [See the documentation](https://docs.ragie.ai/reference/updatedocumentfile).",
10+
version: "0.0.1",
11+
type: "action",
12+
props: {
13+
ragie,
14+
documentId: {
15+
propDefinition: [
16+
ragie,
17+
"documentId",
18+
],
19+
},
20+
mode: {
21+
propDefinition: [
22+
ragie,
23+
"documentMode",
24+
],
25+
optional: true,
26+
},
27+
file: {
28+
propDefinition: [
29+
ragie,
30+
"documentFile",
31+
],
32+
},
33+
},
34+
async run({ $ }) {
35+
const data = new FormData();
36+
data.append("file", fs.createReadStream(checkTmp(this.file)));
37+
if (this.mode) data.append("mode", this.mode);
38+
39+
const response = await this.ragie.updateDocumentFile({
40+
$,
41+
documentId: this.documentId,
42+
data,
43+
headers: data.getHeaders(),
44+
});
45+
$.export("$summary", `Successfully updated document file with ID: ${this.documentId}`);
46+
return response;
47+
},
48+
};

‎components/ragie/common/constants.mjs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
export const SCOPE_OPTIONS = [
2+
{
3+
label: "Document",
4+
value: "document",
5+
},
6+
{
7+
label: "Chunk",
8+
value: "chunk",
9+
},
10+
];
11+
12+
export const DOCUMENT_MODE_OPTIONS = [
13+
"hi_res",
14+
"fast",
15+
];

‎components/ragie/common/utils.mjs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
export const checkTmp = (filename) => {
2+
if (!filename.startsWith("/tmp")) {
3+
return `/tmp/${filename}`;
4+
}
5+
return filename;
6+
};
7+
8+
export const parseObject = (obj) => {
9+
if (!obj) return undefined;
10+
11+
if (Array.isArray(obj)) {
12+
return obj.map((item) => {
13+
if (typeof item === "string") {
14+
try {
15+
return JSON.parse(item);
16+
} catch (e) {
17+
return item;
18+
}
19+
}
20+
return item;
21+
});
22+
}
23+
if (typeof obj === "string") {
24+
try {
25+
return JSON.parse(obj);
26+
} catch (e) {
27+
return obj;
28+
}
29+
}
30+
return obj;
31+
};

‎components/ragie/package.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/ragie",
3-
"version": "0.0.1",
3+
"version": "0.1.0",
44
"description": "Pipedream Ragie Components",
55
"main": "ragie.app.mjs",
66
"keywords": [
@@ -11,5 +11,8 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.0.3"
1417
}
15-
}
18+
}

‎components/ragie/ragie.app.mjs‎

Lines changed: 120 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,127 @@
1+
import { axios } from "@pipedream/platform";
2+
import { DOCUMENT_MODE_OPTIONS } from "./common/constants.mjs";
3+
14
export default {
25
type: "app",
36
app: "ragie",
4-
propDefinitions: {},
7+
propDefinitions: {
8+
documentFile: {
9+
type: "string",
10+
label: "File",
11+
description: "The path to the file in the `/tmp` directory. [See the documentation on working with files](https://pipedream.com/docs/code/nodejs/working-with-files/#the-tmp-directory).",
12+
},
13+
documentMode: {
14+
type: "string",
15+
label: "Mode",
16+
description: "Partition strategy for the document. Options are 'hi_res' or 'fast'.",
17+
optional: true,
18+
options: DOCUMENT_MODE_OPTIONS,
19+
},
20+
partition: {
21+
type: "string",
22+
label: "Partition",
23+
description: "An optional partition identifier. Partitions must be lowercase alphanumeric and may only include the special characters '_' and '-'.",
24+
},
25+
documentId: {
26+
type: "string",
27+
label: "Document ID",
28+
description: "The ID of the document to update.",
29+
async options({ prevContext }) {
30+
const {
31+
documents, pagination,
32+
} = await this.listDocuments({
33+
params: {
34+
cursor: prevContext.cursor,
35+
},
36+
});
37+
return {
38+
options: documents.map(({
39+
id: value, name: label,
40+
}) => ({
41+
label,
42+
value,
43+
})),
44+
context: {
45+
cursor: pagination.next_cursor,
46+
},
47+
};
48+
},
49+
},
50+
},
551
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
52+
_baseUrl() {
53+
return "https://api.ragie.ai";
54+
},
55+
_headers(headers = {}) {
56+
return {
57+
...headers,
58+
"Authorization": `Bearer ${this.$auth.api_key}`,
59+
};
60+
},
61+
_makeRequest({
62+
$ = this, path, headers, ...opts
63+
}) {
64+
return axios($, {
65+
url: this._baseUrl() + path,
66+
headers: this._headers(headers),
67+
...opts,
68+
});
69+
},
70+
createDocument(opts = {}) {
71+
return this._makeRequest({
72+
method: "POST",
73+
path: "/documents",
74+
...opts,
75+
});
76+
},
77+
listDocuments(opts = {}) {
78+
return this._makeRequest({
79+
path: "/documents",
80+
params: opts,
81+
});
82+
},
83+
updateDocumentFile({
84+
documentId, ...opts
85+
}) {
86+
return this._makeRequest({
87+
method: "PUT",
88+
path: `/documents/${documentId}/file`,
89+
...opts,
90+
});
91+
},
92+
listConnections(opts = {}) {
93+
return this._makeRequest({
94+
path: "/connections",
95+
params: opts,
96+
});
97+
},
98+
async *paginate({
99+
fn, params = {}, fieldName, maxResults = null, ...opts
100+
}) {
101+
let count = 0;
102+
let nextCursor;
103+
104+
do {
105+
params.cursor = nextCursor;
106+
const {
107+
pagination,
108+
...data
109+
} = await fn({
110+
params,
111+
...opts,
112+
});
113+
const items = data[fieldName];
114+
115+
for (const d of items) {
116+
yield d;
117+
118+
if (maxResults && ++count === maxResults) {
119+
return count;
120+
}
121+
}
122+
123+
nextCursor = pagination?.next_cursor;
124+
} while (nextCursor);
9125
},
10126
},
11127
};

‎components/ragie/sources/common/base.mjs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
2+
import ragie from "../../ragie.app.mjs";
3+
4+
export default {
5+
props: {
6+
ragie,
7+
db: "$.service.db",
8+
timer: {
9+
type: "$.interface.timer",
10+
default: {
11+
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
12+
},
13+
},
14+
},
15+
methods: {
16+
_getLastDate() {
17+
return this.db.get("lastDate") || 0;
18+
},
19+
_setLastDate(lastDate) {
20+
this.db.set("lastDate", lastDate);
21+
},
22+
async emitEvent(maxResults = false) {
23+
const lastDate = this._getLastDate();
24+
25+
const response = this.ragie.paginate({
26+
fn: this.getFunction(),
27+
fieldName: this.getFieldName(),
28+
});
29+
30+
let responseArray = [];
31+
for await (const item of response) {
32+
if (Date.parse(item.created_at) <= lastDate) break;
33+
responseArray.push(item);
34+
}
35+
36+
if (responseArray.length) {
37+
if (maxResults && (responseArray.length > maxResults)) {
38+
responseArray.length = maxResults;
39+
}
40+
this._setLastDate(Date.parse(responseArray[0].created_at));
41+
}
42+
43+
for (const item of responseArray.reverse()) {
44+
this.$emit(item, {
45+
id: item.id,
46+
summary: this.getSummary(item),
47+
ts: Date.parse(item.created_at || new Date()),
48+
});
49+
}
50+
},
51+
},
52+
hooks: {
53+
async deploy() {
54+
await this.emitEvent(25);
55+
},
56+
},
57+
async run() {
58+
await this.emitEvent();
59+
},
60+
};

0 commit comments

Comments
 (0)