Skip to content

Commit c351a0e

Browse files
ewelsewels
committed
Added new 'datalink-poll' node, triggers when new files are found
1 parent 25c9cb1 commit c351a0e

File tree

8 files changed

+474
-152
lines changed

8 files changed

+474
-152
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Changelog
22

3+
## [0.2.0] - 2025-05-18
4+
5+
- Added new nodes for listing files in Data Explorer:
6+
- `seqera-datalink-list`
7+
- `seqera-datalink-poll`
8+
39
## [0.1.2] - 2025-05-17
410

511
- Added GitHub action to publish to npm from GitHub releases automatically

‎README.md‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ Gives new Node-RED node types for your automation workflows, which are designed
1919
- [Create Dataset](#create-dataset)
2020
- [Launch and Monitor a Run](#launch-and-monitor-a-run)
2121
- [List Files from Data Explorer](#list-data-link-files)
22+
- [Poll Data Link Files](#poll-data-link-files)
2223

2324
Also [Launch](#launch) and [Workflow](#workflow) nodes for more custom workflows where polling workflow status is not required and it's helpful to have full control.
2425

@@ -146,8 +147,25 @@ Lists files and folders from a Seqera Platform **Data Explorer** link.
146147

147148
- `msg.payload` (array): Array of objects returned by the API after filtering.
148149
- `msg.files` (array): Convenience array containing only the file names.
149-
- `msg._seqera_request`: Details of the API request (useful for debugging).
150-
- `msg._seqera_error`: Error details if the request fails.
150+
151+
## Poll Data Link Files
152+
153+
Like **List Data Link Files**, but runs automatically on a timer so that you can trigger downstream automation whenever new data appears.
154+
155+
This node has **no inputs** – it starts polling as soon as the Node-RED flow is deployed.
156+
157+
### Inputs (typed-input fields)
158+
159+
Same as _List Data Link Files_, plus:
160+
161+
- **pollFrequency** (number): How often to poll, expressed in **minutes** (default: 15).
162+
163+
### Outputs (two outputs)
164+
165+
1. **All results** – Fired every poll with the full list returned from the API.
166+
2. **New results** – Fired only when at least one object is detected that wasn't present in the previous poll (will not send anything if there are no new objects).
167+
168+
Each message contains the same properties as _List Data Link Files_ (`payload`, `files`).
151169

152170
## Launch
153171

‎nodes/datalink-list.html‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666
1. Standard output
6767
: payload (array) : Array of file objects aggregated from the API.
6868
: files (array) : Array of file names.
69-
: _seqera_request (object) : Details of the final API request (for debugging).
7069

7170
### Details
7271

‎nodes/datalink-list.js‎

Lines changed: 5 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ module.exports = function (RED) {
3030
node.defaultBaseUrl = (node.seqeraConfig && node.seqeraConfig.baseUrl) || "https://api.cloud.seqera.io";
3131
node.credentials = RED.nodes.getCredentials(node.id);
3232

33-
const axios = require("axios");
33+
const datalinkUtils = require("./datalink-utils");
3434

3535
// Helper to format date as yyyy-mm-dd HH:MM:SS
3636
const formatDateTime = () => {
@@ -42,153 +42,11 @@ module.exports = function (RED) {
4242
node.on("input", async function (msg, send, done) {
4343
node.status({ fill: "blue", shape: "ring", text: `listing: ${formatDateTime()}` });
4444

45-
// Helper to evaluate properties (supports jsonata)
46-
const evalProp = async (p, t) => {
47-
if (t === "jsonata") {
48-
const expr = RED.util.prepareJSONataExpression(p, node);
49-
return await new Promise((resolve, reject) => {
50-
RED.util.evaluateJSONataExpression(expr, msg, (err, value) => {
51-
if (err) return reject(err);
52-
resolve(value);
53-
});
54-
});
55-
}
56-
return RED.util.evaluateNodeProperty(p, t, node, msg);
57-
};
58-
5945
try {
60-
// Evaluate all properties
61-
const dataLinkName = await evalProp(node.dataLinkNameProp, node.dataLinkNamePropType);
62-
const basePathRaw = await evalProp(node.basePathProp, node.basePathPropType);
63-
const prefix = await evalProp(node.prefixProp, node.prefixPropType);
64-
const patternRaw = await evalProp(node.patternProp, node.patternPropType);
65-
const maxResultsRaw = await evalProp(node.maxResultsProp, node.maxResultsPropType);
66-
const workspaceIdOverride = await evalProp(node.workspaceIdProp, node.workspaceIdPropType);
67-
const baseUrlOverride = await evalProp(node.baseUrlProp, node.baseUrlPropType);
68-
const tokenOverride = await evalProp(node.tokenProp, node.tokenPropType);
69-
const depthRaw = await evalProp(node.depthProp, node.depthPropType);
70-
const depthVal = Math.max(0, parseInt(depthRaw, 10) || 0);
71-
72-
const baseUrl = baseUrlOverride || (node.seqeraConfig && node.seqeraConfig.baseUrl) || node.defaultBaseUrl;
73-
const workspaceId = workspaceIdOverride || (node.seqeraConfig && node.seqeraConfig.workspaceId) || null;
74-
const maxResults = parseInt(maxResultsRaw, 10) || 100;
75-
const basePath = basePathRaw != null ? String(basePathRaw) : "";
76-
77-
if (!dataLinkName) throw new Error("dataLinkName not provided");
78-
79-
// Helper to build auth headers
80-
const buildHeaders = () => {
81-
const headers = { Accept: "application/json" };
82-
const token =
83-
tokenOverride ||
84-
(node.seqeraConfig && node.seqeraConfig.credentials && node.seqeraConfig.credentials.token) ||
85-
(node.credentials && node.credentials.token);
86-
if (token) headers["Authorization"] = `Bearer ${token}`;
87-
return headers;
88-
};
89-
90-
// 1) Resolve dataLinkId and credentialsId by name search
91-
let dataLinkId;
92-
let credentialsId;
93-
{
94-
const qs = new URLSearchParams();
95-
if (workspaceId != null) qs.append("workspaceId", workspaceId);
96-
qs.append("pageSize", "2");
97-
qs.append("search", dataLinkName);
98-
const searchUrl = `${baseUrl.replace(/\/$/, "")}/data-links/?${qs.toString()}`;
99-
msg._seqera_datalink_search_request = { method: "GET", url: searchUrl, headers: buildHeaders() };
100-
const resp = await axios.get(searchUrl, { headers: buildHeaders() });
101-
const links = resp.data?.dataLinks || [];
102-
if (links.length == 0) {
103-
throw new Error(`Could not find Data Link '${dataLinkName}'`);
104-
}
105-
if (links.length !== 1) {
106-
throw new Error(`Found more than one Data Link matching '${dataLinkName}'`);
107-
}
108-
dataLinkId = links[0].id;
109-
if (!dataLinkId) throw new Error("Matched data link does not have an id");
110-
if (Array.isArray(links[0].credentials) && links[0].credentials.length) {
111-
credentialsId = links[0].credentials[0].id;
112-
}
113-
}
114-
115-
// Recursive fetch respecting depth
116-
const allItems = [];
117-
118-
const fetchPath = async (currentPath, currentDepth) => {
119-
if (allItems.length >= maxResults) return;
120-
121-
let nextPage = null;
122-
do {
123-
const encodedPath = currentPath
124-
.split("/")
125-
.filter((p) => p !== "")
126-
.map(encodeURIComponent)
127-
.join("/");
128-
129-
let url = `${baseUrl.replace(/\/$/, "")}/data-links/${encodeURIComponent(
130-
dataLinkId,
131-
)}/browse/${encodedPath}`;
132-
133-
const qs = new URLSearchParams();
134-
if (workspaceId != null) qs.append("workspaceId", workspaceId);
135-
if (prefix != null && prefix !== "") qs.append("search", prefix);
136-
if (credentialsId) qs.append("credentialsId", credentialsId);
137-
if (nextPage) qs.append("nextPageToken", nextPage);
138-
const qsStr = qs.toString();
139-
if (qsStr.length) url += `?${qsStr}`;
140-
141-
const resp = await axios.get(url, { headers: buildHeaders() });
142-
const data = resp.data || {};
143-
const objects = Array.isArray(data.objects) ? data.objects : [];
144-
145-
if (objects.length) {
146-
const mapped = objects.map((o) => {
147-
const prefix = currentPath ? `${currentPath}/` : "";
148-
return { ...o, name: `${prefix}${o.name}` };
149-
});
150-
const remaining = maxResults - allItems.length;
151-
allItems.push(...mapped.slice(0, remaining));
152-
}
153-
154-
// Recurse into folders if depth allows
155-
if (currentDepth < depthVal && allItems.length < maxResults) {
156-
const folders = objects.filter((o) => (o.type || "").toUpperCase() === "FOLDER");
157-
for (const folder of folders) {
158-
if (allItems.length >= maxResults) break;
159-
const folderNameClean = folder.name.replace(/\/$/, "");
160-
const newPath = currentPath ? `${currentPath}/${folderNameClean}` : folderNameClean;
161-
await fetchPath(newPath, currentDepth + 1);
162-
}
163-
}
164-
165-
nextPage = data.nextPageToken || data.nextPage || null;
166-
} while (nextPage && allItems.length < maxResults);
167-
};
168-
169-
await fetchPath(basePath, 0);
170-
171-
// Apply regex pattern filter if provided
172-
let finalItems = allItems;
173-
if (patternRaw && patternRaw !== "") {
174-
try {
175-
const regex = new RegExp(patternRaw);
176-
finalItems = allItems.filter((it) => regex.test(it.name));
177-
} catch (e) {
178-
node.warn(`Invalid regex pattern: ${patternRaw}`);
179-
}
180-
}
181-
182-
// Filter by returnType option
183-
if (node.returnType === "files") {
184-
finalItems = finalItems.filter((it) => (it.type || "").toUpperCase() === "FILE");
185-
} else if (node.returnType === "folders") {
186-
finalItems = finalItems.filter((it) => (it.type || "").toUpperCase() === "FOLDER");
187-
}
188-
189-
msg.payload = finalItems;
190-
msg.files = finalItems.map((f) => f.name);
191-
node.status({ fill: "green", shape: "dot", text: `done: ${finalItems.length} items` });
46+
const result = await datalinkUtils.listDataLink(RED, node, msg);
47+
msg.payload = result.items;
48+
msg.files = result.files;
49+
node.status({ fill: "green", shape: "dot", text: `${result.items.length} items: ${formatDateTime()}` });
19250
send(msg);
19351
if (done) done();
19452
} catch (err) {

‎nodes/datalink-poll.html‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
<script type="text/html" data-template-name="seqera-datalink-poll">
2+
<!-- Seqera config and node name -->
3+
<div class="form-row">
4+
<label for="node-input-seqera"><i class="icon-globe"></i> Seqera config</label>
5+
<input type="text" id="node-input-seqera" data-type="seqera-config" />
6+
</div>
7+
<div class="form-row">
8+
<label for="node-input-name"><i class="fa fa-tag"></i> Name</label>
9+
<input type="text" id="node-input-name" />
10+
</div>
11+
12+
<!-- Polling specific -->
13+
<div class="form-row">
14+
<label for="node-input-pollFrequency"><i class="fa fa-clock-o"></i> Poll frequency (min)</label>
15+
<input type="text" id="node-input-pollFrequency" />
16+
</div>
17+
18+
<!-- Data link parameters (same as list node) -->
19+
<div class="form-row">
20+
<label for="node-input-dataLinkName"><i class="fa fa-link"></i> Data link name</label>
21+
<input type="text" id="node-input-dataLinkName" />
22+
</div>
23+
<div class="form-row">
24+
<label for="node-input-returnType"><i class="fa fa-filter"></i> Return type</label>
25+
<select id="node-input-returnType">
26+
<option value="files">Files only</option>
27+
<option value="folders">Folders only</option>
28+
<option value="all">Everything</option>
29+
</select>
30+
</div>
31+
<div class="form-row">
32+
<label for="node-input-basePath"><i class="fa fa-folder-open-o"></i> Base path</label>
33+
<input type="text" id="node-input-basePath" />
34+
</div>
35+
<div class="form-row">
36+
<label for="node-input-prefix"><i class="fa fa-filter"></i> Prefix</label>
37+
<input type="text" id="node-input-prefix" />
38+
</div>
39+
<div class="form-row">
40+
<label for="node-input-pattern"><i class="fa fa-file-code-o"></i> Pattern (regex)</label>
41+
<input type="text" id="node-input-pattern" />
42+
</div>
43+
<div class="form-row">
44+
<label for="node-input-maxResults"><i class="fa fa-sort-numeric-asc"></i> Max results</label>
45+
<input type="text" id="node-input-maxResults" />
46+
</div>
47+
<div class="form-row">
48+
<label for="node-input-depth"><i class="fa fa-level-down"></i> Depth</label>
49+
<input type="text" id="node-input-depth" />
50+
</div>
51+
<div class="form-row">
52+
<label for="node-input-workspaceId"><i class="icon-tasks"></i> Workspace ID</label>
53+
<input type="text" id="node-input-workspaceId" />
54+
</div>
55+
</script>
56+
57+
<!-- prettier-ignore -->
58+
<script type="text/markdown" data-help-name="seqera-datalink-poll">
59+
Polls a Seqera Platform Data Explorer link at a fixed interval.
60+
61+
### Inputs
62+
63+
: pollFrequency (number) : The frequency of the poll in minutes (integer, minimum 1, default 15).
64+
: dataLinkName (string) : The name of the data explorer link.
65+
: basePath (string) : Path within the data link to start browsing. Leave blank for the root.
66+
: prefix (string) : Optional prefix filter for results (applies to folders and files)
67+
: pattern (string) : Optional regex pattern filter for results (applies to files only)
68+
: returnType (string) : Select whether to return files, folders or everything.
69+
: maxResults (number) : Maximum number of results to return (default 100).
70+
: workspaceId (string) : Override the workspace ID from the config node.
71+
72+
All inputs support msg._, flow._, global.\*, env, or JSONata expressions via the **typedInput**.
73+
74+
### Outputs
75+
76+
The node has two outputs:
77+
78+
1. All results on every poll.
79+
2. New objects since the previous poll (nothing sent if no new objects).
80+
81+
Both outputs have the following properties:
82+
83+
: payload (array) : Fle information aggregated from the API (array of objects).
84+
: files (array) : File names (array of strings).
85+
86+
All typed-input fields are identical to the _List files_ node with the addition of **poll frequency** (minutes).
87+
</script>
88+
89+
<script type="text/javascript">
90+
RED.nodes.registerType("seqera-datalink-poll", {
91+
category: "seqera",
92+
color: "#A9A1C6",
93+
inputs: 0,
94+
outputs: 2,
95+
icon: "seqera.svg",
96+
align: "left",
97+
paletteLabel: "Poll files",
98+
label: function () {
99+
return this.name || "Poll files";
100+
},
101+
outputLabels: ["All objects", "Only new objects"],
102+
defaults: {
103+
name: { value: "" },
104+
seqera: { value: "", type: "seqera-config" },
105+
dataLinkName: { value: "" },
106+
dataLinkNameType: { value: "str" },
107+
basePath: { value: "" },
108+
basePathType: { value: "str" },
109+
prefix: { value: "" },
110+
prefixType: { value: "str" },
111+
pattern: { value: "" },
112+
patternType: { value: "str" },
113+
maxResults: { value: "100" },
114+
maxResultsType: { value: "num" },
115+
depth: { value: "0" },
116+
depthType: { value: "num" },
117+
workspaceId: { value: "" },
118+
workspaceIdType: { value: "str" },
119+
pollFrequency: { value: "15" },
120+
returnType: { value: "files" },
121+
},
122+
oneditprepare: function () {
123+
function ti(id, val, type, def = "str") {
124+
const types = ["str", "num", "msg", "flow", "global", "env", "jsonata"];
125+
if (def === "num") types.splice(types.indexOf("str"), 1);
126+
$(id).typedInput({ default: def, types });
127+
$(id).typedInput("value", val);
128+
$(id).typedInput("type", type);
129+
}
130+
ti("#node-input-dataLinkName", this.dataLinkName || "", this.dataLinkNameType || "str");
131+
ti("#node-input-basePath", this.basePath || "", this.basePathType || "str");
132+
ti("#node-input-prefix", this.prefix || "", this.prefixType || "str");
133+
ti("#node-input-pattern", this.pattern || "", this.patternType || "str");
134+
ti("#node-input-maxResults", this.maxResults || "100", this.maxResultsType || "num", "num");
135+
ti("#node-input-depth", this.depth || "0", this.depthType || "num", "num");
136+
ti("#node-input-workspaceId", this.workspaceId || "", this.workspaceIdType || "str");
137+
ti("#node-input-pollFrequency", this.pollFrequency || "15", "num", "num");
138+
139+
$("#node-input-returnType").val(this.returnType || "files");
140+
},
141+
oneditsave: function () {
142+
function save(id, prop, propType) {
143+
this[prop] = $(id).typedInput("value");
144+
this[propType] = $(id).typedInput("type");
145+
}
146+
save.call(this, "#node-input-dataLinkName", "dataLinkName", "dataLinkNameType");
147+
save.call(this, "#node-input-basePath", "basePath", "basePathType");
148+
save.call(this, "#node-input-prefix", "prefix", "prefixType");
149+
save.call(this, "#node-input-pattern", "pattern", "patternType");
150+
save.call(this, "#node-input-maxResults", "maxResults", "maxResultsType");
151+
save.call(this, "#node-input-depth", "depth", "depthType");
152+
save.call(this, "#node-input-workspaceId", "workspaceId", "workspaceIdType");
153+
this.pollFrequency = $("#node-input-pollFrequency").typedInput("value");
154+
this.returnType = $("#node-input-returnType").val();
155+
},
156+
});
157+
</script>

0 commit comments

Comments
 (0)