Skip to content

Commit 3d233f7

Browse files
jcortesjcortes
authored
Merging pull request #18158 
1 parent e45fc90 commit 3d233f7

File tree

7 files changed

+364
-6
lines changed

7 files changed

+364
-6
lines changed

‎components/nasdaq_data_link_time_series_and_table_data_/actions/export-table/export-table.mjs‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
2+
import utils from "../../common/utils.mjs";
3+
4+
export default {
5+
key: "nasdaq_data_link_time_series_and_table_data_-export-table",
6+
name: "Export Table (Bulk Download)",
7+
description: "Exports an entire table or a filtered subset as a zipped CSV file. Returns a download link for the data. Premium subscribers can use this feature up to 60 times per hour. [See the documentation](https://docs.data.nasdaq.com/docs/large-table-download)",
8+
version: "0.0.1",
9+
type: "action",
10+
props: {
11+
app,
12+
publisher: {
13+
propDefinition: [
14+
app,
15+
"publisher",
16+
],
17+
},
18+
table: {
19+
propDefinition: [
20+
app,
21+
"table",
22+
],
23+
},
24+
columns: {
25+
propDefinition: [
26+
app,
27+
"columns",
28+
({
29+
publisher, table,
30+
}) => ({
31+
publisher,
32+
table,
33+
}),
34+
],
35+
},
36+
filters: {
37+
type: "object",
38+
label: "Row Filters",
39+
description: "Filter rows based on column values. Use column names as keys and values to filter by. For example: `{ \"ticker\": \"SPY\", \"date\": \"2024-01-01\" }`. Only filterable columns can be used (check table metadata).",
40+
optional: true,
41+
},
42+
filterOperators: {
43+
type: "object",
44+
label: "Filter Operators",
45+
description: "Apply operators to filters. Format: `{ \"column.operator\": \"value\" }`. Available operators: `.gt` (greater than), `.lt` (less than), `.gte` (greater than or equal), `.lte` (less than or equal). Example: `{ \"date.gte\": \"2024-01-01\", \"date.lte\": \"2024-12-31\" }`",
46+
optional: true,
47+
},
48+
},
49+
async run({ $ }) {
50+
const {
51+
app,
52+
publisher,
53+
table,
54+
columns,
55+
filters,
56+
filterOperators,
57+
} = this;
58+
59+
const response = await app.tableData({
60+
$,
61+
publisher,
62+
table,
63+
params: {
64+
"qopts.export": true,
65+
...utils.parseJson(filters),
66+
...utils.parseJson(filterOperators),
67+
...(Array.isArray(columns) && columns?.length
68+
? {
69+
"qopts.columns": columns.join(","),
70+
}
71+
: undefined
72+
),
73+
},
74+
});
75+
76+
const status = response?.datatable_bulk_download?.file?.status;
77+
const link = response?.datatable_bulk_download?.file?.link;
78+
79+
if (status === "fresh" && link) {
80+
$.export("$summary", `Table ${publisher}/${table} is ready for download. The download link is valid for 30 minutes.`);
81+
82+
} else if (status === "creating" || status === "regenerating") {
83+
$.export("$summary", `Export job for table ${publisher}/${table} is ${status}. Please retry in a few moments to get the download link.`);
84+
85+
} else {
86+
$.export("$summary", `Export initiated for table ${publisher}/${table}`);
87+
}
88+
89+
return response;
90+
},
91+
};

‎components/nasdaq_data_link_time_series_and_table_data_/actions/get-table-data/get-table-data.mjs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
2+
import utils from "../../common/utils.mjs";
3+
4+
export default {
5+
key: "nasdaq_data_link_time_series_and_table_data_-get-table-data",
6+
name: "Get Table Data",
7+
description: "Retrieves data from a specific Nasdaq Data Link table with automatic pagination. Supports filtering by columns and rows. [See the documentation](https://docs.data.nasdaq.com/docs/tables-1)",
8+
version: "0.0.1",
9+
type: "action",
10+
props: {
11+
app,
12+
publisher: {
13+
propDefinition: [
14+
app,
15+
"publisher",
16+
],
17+
},
18+
table: {
19+
propDefinition: [
20+
app,
21+
"table",
22+
],
23+
},
24+
columns: {
25+
propDefinition: [
26+
app,
27+
"columns",
28+
({
29+
publisher, table,
30+
}) => ({
31+
publisher,
32+
table,
33+
}),
34+
],
35+
},
36+
filters: {
37+
type: "object",
38+
label: "Row Filters",
39+
description: "Filter rows based on column values. Use column names as keys and values to filter by. For example: `{ \"ticker\": \"SPY\", \"date\": \"2024-01-01\" }`. Only filterable columns can be used (check table metadata).",
40+
optional: true,
41+
},
42+
filterOperators: {
43+
type: "object",
44+
label: "Filter Operators",
45+
description: "Apply operators to filters. Format: `{ \"column.operator\": \"value\" }`. Available operators: `.gt` (greater than), `.lt` (less than), `.gte` (greater than or equal), `.lte` (less than or equal). Example: `{ \"date.gte\": \"2024-01-01\", \"date.lte\": \"2024-12-31\" }`",
46+
optional: true,
47+
},
48+
},
49+
async run({ $ }) {
50+
const {
51+
app,
52+
publisher,
53+
table,
54+
columns,
55+
filters,
56+
filterOperators,
57+
} = this;
58+
59+
const response = await app.paginate({
60+
fn: app.tableData,
61+
args: {
62+
$,
63+
publisher,
64+
table,
65+
params: {
66+
...utils.parseJson(filters),
67+
...utils.parseJson(filterOperators),
68+
...(Array.isArray(columns) && columns?.length
69+
? {
70+
"qopts.columns": columns.join(","),
71+
}
72+
: undefined
73+
),
74+
},
75+
},
76+
});
77+
78+
$.export("$summary", `Successfully retrieved ${response.length} records`);
79+
return response;
80+
},
81+
};

‎components/nasdaq_data_link_time_series_and_table_data_/actions/get-table-metadata/get-table-metadata.mjs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import app from "../../nasdaq_data_link_time_series_and_table_data_.app.mjs";
2+
3+
export default {
4+
key: "nasdaq_data_link_time_series_and_table_data_-get-table-metadata",
5+
name: "Get Table Metadata",
6+
description: "Retrieves metadata for a specific Nasdaq Data Link table, including column names, types, filterable columns, and primary keys. [See the documentation](https://docs.data.nasdaq.com/docs/tables-1)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
app,
11+
publisher: {
12+
propDefinition: [
13+
app,
14+
"publisher",
15+
],
16+
},
17+
table: {
18+
propDefinition: [
19+
app,
20+
"table",
21+
],
22+
},
23+
},
24+
async run({ $ }) {
25+
const {
26+
app,
27+
publisher,
28+
table,
29+
} = this;
30+
31+
const response = await app.tableMetadata({
32+
$,
33+
publisher,
34+
table,
35+
});
36+
37+
$.export("$summary", `Successfully retrieved metadata for table \`${publisher}/${table}\``);
38+
return response;
39+
},
40+
};

‎components/nasdaq_data_link_time_series_and_table_data_/common/utils.mjs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
const parseJson = (input, maxDepth = 100) => {
2+
const seen = new WeakSet();
3+
const parse = (value) => {
4+
if (maxDepth <= 0) {
5+
return value;
6+
}
7+
if (typeof(value) === "string") {
8+
// Only parse if the string looks like a JSON object or array
9+
const trimmed = value.trim();
10+
if (
11+
(trimmed.startsWith("{") && trimmed.endsWith("}")) ||
12+
(trimmed.startsWith("[") && trimmed.endsWith("]"))
13+
) {
14+
try {
15+
return parseJson(JSON.parse(value), maxDepth - 1);
16+
} catch (e) {
17+
return value;
18+
}
19+
}
20+
return value;
21+
} else if (typeof(value) === "object" && value !== null && !Array.isArray(value)) {
22+
if (seen.has(value)) {
23+
return value;
24+
}
25+
seen.add(value);
26+
return Object.entries(value)
27+
.reduce((acc, [
28+
key,
29+
val,
30+
]) => Object.assign(acc, {
31+
[key]: parse(val),
32+
}), {});
33+
} else if (Array.isArray(value)) {
34+
return value.map((item) => parse(item));
35+
}
36+
return value;
37+
};
38+
39+
return parse(input);
40+
};
41+
42+
export default {
43+
parseJson,
44+
};

‎components/nasdaq_data_link_time_series_and_table_data_/nasdaq_data_link_time_series_and_table_data_.app.mjs‎

Lines changed: 99 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,106 @@
1+
import { axios } from "@pipedream/platform";
2+
13
export default {
24
type: "app",
35
app: "nasdaq_data_link_time_series_and_table_data_",
4-
propDefinitions: {},
6+
propDefinitions: {
7+
publisher: {
8+
type: "string",
9+
label: "Publisher Code",
10+
description: "The publisher code (e.g., `MER`, `ETFG`, `AR`, `NDAQ`). This is the first part of the datatable code. If the code is `MER/F1`, then `MER` is the publisher code and `F1` is the table code.",
11+
},
12+
table: {
13+
type: "string",
14+
label: "Table Code",
15+
description: "The table code (e.g., `F1`, `FUND`, `MWCS`, `RTAT10`). This is the second part of the datatable code. If the code is `MER/F1`, then `F1` is the table code.",
16+
},
17+
columns: {
18+
type: "string[]",
19+
label: "Columns",
20+
description: "Request data from specific columns. If you want to query for multiple columns, include the column names as array items",
21+
optional: true,
22+
async options({
23+
publisher, table,
24+
}) {
25+
if (!publisher || !table) {
26+
return [];
27+
}
28+
const { datatable: { columns } } = await this.tableMetadata({
29+
publisher,
30+
table,
31+
});
32+
return columns.map(({ name }) => name);
33+
},
34+
},
35+
},
536
methods: {
6-
// this.$auth contains connected account data
7-
authKeys() {
8-
console.log(Object.keys(this.$auth));
37+
getUrl(path) {
38+
return `https://data.nasdaq.com/api/v3${path}`;
39+
},
40+
getHeaders(headers) {
41+
return {
42+
...headers,
43+
"Accept": "application/json",
44+
"X-Api-Token": this.$auth.api_key,
45+
};
46+
},
47+
makeRequest({
48+
$ = this, path, headers, ...args
49+
} = {}) {
50+
return axios($, {
51+
...args,
52+
url: this.getUrl(path),
53+
headers: this.getHeaders(headers),
54+
});
55+
},
56+
tableMetadata({
57+
publisher, table, ...args
58+
}) {
59+
return this.makeRequest({
60+
path: `/datatables/${publisher}/${table}/metadata.json`,
61+
...args,
62+
});
63+
},
64+
tableData({
65+
publisher, table, ...args
66+
}) {
67+
return this.makeRequest({
68+
path: `/datatables/${publisher}/${table}.json`,
69+
...args,
70+
});
71+
},
72+
async paginate({
73+
fn, args = {}, maxRequests = 3,
74+
} = {}) {
75+
let allData = [];
76+
let cursorId = null;
77+
let requestCount = 0;
78+
let hasMorePages = true;
79+
80+
while (hasMorePages && requestCount < maxRequests) {
81+
const response = await fn({
82+
...args,
83+
params: {
84+
...args.params,
85+
"qopts.per_page": 100,
86+
...(cursorId
87+
? {
88+
"qopts.cursor_id": cursorId,
89+
}
90+
: undefined
91+
),
92+
},
93+
});
94+
95+
const pageData = response?.datatable?.data || [];
96+
allData = allData.concat(pageData);
97+
98+
cursorId = response?.meta?.next_cursor_id;
99+
hasMorePages = !!cursorId;
100+
requestCount++;
101+
}
102+
103+
return allData;
9104
},
10105
},
11106
};

‎components/nasdaq_data_link_time_series_and_table_data_/package.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@pipedream/nasdaq_data_link_time_series_and_table_data_",
3-
"version": "0.0.3",
3+
"version": "0.1.0",
44
"description": "Pipedream Nasdaq Data Link (Time Series and Table data) Components",
55
"main": "nasdaq_data_link_time_series_and_table_data_.app.mjs",
66
"keywords": [
@@ -11,5 +11,8 @@
1111
"author": "Pipedream <[email protected]> (https://pipedream.com/)",
1212
"publishConfig": {
1313
"access": "public"
14+
},
15+
"dependencies": {
16+
"@pipedream/platform": "^3.1.0"
1417
}
1518
}

0 commit comments

Comments
 (0)