Skip to content

Commit 12afddb

Browse files
nleushnleush
authored
Return additional data (#584)
* allow return extra options: fetch made, fetched uri * Revert "allow return extra options: fetch made, fetched uri" * return requested `extraParams` from `context` * rename `extraParams` to `dataParams` * send data params when error * add `getVars` plugin method * fix collecting data for error response * options.registerFetch functions * return fetches in result * fix typo * detect if data was from cache * return fetches data with error * Convert runtime `error` to `error_code` and `error_message`
1 parent f9dd1c1 commit 12afddb

File tree

6 files changed

+184
-9
lines changed

6 files changed

+184
-9
lines changed

‎lib/core.js‎

Lines changed: 100 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -940,6 +940,28 @@
940940
}
941941
}
942942
}
943+
} else if (r.method.name === "getVars") {
944+
for(var key in r.data) {
945+
var v = r.data[key];
946+
947+
if (v !== '' && v !== null && ((typeof v === 'string' && !/^\s+$/.test(v)) || typeof v === 'number')) {
948+
949+
// Check meta plugins order.
950+
allResults.vars._sources = allResults.vars._sources || {};
951+
952+
var prevOrder = null, nextOrder = null, pluginId = allResults.vars._sources[key];
953+
954+
if (pluginId && plugins[pluginId] && plugins[r.method.pluginId]) {
955+
prevOrder = plugins[pluginId].order;
956+
nextOrder = plugins[r.method.pluginId].order;
957+
}
958+
959+
if (!prevOrder || !nextOrder || prevOrder < nextOrder) {
960+
allResults.vars[key] = v;
961+
allResults.vars._sources[key] = r.method.pluginId;
962+
}
963+
}
964+
}
943965
}
944966
}
945967
}
@@ -1072,6 +1094,11 @@
10721094
delete result.h2;
10731095
delete result.allData;
10741096
delete result.meta._sources;
1097+
delete result.vars._sources;
1098+
}
1099+
1100+
if (Object.keys(result.vars).length === 0) {
1101+
delete result.vars;
10751102
}
10761103

10771104
var links = result.links;
@@ -1219,6 +1246,10 @@
12191246
result.vary = options._usedProviderOptions;
12201247
result.vary.sort && result.vary.sort();
12211248
}
1249+
1250+
if (options.returnFetchUsage) {
1251+
result.fetches = options._fetches;
1252+
}
12221253
}
12231254

12241255
export function sortLinks(links) {
@@ -1413,6 +1444,47 @@
14131444
}
14141445
}
14151446

1447+
function generateRegisterFetchFunc(url, options) {
1448+
if (options.returnFetchUsage) {
1449+
options._fetches = options._fetches || [];
1450+
}
1451+
1452+
options.registerFetch = function(data) {
1453+
if (options.returnFetchUsage) {
1454+
options._fetches.push(data);
1455+
}
1456+
};
1457+
options.registerFetchError = function(data) {
1458+
if (options.returnFetchUsage) {
1459+
1460+
var fetch_data = {...data};
1461+
1462+
if (!data.error) {
1463+
console.warn('No error param in registerFetchError data');
1464+
} else {
1465+
// Convert runtime `error` to `error_code` and `error_message`.
1466+
if (typeof data.error === 'string') {
1467+
fetch_data.error_code = data.error
1468+
} else {
1469+
if (data.error.code) {
1470+
fetch_data.error_code = data.error.code;
1471+
}
1472+
if (data.error.message) {
1473+
fetch_data.error_message = data.error.message;
1474+
}
1475+
if (!data.error.code && !data.error.message) {
1476+
fetch_data.error_message = '' + data.error;
1477+
}
1478+
}
1479+
}
1480+
1481+
fetch_data.error = true;
1482+
1483+
options._fetches.push(fetch_data);
1484+
}
1485+
};
1486+
}
1487+
14161488
function generateProviderOptionsFunc(url, options) {
14171489
if (options.returnProviderOptionsUsage) {
14181490
options._usedProviderOptions = options._usedProviderOptions || [];
@@ -1551,6 +1623,8 @@
15511623

15521624
generateProviderOptionsFunc(uri, options);
15531625

1626+
generateRegisterFetchFunc(uri, options);
1627+
15541628
if (utils.isBlocked(uri, options, cb)) {
15551629
if (options.v === '1.3') {
15561630
cb({
@@ -1637,13 +1711,27 @@
16371711
allResults = {
16381712
meta: {},
16391713
links: [],
1714+
vars: {},
16401715
messages: [],
16411716
allData: []
16421717
},
16431718
usedMethods = {},
16441719

16451720
aborted = false,
16461721

1722+
appendDataParams = function(result) {
1723+
if (options.dataParams) {
1724+
for(var i = 0; i < options.dataParams.length; i++) {
1725+
var param = options.dataParams[i];
1726+
var value = context[param];
1727+
if (value) {
1728+
var e = result.data = allResults.data || {};
1729+
e[param] = value;
1730+
}
1731+
}
1732+
}
1733+
},
1734+
16471735
abortCurrentRequest = function() {
16481736
if (context.htmlparser && context.htmlparser.abortController) {
16491737
context.htmlparser.abortController.abort();
@@ -1672,6 +1760,8 @@
16721760
}
16731761
}
16741762

1763+
appendDataParams(allResults);
1764+
16751765
if (options.whitelist) {
16761766
allResults.whitelist = options.getWhitelistRecord && options.getWhitelistRecord(uri, {disableWildcard: true});
16771767
}
@@ -1713,6 +1803,10 @@
17131803
return;
17141804
}
17151805

1806+
// Gather results.
1807+
// Run before `responseError` to collect data and send in error.
1808+
var hasNewData = useResult(usedMethods, context, pluginsContexts, allResults, result, options, asyncMethodCb);
1809+
17161810
// Abort on error response code.
17171811
var responseError = findResponseError(result, uri);
17181812
if (responseError) {
@@ -1739,16 +1833,19 @@
17391833
error.messages = allResults.messages;
17401834
}
17411835

1836+
appendDataParams(error);
1837+
1838+
if (options.returnFetchUsage) {
1839+
error.fetches = options._fetches;
1840+
}
1841+
17421842
return cb(error);
17431843

17441844
} else {
17451845
return cb(responseError);
17461846
}
17471847
}
17481848

1749-
// Gather results.
1750-
var hasNewData = useResult(usedMethods, context, pluginsContexts, allResults, result, options, asyncMethodCb);
1751-
17521849
if (options.fetchParam && options.fetchParam in context) {
17531850
abortCurrentRequest();
17541851
cb(null, context[options.fetchParam]);

‎lib/loader/utils.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,10 @@
3838
"getLinks",
3939
"getData",
4040
"getMeta",
41+
"getVars",
4142
"prepareLink"
4243
];
4344

4445
export const PLUGIN_FIELDS = PLUGIN_METHODS.concat([
4546
"mixins"
46-
]);
47+
]);

‎lib/plugins/system/htmlparser/htmlparser.js‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,20 @@ export default {
2323
reuseCookies: !!options.followHTTPRedirect
2424
}};
2525

26+
options.registerFetch({
27+
source: 'html',
28+
url: url
29+
});
30+
2631
getUrlFunctional(url, options2, {
2732
onError: function(error) {
33+
34+
options.registerFetchError({
35+
source: 'html',
36+
url: url,
37+
error: error
38+
});
39+
2840
if (error.code === 'ENOTFOUND') {
2941
if (!!options.exposeStatusCode) {
3042
cb(null, {
@@ -126,6 +138,13 @@ export default {
126138
}
127139

128140
if (resp.status !== 200) {
141+
142+
options.registerFetchError({
143+
source: 'html',
144+
url: url,
145+
status_code: resp.status
146+
});
147+
129148
if (!!options.exposeStatusCode) {
130149
return cacheAndRespond(null, {
131150
__statusCode: resp.status,
@@ -182,4 +201,4 @@ export default {
182201
}
183202
});
184203
}
185-
};
204+
};

‎lib/plugins/system/meta/cachedMeta.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ export default {
7373
return noCacheFound();
7474
}
7575

76+
options.registerFetch({
77+
source: 'cache',
78+
url: url
79+
});
80+
7681
if (data.error) {
7782

7883
if (data.error.redirect) {
@@ -109,4 +114,4 @@ export default {
109114
}
110115
}
111116

112-
};
117+
};

‎lib/plugins/system/oembed/oembedUtils.js‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,12 @@ export function findOembedLinks(uri, meta) {
185185
}
186186

187187
if (data && !cachedError) {
188+
189+
options && options.registerFetch({
190+
source: 'cache',
191+
url: uri
192+
});
193+
188194
if (data.response && data.data) {
189195
// Fallback to `request` format. Remove after 10.10.2020.
190196
data = data.data;
@@ -227,9 +233,32 @@ export function findOembedLinks(uri, meta) {
227233
});
228234
}
229235

236+
options && options.registerFetch({
237+
source: 'oembed',
238+
url: uri
239+
});
240+
230241
getUrlFunctional(uri, {}, {
231-
onError: cbWrapper,
242+
onError: function(error) {
243+
244+
options && options.registerFetchError({
245+
source: 'oembed',
246+
url: uri,
247+
error: error
248+
});
249+
250+
cbWrapper(error);
251+
},
232252
onResponse: function(res) {
253+
254+
if (res.status !== 200) {
255+
options && options.registerFetchError({
256+
source: 'oembed',
257+
url: uri,
258+
status_code: res.status
259+
});
260+
}
261+
233262
if (res.status == 200) {
234263
parseResponse(res);
235264
} else if (options && options.parseErrorBody) {
@@ -413,4 +442,4 @@ function jsonStream2oembed(stream, callback) {
413442

414443
callback(null, data);
415444
});
416-
}
445+
}

‎lib/request.js‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,20 @@ export default function(options, iframely_options, callback) {
9292
}
9393
}
9494

95+
if (error) {
96+
options.registerFetchError({
97+
source: 'api',
98+
url: options.uri,
99+
error: error
100+
});
101+
} else if (result && result.status !== 200) {
102+
options.registerFetchError({
103+
source: 'api',
104+
url: options.uri,
105+
status_code: result.status
106+
});
107+
}
108+
95109
const response = result && {
96110
statusCode: result.status,
97111
headers: result.headers
@@ -143,6 +157,11 @@ export default function(options, iframely_options, callback) {
143157
options.headers['User-Agent'] = CONFIG.USER_AGENT;
144158
}
145159

160+
options.registerFetch({
161+
source: 'api',
162+
url: options.uri
163+
});
164+
146165
fetchData(utils.prepareRequestOptions(options, iframely_options))
147166
.then(result => {
148167
finish(null, result);
@@ -180,6 +199,11 @@ export default function(options, iframely_options, callback) {
180199
return;
181200
}
182201
}
202+
203+
options.registerFetch({
204+
source: 'cache',
205+
url: options.uri
206+
});
183207

184208
// Send cached data up.
185209
prepareResult(null, Object.assign(data.response, {fromRequestCache: true}), data.data, callback);
@@ -200,4 +224,4 @@ export default function(options, iframely_options, callback) {
200224
}
201225
});
202226
}
203-
};
227+
};

0 commit comments

Comments
 (0)