diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03a5b9e23..c404cd17b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,16 +9,25 @@ jobs: strategy: matrix: - node-version: ['18.x', '20.x'] + node-version: ['20.x'] steps: - uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + run_install: false + - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - cache: 'npm' + cache: 'pnpm' + - name: Install dependencies - run: npm i + run: pnpm install + - name: Running tests - run: npm test \ No newline at end of file + run: pnpm test diff --git a/lib/core.js b/lib/core.js index ecc5e7db7..8c11cd9e1 100644 --- a/lib/core.js +++ b/lib/core.js @@ -1,5 +1,4 @@ - import * as _ from 'underscore'; import * as urlLib from 'url'; import * as pluginUtils from './loader/utils.js'; import * as utils from './utils.js'; @@ -12,6 +11,7 @@ import * as htmlUtils from './html-utils.js'; import * as metaUtils from './plugins/system/meta/utils.js'; import mediaPlugin from './plugins/validators/media.js'; + import { difference, intersection } from '../utils.js'; const plugins = pluginLoader._plugins, pluginsModules = pluginLoader._pluginsModules, @@ -92,7 +92,7 @@ * mandatoryParams - list of new params not used by plugins. Core will find what can use them. * 'mandatoryParams' enables mandatory mode: function will use _only_ methods which has this input 'mandatoryParams'. * This is used for "go down by tree" algorithm. - * var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); + * var mandatoryParams = difference(loadedParams, Object.keys(usedParams)); * mandatoryParams = [ * paramName * ] @@ -140,13 +140,13 @@ // If mandatory params mode. if (mandatoryParams && mandatoryParams.length > 0) { - if (_.intersection(params, mandatoryParams).length === 0) { + if (intersection(params, mandatoryParams).length === 0) { // Skip method if its not using mandatory params. continue; } } - var absentParams = _.difference(params, loadedParams); + var absentParams = difference(params, loadedParams); // If "__" (as in "__statusCode") or "...Error" (as in "oembedError") // super mandatory params are absent - skip the plugin. @@ -362,7 +362,7 @@ var loadedParams = Object.keys(context); - // var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams)); + // var mandatoryParams = difference(loadedParams, Object.keys(usedParams)); // Reset scanned plugins for each iteration. var scannedPluginsIds = {}; @@ -523,7 +523,7 @@ for(var k = 0; k < paramPlugins.length; k++) { var foundPluginId = paramPlugins[k]; - var exists = _.find(initialPlugins, function(plugin) { + var exists = initialPlugins.find(function(plugin) { return plugin.id === foundPluginId; }); if (!exists) { @@ -830,7 +830,7 @@ usedPluginMethods[method.name] = usedPluginMethods[method.name] - 1; if (r.error && options.debug) { - console.error(" -- Plugin error", method.pluginId, method.name, result.error); + console.error(" -- Plugin error", method.pluginId, method.name, r.error); } // Collect total result. @@ -1013,7 +1013,7 @@ links = [links]; } - links = _.compact(links); + links = links.filter(Boolean); for(var j = 0; j < links.length; j++) { var link = links[j]; @@ -1296,7 +1296,7 @@ // Sort links in order of REL according to CONFIG.REL_GROUPS. function getRelIndex(rel) { - var rels = _.intersection(rel, CONFIG.REL_GROUPS); + var rels = intersection(rel, CONFIG.REL_GROUPS); var gr = CONFIG.REL_GROUPS.length + 1; if (rels.length > 0) { for(var i = 0; i < rels.length; i++) { diff --git a/lib/fetch.js b/lib/fetch.js index 242a520bf..dabae460a 100644 --- a/lib/fetch.js +++ b/lib/fetch.js @@ -61,6 +61,11 @@ function doFetch(fetch_func, h1_fetch_func, options) { a_fetch_func(uri, fetch_options) .then(response => { var headers = response.headers.plain(); + var cookies = response.headers.raw()['set-cookie']; + if (cookies) { + // Keep cookies as array of strings. + headers['set-cookie'] = cookies; + } var stream = response.body; stream.on('end', () => { clearTimeout(timeoutTimerId); @@ -264,20 +269,32 @@ const cookiesOptions = [ 'SameSite', ]; -export function extendCookiesJar(jar, headers) { - if (headers && headers['set-cookie']) { - var cookies = parseCookie(headers['set-cookie']); - // Filter cookies options. - cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k))); - jar = jar || {}; - jar = {...jar, ...cookies}; +export function extendCookiesJar(uri, jar, headers) { + var cookiesValue = headers && headers['set-cookie']; + if (cookiesValue) { + var cookiesArray = Array.isArray(cookiesValue) ? cookiesValue : [cookiesValue]; + try { + var cookies = cookiesArray.reduce((allCookies, cookieStr) => { + return { ...allCookies, ...parseCookie(cookieStr) }; + }, {}); + // Filter cookies options. + cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k))); + jar = jar || {}; + jar = {...jar, ...cookies}; + } catch(ex) { + log('Error parse cookie', uri, ex.message); + } } return jar; } -export function setCookieFromJar(headers, jar) { +export function setCookieFromJar(uri, headers, jar) { if (jar) { - var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; '); - headers['Cookie'] = cookies; + try{ + var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; '); + headers['Cookie'] = cookies; + } catch(ex) { + log('Error serialize cookie', uri, ex.message); + } } } diff --git a/lib/html-utils.js b/lib/html-utils.js index 3a97f5ded..3913a954f 100644 --- a/lib/html-utils.js +++ b/lib/html-utils.js @@ -1,10 +1,12 @@ - import cheerio from 'cheerio'; - - import * as _ from 'underscore'; + import * as cheerio from 'cheerio'; import CONFIG from '../config.loader.js'; var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO; + function createCheerioElement(name) { + return cheerio.load(`<${name}>`)(name); + } + function wrapContainer($element, data, options) { var aspectWrapperClass = options && options.aspectWrapperClass; @@ -30,7 +32,7 @@ } } - var $container = cheerio('
') + var $container = createCheerioElement('div') .append($element); if (aspectWrapperClass) { @@ -45,7 +47,7 @@ var hasMaxWidth = media && (media["max-width"] || media["min-width"] || media["width"] || verticalAspect); if (hasMaxWidth || forceWidthLimitContainer) { - $widthLimitContainer = cheerio('
') + $widthLimitContainer = createCheerioElement('div') .append($container); } @@ -166,7 +168,7 @@ && data.href; }, generate: function(data) { - var $img = cheerio('') + var $img = createCheerioElement('img') .attr('src', data.href); if (data.title) { $img @@ -190,7 +192,7 @@ var givf = data.rel.indexOf('gifv') > -1; var autoplay = data.rel.indexOf('autoplay') > -1 || givf; - var $video = cheerio('Your browser does not support HTML5 video.'); + var $video = cheerio.load('Your browser does not support HTML5 video.')('video'); if (iframelyData && iframelyData.links) { @@ -279,8 +281,7 @@ return data.type === "text/html" && data.href; }, generate: function(data, options) { - - var $iframe = cheerio('