[Chores] Update package dependencies 2025-10 (#602)

* update some libs

* update `cheerio` import

* oembed: fix getting iframe

* html-utils: update cheerio usage to generate html code

* fix oembed-description cheerio usage

* review cheerio usage, add comments

* fix tumblr.text

* fix youtube.video

* simpler oembed.js

* update `cheerio.load` usage

* remove todos

* fix `tumblr.api` cheerio usage

* fix `tumblr` cheerio usage

* oembed: getIframe - reorganize ifs

* update `supertest` 6.3.3 to 7.1.4

* remove `mocha` obsolete dependency

* update `chai` from 4.3.2 to 6.2.0

* add new verion of `mocha`

* fix node supported to 20 only

* use pnpm for tests

* update `chokidar`

* tests: fix install pnpm

* force fix deep dependencies

* replace `_.find` to native array method

* Delete pnpm-workspace.yaml

* fix lock file

* Revert "fix lock file"

This reverts commit 78e0dd4.

* Revert "Delete pnpm-workspace.yaml"

This reverts commit 3ddd6ac.

* try fix pnpm workspace

* tests: fix pnpm version

* Revert "try fix pnpm workspace"

This reverts commit fb53400.

* `.keys` -> `Object.keys`

* `_.extend` -> `Object.assign`

* `_.values` -> `Object.values`

* `_.all -> array.every`

* `_.compact` -> `array.filter(Boolean)`

* `_.flatten` -> `array.flat`

* `_.isString` -> typeof

* remove `_.isArray` and `_.object`

* move `mocha` to `devDependencies`

* fix parsing multiple cookies

* remove debug log

* bugfix unefined

* bugfix in log

* remove underscore

Author: nleush

.github/workflows/tests.yml

@@ -9,16 +9,25 @@ jobs:
 
     strategy:
       matrix:
-        node-version: ['18.x', '20.x']
+        node-version: ['20.x']
 
     steps:
       - uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+          run_install: false
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
-          cache: 'npm'
+          cache: 'pnpm'
+
       - name: Install dependencies
-        run: npm i
+        run: pnpm install
+
       - name: Running tests
-        run: npm test
+        run: pnpm test

lib/core.js

@@ -1,5 +1,4 @@
 
-    import * as _ from 'underscore';
     import * as urlLib from 'url';
     import * as pluginUtils from './loader/utils.js';
     import * as utils from './utils.js';
@@ -12,6 +11,7 @@
     import * as htmlUtils from './html-utils.js';
     import * as metaUtils from './plugins/system/meta/utils.js';
     import mediaPlugin from './plugins/validators/media.js';
+    import { difference, intersection } from '../utils.js';
 
     const plugins = pluginLoader._plugins,
         pluginsModules = pluginLoader._pluginsModules,
@@ -92,7 +92,7 @@
         * mandatoryParams - list of new params not used by plugins. Core will find what can use them.
         * 'mandatoryParams' enables mandatory mode: function will use _only_ methods which has this input 'mandatoryParams'.
         * This is used for "go down by tree" algorithm.
-        * var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
+        * var mandatoryParams = difference(loadedParams, Object.keys(usedParams));
         * mandatoryParams = [
         *   paramName
         * ]
@@ -140,13 +140,13 @@
 
                 // If mandatory params mode.
                 if (mandatoryParams && mandatoryParams.length > 0) {
-                    if (_.intersection(params, mandatoryParams).length === 0) {
+                    if (intersection(params, mandatoryParams).length === 0) {
                         // Skip method if its not using mandatory params.
                         continue;
                     }
                 }
 
-                var absentParams = _.difference(params, loadedParams);
+                var absentParams = difference(params, loadedParams);
 
                 // If "__" (as in "__statusCode") or "...Error" (as in "oembedError") 
                 // super mandatory params are absent - skip the plugin.
@@ -362,7 +362,7 @@
 
         var loadedParams = Object.keys(context);
 
-        // var mandatoryParams = _.difference(loadedParams, Object.keys(usedParams));
+        // var mandatoryParams = difference(loadedParams, Object.keys(usedParams));
 
         // Reset scanned plugins for each iteration.
         var scannedPluginsIds = {};
@@ -523,7 +523,7 @@
                         for(var k = 0; k < paramPlugins.length; k++) {
                             var foundPluginId = paramPlugins[k];
 
-                            var exists = _.find(initialPlugins, function(plugin) {
+                            var exists = initialPlugins.find(function(plugin) {
                                 return plugin.id === foundPluginId;
                             });
                             if (!exists) {
@@ -830,7 +830,7 @@
             usedPluginMethods[method.name] = usedPluginMethods[method.name] - 1;
 
             if (r.error && options.debug) {
-                console.error("   -- Plugin error", method.pluginId, method.name, result.error);
+                console.error("   -- Plugin error", method.pluginId, method.name, r.error);
             }
 
             // Collect total result.
@@ -1013,7 +1013,7 @@
                     links = [links];
                 }
 
-                links = _.compact(links);
+                links = links.filter(Boolean);
 
                 for(var j = 0; j < links.length; j++) {
                     var link = links[j];
@@ -1296,7 +1296,7 @@
 
         // Sort links in order of REL according to CONFIG.REL_GROUPS.
         function getRelIndex(rel) {
-            var rels = _.intersection(rel, CONFIG.REL_GROUPS);
+            var rels = intersection(rel, CONFIG.REL_GROUPS);
             var gr = CONFIG.REL_GROUPS.length + 1;
             if (rels.length > 0) {
                 for(var i = 0; i < rels.length; i++) {

lib/fetch.js

@@ -61,6 +61,11 @@ function doFetch(fetch_func, h1_fetch_func, options) {
         a_fetch_func(uri, fetch_options)
             .then(response => {
                 var headers = response.headers.plain();
+                var cookies = response.headers.raw()['set-cookie'];
+                if (cookies) {
+                    // Keep cookies as array of strings.
+                    headers['set-cookie'] = cookies;
+                }
                 var stream = response.body;
                 stream.on('end', () => {
                     clearTimeout(timeoutTimerId);
@@ -264,20 +269,32 @@ const cookiesOptions = [
     'SameSite',
 ];
 
-export function extendCookiesJar(jar, headers) {
-    if (headers && headers['set-cookie']) {
-        var cookies = parseCookie(headers['set-cookie']);
-        // Filter cookies options.
-        cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k)));
-        jar = jar || {};
-        jar = {...jar, ...cookies};
+export function extendCookiesJar(uri, jar, headers) {
+    var cookiesValue = headers && headers['set-cookie'];
+    if (cookiesValue) {
+        var cookiesArray = Array.isArray(cookiesValue) ? cookiesValue : [cookiesValue];
+        try {
+            var cookies = cookiesArray.reduce((allCookies, cookieStr) => {
+                return { ...allCookies, ...parseCookie(cookieStr) };
+            }, {});
+            // Filter cookies options.
+            cookies = Object.fromEntries(Object.entries(cookies).filter(([k,v]) => !cookiesOptions.includes(k)));
+            jar = jar || {};
+            jar = {...jar, ...cookies};
+        } catch(ex) {
+            log('Error parse cookie', uri, ex.message);
+        }
     }
     return jar;
 }
 
-export function setCookieFromJar(headers, jar) {
+export function setCookieFromJar(uri, headers, jar) {
     if (jar) {
-        var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; ');
-        headers['Cookie'] = cookies;
+        try{
+            var cookies = Object.entries(jar).map(([k,v]) => serializeCookie(k, v)).join('; ');
+            headers['Cookie'] = cookies;
+        } catch(ex) {
+            log('Error serialize cookie', uri, ex.message);
+        }
     }
 }

lib/html-utils.js

@@ -1,10 +1,12 @@
-    import cheerio from 'cheerio';
-    
-    import * as _ from 'underscore';
+    import * as cheerio from 'cheerio';
     import CONFIG from '../config.loader.js';
 
     var defaultPaddingBottom = 100 / CONFIG.DEFAULT_ASPECT_RATIO;
 
+    function createCheerioElement(name) {
+        return cheerio.load(`<${name}></${name}>`)(name);
+    }
+
     function wrapContainer($element, data, options) {
 
         var aspectWrapperClass = options && options.aspectWrapperClass;
@@ -30,7 +32,7 @@
             }
         }
 
-        var $container = cheerio('<div>')
+        var $container = createCheerioElement('div')
             .append($element);
 
         if (aspectWrapperClass) {
@@ -45,7 +47,7 @@
         var hasMaxWidth = media && (media["max-width"] || media["min-width"] || media["width"] || verticalAspect);
 
         if (hasMaxWidth || forceWidthLimitContainer) {
-            $widthLimitContainer = cheerio('<div>')
+            $widthLimitContainer = createCheerioElement('div')
                 .append($container);
         }
 
@@ -166,7 +168,7 @@
                     && data.href;
             },
             generate: function(data) {
-                var $img = cheerio('<img>')
+                var $img = createCheerioElement('img')
                     .attr('src', data.href);
                 if (data.title) {
                     $img
@@ -190,7 +192,7 @@
                 var givf = data.rel.indexOf('gifv') > -1;
                 var autoplay = data.rel.indexOf('autoplay') > -1 || givf;
 
-                var $video = cheerio('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>');
+                var $video = cheerio.load('<video' + (givf ? ' loop muted webkit-playsinline' : ' controls') + (autoplay ? ' autoplay' : '') + '>Your browser does not support HTML5 video.</video>')('video');
 
                 if (iframelyData && iframelyData.links) {
 
@@ -279,8 +281,7 @@
                 return data.type === "text/html" && data.href;
             },
             generate: function(data, options) {
-
-                var $iframe = cheerio('<iframe>')
+                var $iframe = createCheerioElement('iframe')
                     .attr('src', data.href)
                     .css('border', '0')
                     .attr('allowfullscreen', '');
@@ -334,25 +335,15 @@
         }
     };
 
-    export function generateElementWrapperHtml(element, link, options) {
-
-        if (typeof element === 'string') {
-            element = cheerio(element);
-        }
-
-        var $el = wrapContainer(element, link, options);
-        return cheerio('<div>').append($el).html();
-    };
-
     export function generateLinkElementHtml(link, options) {
         var $el = generateLinkElement(link, options);
-        if (_.isString($el)) {
+        if (typeof $el === 'string') {
             return $el;
         } else if ($el) {
             if (options && options.canonical && link.href !== options.canonical) {
                 $el.attr('data-embed-canonical', options.canonical);
             }
-            return cheerio('<div>').append($el).html();
+            return $el.prop('outerHTML');
         } else {
             return '';
         }
@@ -647,7 +638,7 @@
             sortLinks(iframely_data.links, options.autoplayMode);
         }
 
-        var player = iframely_data && _.find(iframely_data.links, findPlayer);
+        var player = iframely_data?.links.find(findPlayer);
 
         if (player) {
 
@@ -674,7 +665,7 @@
     }
 
     function getInlineApp(iframely_data) {
-        return iframely_data && _.find(iframely_data.links, function(l) {
+        return iframely_data?.links.find(function(l) {
             // `html` can be added by `generateLinksHtml`, so also check if no `href`.
             return l.html && !l.href && l.type === 'text/html';
         });
@@ -706,4 +697,4 @@
         }
 
         return selectedLink;
-    };
+    };

lib/oembed.js

@@ -1,5 +1,5 @@
-import * as _ from 'underscore';
 import * as htmlUtils from './html-utils.js';
+import { intersection } from '../utils.js';
 
 export function getOembed(uri, data, options) {
 
@@ -63,7 +63,7 @@ export function getOembed(uri, data, options) {
 
     var link;
     htmlUtils.sortLinks(data.links);
-    var foundRel = _.find(options && options.mediaPriority ? CONFIG.OEMBED_RELS_MEDIA_PRIORITY : CONFIG.OEMBED_RELS_PRIORITY, function(rel) {
+    var foundRel = (options?.mediaPriority ? CONFIG.OEMBED_RELS_MEDIA_PRIORITY : CONFIG.OEMBED_RELS_PRIORITY).find(function(rel) {
         link = htmlUtils.filterLinksByRel(rel, data.links, {
             returnOne: true,
             excludeRel: CONFIG.R.autoplay
@@ -72,15 +72,15 @@ export function getOembed(uri, data, options) {
     });
 
     if (!link) {
-        link = _.find(data.links, function(link) {
+        link = data.links.find(function(link) {
             return link.type.indexOf('image') === 0 && link.rel.indexOf(CONFIG.R.file) > -1;
         });
         if (link) {
             foundRel = CONFIG.R.image;
         }
     }
 
-    var inlineReader = link && _.intersection(link.rel, [CONFIG.R.inline, CONFIG.R.reader]).length == 2;
+    var inlineReader = link && intersection(link.rel, [CONFIG.R.inline, CONFIG.R.reader]).length == 2;
 
     var m = link && link.media;
     if (m) {
@@ -136,7 +136,7 @@ export function getOembed(uri, data, options) {
             };
 
             if (CONFIG.GENERATE_LINK_PARAMS) {
-                _.extend(generateLinkOptions, CONFIG.GENERATE_LINK_PARAMS);
+                Object.assign(generateLinkOptions, CONFIG.GENERATE_LINK_PARAMS);
             }
 
             oembed.html = htmlUtils.generateLinkElementHtml(link, generateLinkOptions);

lib/plugins/system/cheerio.js

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 import htmlparser2 from "htmlparser2";
 var DomHandler = htmlparser2.DomHandler;
 
@@ -24,4 +24,4 @@ export default {
         htmlparser.addHandler(domHandler);
     }
 
-};
+};

lib/plugins/system/htmlparser/htmlparser.js

@@ -126,7 +126,7 @@ export default {
                         // Prevent cache self redirect. Some sites changes cookies and stops redirect loop (e.g. https://miro.com/app/live-embed/o9J_lBwNMhI=/?embedAutoplay=true)
                         const redirectUrl = urlLib.resolve(url, headers.location);
                         const preventCache = redirectUrl === url;
-                        options.jar = extendCookiesJar(options.jar, headers);
+                        options.jar = extendCookiesJar(url, options.jar, headers);
                         // TODO: do not cache when has unique cookie???
                         if (options2.maxRedirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) {
                             options.maxRedirects = options2.maxRedirects; // Maybe set in prepareRequestOptions for a proxy.

lib/plugins/system/oembed/oembed.js

@@ -1,5 +1,5 @@
 import * as oembedUtils from './oembedUtils.js';
-import cheerio from 'cheerio';
+import * as cheerio from 'cheerio';
 
 import * as entities from 'entities';
 import * as URL from 'url';
@@ -44,20 +44,21 @@ function _getOembedIframe(oembed) {
                 html = entities.decodeHTML(html);
             }
 
-            var $container = cheerio('<div>');
+            var $iframe = null;
             try {
-                $container.html(html);
+                var $iframe = cheerio.load(html)('iframe');
             } catch (ex) {}
-            var $iframe = $container.find('iframe');
 
-            if ($iframe.length === 2 && /<iframe>$/i.test(html)) { 
-                // Forgive mis-closed iFrame tag
-                $iframe = $iframe.first();
-            }
-
-            if ($iframe.length === 1) {
-                _iframe = fixOembedIframeAttributes($iframe[0].attribs);
-                _iframe.placeholder = oembed.thumbnail_url;
+            if ($iframe) {
+                if ($iframe.length === 2 && /<iframe>$/i.test(html)) { 
+                    // Forgive mis-closed iFrame tag
+                    $iframe = $iframe.first();
+                }
+    
+                if ($iframe.length === 1) {
+                    _iframe = fixOembedIframeAttributes($iframe[0].attribs);
+                    _iframe.placeholder = oembed.thumbnail_url;
+                }
             }
 
         // When oembed is in fact iframe from domain fallbacks on oEmbedError