[pull] master from microlinkhq:master

CHANGELOG.md

@@ -3,6 +3,22 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [5.49.11](https://github.com/microlinkhq/metascraper/compare/v5.49.10...v5.49.11) (2025-12-23)
+
+**Note:** Version bump only for package metascraper
+
+## [5.49.10](https://github.com/microlinkhq/metascraper/compare/v5.49.9...v5.49.10) (2025-12-23)
+
+**Note:** Version bump only for package metascraper
+
+## [5.49.9](https://github.com/microlinkhq/metascraper/compare/v5.49.8...v5.49.9) (2025-12-23)
+
+**Note:** Version bump only for package metascraper
+
+## [5.49.8](https://github.com/microlinkhq/metascraper/compare/v5.49.7...v5.49.8) (2025-12-23)
+
+**Note:** Version bump only for package metascraper
+
 ## [5.49.7](https://github.com/microlinkhq/metascraper/compare/v5.49.6...v5.49.7) (2025-12-03)
 
 **Note:** Version bump only for package metascraper

lerna.json

@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "5.49.7",
+  "version": "5.49.11",
   "command": {
     "bootstrap": {
       "npmClientArgs": [

packages/metascraper-x/CHANGELOG.md

@@ -3,6 +3,22 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [5.49.11](https://github.com/microlinkhq/metascraper/compare/v5.49.10...v5.49.11) (2025-12-23)
+
+**Note:** Version bump only for package metascraper-x
+
+## [5.49.10](https://github.com/microlinkhq/metascraper/compare/v5.49.9...v5.49.10) (2025-12-23)
+
+**Note:** Version bump only for package metascraper-x
+
+## [5.49.9](https://github.com/microlinkhq/metascraper/compare/v5.49.8...v5.49.9) (2025-12-23)
+
+**Note:** Version bump only for package metascraper-x
+
+## [5.49.8](https://github.com/microlinkhq/metascraper/compare/v5.49.7...v5.49.8) (2025-12-23)
+
+**Note:** Version bump only for package metascraper-x
+
 ## [5.49.7](https://github.com/microlinkhq/metascraper/compare/v5.49.6...v5.49.7) (2025-12-03)
 
 **Note:** Version bump only for package metascraper-x

packages/metascraper-x/README.md

@@ -20,17 +20,11 @@ $ npm install metascraper-x --save
 
 #### options
 
-##### resolveUrls
-
-Type: `boolean`
-
-Set to `true` if you want to resolve `t.co` URLs into the final URL.
-
 ##### resolveUrl
 
 Type: `function`
 
-The function implementation that resolves `t.co` into a the final URL.
+The function implementation that resolves URLs (e.g., `t.co`) into the final URL.
 
 ## License
 

packages/metascraper-x/package.json

@@ -2,7 +2,7 @@
   "name": "metascraper-x",
   "description": "Metascraper integration with x.com",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-x",
-  "version": "5.49.7",
+  "version": "5.49.11",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

packages/metascraper-x/src/index.js

@@ -1,55 +1,75 @@
 'use strict'
 
 const {
-  getUrls,
   $jsonld,
   author,
+  date,
+  description,
+  getUrls,
   image,
   memoizeOne,
   parseUrl,
   title,
   toRule,
-  description,
   url
 } = require('@metascraper/helpers')
 
+const toDescription = toRule(description)
 const toAuthor = toRule(author)
 const toImage = toRule(image)
 const toTitle = toRule(title)
-
-const toDescription = toRule(description)
-
+const toDate = toRule(date)
 const toUrl = toRule(url)
 
 const test = memoizeOne(url =>
   ['twitter.com', 'x.com'].includes(parseUrl(url).domain)
 )
 
-module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
+const getAuthorName = memoizeOne(ogTitle =>
+  ogTitle?.split(' on X:')[0].split(' (@')[0].trim()
+)
+
+module.exports = ({ resolveUrl = url => url } = {}) => {
   const rules = {
     author: [
-      toAuthor($ => {
-        const author = $('meta[property="og:title"]').attr('content')
-        return author?.includes(' on X') ? author.split(' on X')[0] : author
-      })
-    ],
-    title: [
-      toTitle(
-        ($, url) => `@${new URL(url).pathname.toString().split('/')[1]} on X`
+      toAuthor($ =>
+        getAuthorName($('meta[property="og:title"]').attr('content'))
       )
     ],
-    url: [
-      toUrl($ =>
-        $('link[rel="canonical"]').attr('href')?.replace('twitter.com', 'x.com')
-      )
+    title: [
+      toTitle(($, url) => {
+        const username = new URL(url).pathname.split('/')[1]
+        const authorName = getAuthorName(
+          $('meta[property="og:title"]').attr('content')
+        )
+        return authorName
+          ? `${authorName} (@${username}) on X`
+          : `@${username} on X`
+      })
     ],
+    url: [toUrl($ => $('link[rel="canonical"]').attr('href'))],
     description: [
       toDescription(async $ => {
         let description =
           $jsonld('mainEntity.description')($) ||
           $('meta[property="og:description"]').attr('content')
 
-        if (!resolveUrls) return description
+        if (!description) {
+          const ogTitle = $('meta[property="og:title"]').attr('content')
+          if (ogTitle?.includes(' on X: "')) {
+            description = ogTitle.split(' on X: "')[1].split('" / X')[0]
+            const urls = getUrls(description)
+            if (urls.length > 1) {
+              const lastUrl = urls[urls.length - 1]
+              if (description.endsWith(lastUrl)) {
+                description = description.replace(lastUrl, '').trim()
+              }
+            }
+          }
+        }
+
+        if (!description) return
+
         const urls = getUrls(description)
         const resolvedUrls = await Promise.all(urls.map(resolveUrl))
 
@@ -63,14 +83,41 @@ module.exports = ({ resolveUrls = false, resolveUrl = url => url } = {}) => {
     ],
     image: [
       toImage($ => {
-        let imageUrl =
+        const imageUrl =
           $jsonld('mainEntity.image.contentUrl')($) ||
+          $('video').attr('poster') ||
           $('meta[property="og:image"]').attr('content')
 
-        if (imageUrl?.endsWith('_200x200.jpg')) {
-          imageUrl = imageUrl.replace('_200x200.jpg', '_400x400.jpg')
+        return imageUrl?.endsWith('_200x200.jpg')
+          ? imageUrl.replace('_200x200.jpg', '_400x400.jpg')
+          : imageUrl
+      })
+    ],
+    date: [
+      toDate(($, url) => {
+        const { pathname } = new URL(url)
+        const parts = pathname.split('/')
+        const username = parts[1].toLowerCase()
+        const isStatus = parts[2] === 'status'
+
+        if (isStatus) {
+          const statusId = parts[3]
+          return (
+            $(`a[href*="/status/${statusId}"] time`).attr('datetime') ||
+            $('article time').attr('datetime') ||
+            $('time').attr('datetime')
+          )
         }
-        return imageUrl
+
+        return $(`a[href*="/${username}/status/" i] time`)
+          .get()
+          .reduce((acc, el) => {
+            const datetime = $(el).attr('datetime')
+            if (datetime && (!acc || datetime > acc)) {
+              acc = datetime
+            }
+            return acc
+          }, undefined)
       })
     ],
     publisher: () => 'X'

packages/metascraper-x/test/fixtures/post-gif-og.html

@@ -0,0 +1,2 @@
+<!DOCTYPE html><html dir="ltr" lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=0,viewport-fit=cover" /><link rel="preconnect" href="//abs.twimg.com" /><link rel="dns-prefetch" href="//abs.twimg.com" /><link rel="preconnect" href="//api.twitter.com" /><link rel="dns-prefetch" href="//api.twitter.com" /><link rel="preconnect" href="//api.x.com" /><link rel="dns-prefetch" href="//api.x.com" /><link rel="preconnect" href="//pbs.twimg.com" /><link rel="dns-prefetch" href="//pbs.twimg.com" /><link rel="preconnect" href="//t.co" /><link rel="dns-prefetch" href="//t.co" /><link rel="preconnect" href="//video.twimg.com" /><link rel="dns-prefetch" href="//video.twimg.com" /><meta http-equiv="onion-location" content="https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/" /><meta property="fb:app_id" content="2231777543" /><meta content="X (formerly Twitter)" property="og:site_name" /><meta name="google-site-verification" content="600dQ0pZYsH2xOFt4hYmf5f5NpjCbWE_qk5Y04dErYM" /><meta name="facebook-domain-verification" content="x6sdcc8b5ju3bh8nbm59eswogvg6t1" /><meta name="mobile-web-app-capable" content="yes" /><meta name="apple-mobile-web-app-title" content="Twitter" /><meta name="apple-mobile-web-app-status-bar-style" content="white" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X">
+  <link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="manifest" href="/manifest.json" crossOrigin="use-credentials" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X"><link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="shortcut icon" href="//abs.twimg.com/favicons/twitter.3.ico"><meta name="theme-color" media="(prefers-color-scheme: light)" content="#FFFFFF" /><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000000" /><script type="application/ld+json">{"@context":"https://schema.org","@type":"WebSite","url":"https://twitter.com/","potentialAction":{"@type":"SearchAction","query-input":"required name=search_term_string","target":{"@type":"EntryPoint","urlTemplate":"https://twitter.com/search?q={search_term_string}&ref_src=twcamp%5Eseo_searchbox%7Ctwsrc%5Eseo"}}}</script><meta http-equiv="origin-trial" content="AlpCmb40F5ZjDi9ZYe+wnr/V8MF+XmY41K4qUhoq+2mbepJTNd3q4CRqlACfnythEPZqcjryfAS1+ExS0FFRcA8AAABmeyJvcmlnaW4iOiJodHRwczovL3R3aXR0ZXIuY29tOjQ0MyIsImZlYXR1cmUiOiJMYXVuY2ggSGFuZGxlciIsImV4cGlyeSI6MTY1NTI1MTE5OSwiaXNTdWJkb21haW4iOnRydWV9" /><style>html,body{height: 100%;}::cue{white-space:normal}</style><meta content="article" property="og:type" /><meta content="https://twitter.com/Kikobeats/status/880139124791029763" property="og:url" /><meta content="#!/kiko/beats (@Kikobeats) on X" property="og:title" /><meta content="Experimenting with Clearbit API + Apple TV 3D Parallax https://t.co/Qsm163k4mJ" property="og:description" /><meta content="https://pbs.twimg.com/tweet_video_thumb/DDbh3WCXYAAZfz9.jpg:large" property="og:image" /><meta content="Experimenting with Clearbit API + Apple TV 3D Parallax https://t.co/Qsm163k4mJ" name="description" /><meta content="#!/kiko/beats (@Kikobeats) on X" name="title" /><meta content="twitter://status?id=880139124791029763" property="al:ios:url" /><meta content="333903271" property="al:ios:app_store_id" /><meta content="X" property="al:ios:app_name" /><meta content="twitter://status?id=880139124791029763" property="al:android:url" /><meta content="com.twitter.android" property="al:android:package" /><meta content="X" property="al:android:app_name" /></head><body style="background-color: #FFFFFF;"><div id="react-root" style="height:100%;display:flex;"></div>

packages/metascraper-x/test/fixtures/post-image-og.html

@@ -0,0 +1,2 @@
+<!DOCTYPE html><html dir="ltr" lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=0,viewport-fit=cover" /><link rel="preconnect" href="//abs.twimg.com" /><link rel="dns-prefetch" href="//abs.twimg.com" /><link rel="preconnect" href="//api.twitter.com" /><link rel="dns-prefetch" href="//api.twitter.com" /><link rel="preconnect" href="//api.x.com" /><link rel="dns-prefetch" href="//api.x.com" /><link rel="preconnect" href="//pbs.twimg.com" /><link rel="dns-prefetch" href="//pbs.twimg.com" /><link rel="preconnect" href="//t.co" /><link rel="dns-prefetch" href="//t.co" /><link rel="preconnect" href="//video.twimg.com" /><link rel="dns-prefetch" href="//video.twimg.com" /><meta http-equiv="onion-location" content="https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/" /><meta property="fb:app_id" content="2231777543" /><meta content="X (formerly Twitter)" property="og:site_name" /><meta name="google-site-verification" content="600dQ0pZYsH2xOFt4hYmf5f5NpjCbWE_qk5Y04dErYM" /><meta name="facebook-domain-verification" content="x6sdcc8b5ju3bh8nbm59eswogvg6t1" /><meta name="mobile-web-app-capable" content="yes" /><meta name="apple-mobile-web-app-title" content="Twitter" /><meta name="apple-mobile-web-app-status-bar-style" content="white" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X">
+  <link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="manifest" href="/manifest.json" crossOrigin="use-credentials" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X"><link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="shortcut icon" href="//abs.twimg.com/favicons/twitter.3.ico"><meta name="theme-color" media="(prefers-color-scheme: light)" content="#FFFFFF" /><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000000" /><script type="application/ld+json">{"@context":"https://schema.org","@type":"WebSite","url":"https://twitter.com/","potentialAction":{"@type":"SearchAction","query-input":"required name=search_term_string","target":{"@type":"EntryPoint","urlTemplate":"https://twitter.com/search?q={search_term_string}&ref_src=twcamp%5Eseo_searchbox%7Ctwsrc%5Eseo"}}}</script><meta http-equiv="origin-trial" content="AlpCmb40F5ZjDi9ZYe+wnr/V8MF+XmY41K4qUhoq+2mbepJTNd3q4CRqlACfnythEPZqcjryfAS1+ExS0FFRcA8AAABmeyJvcmlnaW4iOiJodHRwczovL3R3aXR0ZXIuY29tOjQ0MyIsImZlYXR1cmUiOiJMYXVuY2ggSGFuZGxlciIsImV4cGlyeSI6MTY1NTI1MTE5OSwiaXNTdWJkb21haW4iOnRydWV9" /><style>html,body{height: 100%;}::cue{white-space:normal}</style><meta content="article" property="og:type" /><meta content="https://twitter.com/UaSmart/status/934106870834454529" property="og:url" /><meta content="SmartUA (@UaSmart) on X" property="og:title" /><meta content="Y terminamos el dia con Cultura de empresa con @patoroco, @flopezluis, Katia, Angélica en @codemotion_es #codemotion2017" property="og:description" /><meta content="https://pbs.twimg.com/media/DPadOKpXcAIL-NW.jpg:large" property="og:image" /><meta content="Y terminamos el dia con Cultura de empresa con @patoroco, @flopezluis, Katia, Angélica en @codemotion_es #codemotion2017" name="description" /><meta content="SmartUA (@UaSmart) on X" name="title" /><meta content="twitter://status?id=934106870834454529" property="al:ios:url" /><meta content="333903271" property="al:ios:app_store_id" /><meta content="X" property="al:ios:app_name" /><meta content="twitter://status?id=934106870834454529" property="al:android:url" /><meta content="com.twitter.android" property="al:android:package" /><meta content="X" property="al:android:app_name" /></head><body style="background-color: #FFFFFF;"><div id="react-root" style="height:100%;display:flex;"></div>

packages/metascraper-x/test/fixtures/post-og.html

@@ -0,0 +1,2 @@
+<!DOCTYPE html><html dir="ltr" lang="en"><head><meta charset="utf-8" /><meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=0,viewport-fit=cover" /><link rel="preconnect" href="//abs.twimg.com" /><link rel="dns-prefetch" href="//abs.twimg.com" /><link rel="preconnect" href="//api.twitter.com" /><link rel="dns-prefetch" href="//api.twitter.com" /><link rel="preconnect" href="//api.x.com" /><link rel="dns-prefetch" href="//api.x.com" /><link rel="preconnect" href="//pbs.twimg.com" /><link rel="dns-prefetch" href="//pbs.twimg.com" /><link rel="preconnect" href="//t.co" /><link rel="dns-prefetch" href="//t.co" /><link rel="preconnect" href="//video.twimg.com" /><link rel="dns-prefetch" href="//video.twimg.com" /><meta http-equiv="onion-location" content="https://twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid.onion/" /><meta property="fb:app_id" content="2231777543" /><meta content="X (formerly Twitter)" property="og:site_name" /><meta name="google-site-verification" content="600dQ0pZYsH2xOFt4hYmf5f5NpjCbWE_qk5Y04dErYM" /><meta name="facebook-domain-verification" content="x6sdcc8b5ju3bh8nbm59eswogvg6t1" /><meta name="mobile-web-app-capable" content="yes" /><meta name="apple-mobile-web-app-title" content="Twitter" /><meta name="apple-mobile-web-app-status-bar-style" content="white" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X">
+  <link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="manifest" href="/manifest.json" crossOrigin="use-credentials" /><link rel="search" type="application/opensearchdescription+xml" href="/os-x.xml" title="X"><link rel="search" type="application/opensearchdescription+xml" href="/os-grok.xml" title="Grok"><link rel="shortcut icon" href="//abs.twimg.com/favicons/twitter.3.ico"><meta name="theme-color" media="(prefers-color-scheme: light)" content="#FFFFFF" /><meta name="theme-color" media="(prefers-color-scheme: dark)" content="#000000" /><script type="application/ld+json">{"@context":"https://schema.org","@type":"WebSite","url":"https://twitter.com/","potentialAction":{"@type":"SearchAction","query-input":"required name=search_term_string","target":{"@type":"EntryPoint","urlTemplate":"https://twitter.com/search?q={search_term_string}&ref_src=twcamp%5Eseo_searchbox%7Ctwsrc%5Eseo"}}}</script><meta http-equiv="origin-trial" content="AlpCmb40F5ZjDi9ZYe+wnr/V8MF+XmY41K4qUhoq+2mbepJTNd3q4CRqlACfnythEPZqcjryfAS1+ExS0FFRcA8AAABmeyJvcmlnaW4iOiJodHRwczovL3R3aXR0ZXIuY29tOjQ0MyIsImZlYXR1cmUiOiJMYXVuY2ggSGFuZGxlciIsImV4cGlyeSI6MTY1NTI1MTE5OSwiaXNTdWJkb21haW4iOnRydWV9" /><style>html,body{height: 100%;}::cue{white-space:normal}</style><meta content="article" property="og:type" /><meta content="https://twitter.com/realDonaldTrump/status/1222907250383245320" property="og:url" /><meta content="Donald J. Trump (@realDonaldTrump) on X" property="og:title" /><meta content="“Schiff blasted for not focusing on California homeless.” @foxandfriends  His District is in terrible shape. He is a corrupt pol who only dreams of the Impeachment Hoax. In my opinion he is mentally deranged!" property="og:description" /><meta content="https://pbs.twimg.com/profile_images/874276197357596672/kUuht00m_200x200.jpg" property="og:image" /><meta content="“Schiff blasted for not focusing on California homeless.” @foxandfriends  His District is in terrible shape. He is a corrupt pol who only dreams of the Impeachment Hoax. In my opinion he is mentally deranged!" name="description" /><meta content="Donald J. Trump (@realDonaldTrump) on X" name="title" /><meta content="twitter://status?id=1222907250383245320" property="al:ios:url" /><meta content="333903271" property="al:ios:app_store_id" /><meta content="X" property="al:ios:app_name" /><meta content="twitter://status?id=1222907250383245320" property="al:android:url" /><meta content="com.twitter.android" property="al:android:package" /><meta content="X" property="al:android:app_name" /></head><body style="background-color: #FFFFFF;"><div id="react-root" style="height:100%;display:flex;"></div>