Merge pull request #185 from extractus/4.0.5

ndaidong · web-flow · commit 0edfc1f4e346 · 2024-05-07T17:47:39.000+07:00
v4.0.5
diff --git a/README.md b/README.md
@@ -29,7 +29,6 @@ yarn add @extractus/oembed-extractor
 ```
 
 ```ts
-// es6 module
 import { extract } from '@extractus/oembed-extractor'
 
 const result = await extract('https://www.youtube.com/watch?v=x2bqscVkGxk')
@@ -39,10 +38,6 @@ console.log(result)
 ### Deno
 
 ```ts
-// deno < 1.28
-import { extract } from 'https://esm.sh/@extractus/oembed-extractor'
-
-// deno > 1.28
 import { extract } from 'npm:@extractus/oembed-extractor'
 ```
 
diff --git a/deno.json b/deno.json
diff --git a/examples/browser-oembed-parser/package.json b/examples/browser-oembed-parser/package.json
@@ -6,7 +6,7 @@
     "start": "node server"
   },
   "dependencies": {
-    "express": "^4.18.2",
-    "got": "^13.0.0"
+    "express": "latest",
+    "got": "latest"
   }
 }
diff --git a/examples/bun-oembed-parser/package.json b/examples/bun-oembed-parser/package.json
@@ -5,10 +5,10 @@
     "start": "bun run index.ts"
   },
   "devDependencies": {
-    "bun-types": "^0.6.13"
+    "bun-types": "latest"
   },
   "dependencies": {
-    "hono": "^3.2.7",
+    "hono": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }
diff --git a/examples/deno-oembed-parser/index.ts b/examples/deno-oembed-parser/index.ts
@@ -1,6 +1,6 @@
 import { serve } from 'https://deno.land/std/http/server.ts'
 
-import { Hono } from 'https://deno.land/x/hono@v3.2.7/mod.ts'
+import { Hono } from 'https://deno.land/x/hono/mod.ts'
 
 import { extract } from 'npm:@extractus/oembed-extractor'
 
diff --git a/examples/node-oembed-parser/package.json b/examples/node-oembed-parser/package.json
@@ -7,7 +7,7 @@
     "start": "node index.js"
   },
   "dependencies": {
-    "express": "^4.18.2",
+    "express": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }
diff --git a/examples/tsnode-oembed-parser/package.json b/examples/tsnode-oembed-parser/package.json
@@ -7,10 +7,10 @@
     "start": "node dist/index.js"
   },
   "devDependencies": {
-    "typescript": "^5.1.6"
+    "typescript": "latest"
   },
   "dependencies": {
-    "express": "^4.18.2",
+    "express": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }
diff --git a/package.json b/package.json
@@ -1,5 +1,5 @@
 {
-  "version": "4.0.4",
+  "version": "4.0.5",
   "name": "@extractus/oembed-extractor",
   "description": "Get oEmbed data from given URL.",
   "homepage": "https://github.com/extractus/oembed-extractor",
@@ -11,10 +11,12 @@
   "main": "./src/main.js",
   "type": "module",
   "imports": {
-    "cross-fetch": "./src/deno/cross-fetch.js"
+    "cross-fetch": "./src/deno/cross-fetch.js",
+    "linkedom": "https://deno.land/x/deno_dom@v0.1.45/deno-dom-wasm.ts"
   },
   "browser": {
-    "cross-fetch": "./src/deno/cross-fetch.js"
+    "cross-fetch": "./src/deno/cross-fetch.js",
+    "linkedom": "./src/browser/linkedom.js"
   },
   "types": "./index.d.ts",
   "engines": {
@@ -30,11 +32,12 @@
     "reset": "node reset"
   },
   "dependencies": {
-    "cross-fetch": "^4.0.0"
+    "cross-fetch": "^4.0.0",
+    "linkedom": "^0.16.11"
   },
   "devDependencies": {
-    "eslint": "^9.1.1",
-    "globals": "^15.0.0",
+    "eslint": "^9.2.0",
+    "globals": "^15.1.0",
     "https-proxy-agent": "^7.0.4",
     "jest": "^29.7.0",
     "nock": "^13.5.4"
diff --git a/src/browser/linkedom.js b/src/browser/linkedom.js
@@ -0,0 +1 @@
+export const DOMParser = window.DOMParser
diff --git a/src/main.js b/src/main.js
@@ -1,6 +1,7 @@
 // main.js
 
 import { isValid as isValidURL } from './utils/linker.js'
+import extractWithDiscovery from './utils/autoDiscovery.js'
 import fetchEmbed from './utils/fetchEmbed.js'
 
 import { getEndpoint } from './utils/provider.js'
@@ -10,12 +11,10 @@ export const extract = async (url, params = {}, options = {}) => {
     throw new Error('Invalid input URL')
   }
   const endpoint = getEndpoint(url)
-  if (!endpoint) {
-    throw new Error(`No provider found with given url "${url}"`)
-  }
 
-  const data = await fetchEmbed(url, params, endpoint, options)
-  return data
+  return endpoint
+    ? fetchEmbed(url, params, endpoint, options)
+    : extractWithDiscovery(url, params, options)
 }
 
 export {
diff --git a/src/utils/autoDiscovery.js b/src/utils/autoDiscovery.js
@@ -0,0 +1,23 @@
+// utils -> autoDiscovery.js
+
+import { DOMParser } from 'linkedom'
+
+import { getHtml, getJson } from './retrieve.js'
+
+export default async (url, params = {}, options = {}) => {
+  const html = await getHtml(url, options)
+  const doc = new DOMParser().parseFromString(html, 'text/html')
+  const elm = doc.querySelector('link[type="application/json+oembed"]')
+  const href = elm.getAttribute('href')
+  const q = new URL(href)
+  const { origin, pathname, searchParams } = q
+  Object.keys(params).forEach((key) => {
+    if (!searchParams.has(key)) {
+      searchParams.append(key, params[key])
+    }
+  })
+  const link = `${origin}${pathname}?${searchParams.toString()}`
+  const body = await getJson(link, options)
+  body.method = 'auto-discovery'
+  return body
+}
diff --git a/src/utils/autoDiscovery.test.js b/src/utils/autoDiscovery.test.js
@@ -0,0 +1,53 @@
+// autoDiscovery.test
+/* eslint-env jest */
+
+import nock from 'nock'
+
+import autoDiscovery from './autoDiscovery.js'
+
+const parseUrl = (url) => {
+  const re = new URL(url)
+  return {
+    baseUrl: `${re.protocol}//${re.host}`,
+    path: re.pathname,
+  }
+}
+
+describe('test if autoDiscovery() works correctly', () => {
+  const url = 'https://www.bitchute.com/video/8hXWnkvA8Ao/'
+  test(`check fetchEmbed("${url}")`, async () => {
+    const htmlFile = './test-data/bitchute.html'
+    const jsonFile = './test-data/bitchute.json'
+
+    const { baseUrl, path } = parseUrl(url)
+    const scope = nock(baseUrl)
+    scope.get(path)
+      .replyWithFile(200, htmlFile, {
+        'Content-Type': 'text/html',
+      })
+
+    const endpoint = 'https://www.bitchute.com/oembed/'
+    const { baseUrl: endpointBaseUrl, path: endpointPath } = parseUrl(endpoint)
+
+    const params = {
+      maxwidth: 600,
+      maxheight: 400,
+    }
+
+    const jsonScope = nock(endpointBaseUrl, { encodedQueryParams: true })
+    const queries = new URLSearchParams({
+      url: 'https://www.bitchute.com/video/8hXWnkvA8Ao/',
+      format: 'json',
+      ...params,
+    })
+    jsonScope.get(endpointPath)
+      .query(queries)
+      .replyWithFile(200, jsonFile, {
+        'Content-Type': 'application/json',
+      })
+
+    const result = await autoDiscovery(url, params)
+    expect(result).toBeTruthy()
+    nock.cleanAll()
+  })
+})
diff --git a/src/utils/fetchEmbed.js b/src/utils/fetchEmbed.js
@@ -1,6 +1,6 @@
 // utils -> fetchEmbed
 
-import retrieve from './retrieve.js'
+import { getJson } from './retrieve.js'
 import { getDomain } from './linker.js'
 
 const isFacebookGraphDependent = (url) => {
@@ -34,6 +34,7 @@ export default async (url, params = {}, endpoint = '', options = {}) => { // esl
 
   const queryParams = new URLSearchParams(query).toString()
   const link = endpoint + '?' + queryParams
-  const body = retrieve(link, options)
+  const body = await getJson(link, options)
+  body.method = 'provider-api'
   return body
 }
diff --git a/src/utils/providers.latest.js b/src/utils/providers.latest.js
@@ -1,4 +1,4 @@
-// provider data, synchronized at 2024-04-26T08:46:02.055Z
+// provider data, synchronized at 2024-05-07T10:41:10.221Z
 
 /* eslint-disable */ 
 
@@ -198,6 +198,12 @@ export const providers = [
     ],
     "e": "www.behance.net/services/oembed"
   },
+  {
+    "s": [
+      "cloud\\.biqapp\\.com/*"
+    ],
+    "e": "biqapp.com/api/v1/video/oembed"
+  },
   {
     "s": [
       "blackfire\\.io/profiles/*/graph",
@@ -277,6 +283,12 @@ export const providers = [
     ],
     "e": "img.catbo.at/oembed.json"
   },
+  {
+    "s": [
+      "embeds\\.celero\\.io/*"
+    ],
+    "e": "api.celero.io/api/oembed"
+  },
   {
     "s": [
       "view\\.ceros\\.com/*"
@@ -816,6 +828,12 @@ export const providers = [
     ],
     "e": "www.hulu.com/api/oembed.json"
   },
+  {
+    "s": [
+      "oembed\\.ideamapper\\.com/*"
+    ],
+    "e": "oembed.ideamapper.com"
+  },
   {
     "s": [
       "*\\.idomoo\\.com/*"
diff --git a/src/utils/providers.orginal.json b/src/utils/providers.orginal.json
@@ -415,6 +415,19 @@
       }
     ]
   },
+  {
+    "provider_name": "biqnetwork",
+    "provider_url": "https://biqapp.com/",
+    "endpoints": [
+      {
+        "schemes": [
+          "https://cloud.biqapp.com/*"
+        ],
+        "url": "https://biqapp.com/api/v1/video/oembed",
+        "discovery": true
+      }
+    ]
+  },
   {
     "provider_name": "Blackfire.io",
     "provider_url": "https://blackfire.io",
@@ -591,6 +604,22 @@
       }
     ]
   },
+  {
+    "provider_name": "Celero",
+    "provider_url": "https://www.celero.io",
+    "endpoints": [
+      {
+        "schemes": [
+          "https://embeds.celero.io/*"
+        ],
+        "url": "https://api.celero.io/api/oembed",
+        "discovery": true,
+        "formats": [
+          "json"
+        ]
+      }
+    ]
+  },
   {
     "provider_name": "Ceros",
     "provider_url": "http://www.ceros.com/",
@@ -1698,6 +1727,19 @@
       }
     ]
   },
+  {
+    "provider_name": "Ideamapper",
+    "provider_url": "http://oembed.ideamapper.com",
+    "endpoints": [
+      {
+        "schemes": [
+          "http://oembed.ideamapper.com/*"
+        ],
+        "url": "http://oembed.ideamapper.com",
+        "discovery": true
+      }
+    ]
+  },
   {
     "provider_name": "Idomoo",
     "provider_url": "https://idomoo.com/",
diff --git a/src/utils/providers.prev.js b/src/utils/providers.prev.js
diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js
diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js
diff --git a/sync.js b/sync.js
diff --git a/test-data/bitchute.html b/test-data/bitchute.html
diff --git a/test-data/bitchute.json b/test-data/bitchute.json

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`"start": "node server"`
`7`	`7`	`},`
`8`	`8`	`"dependencies": {`
`9`		`- "express": "^4.18.2",`
`10`		`- "got": "^13.0.0"`
	`9`	`+ "express": "latest",`
	`10`	`+ "got": "latest"`
`11`	`11`	`}`
`12`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,10 +5,10 @@`
`5`	`5`	`"start": "bun run index.ts"`
`6`	`6`	`},`
`7`	`7`	`"devDependencies": {`
`8`		`- "bun-types": "^0.6.13"`
	`8`	`+ "bun-types": "latest"`
`9`	`9`	`},`
`10`	`10`	`"dependencies": {`
`11`		`- "hono": "^3.2.7",`
	`11`	`+ "hono": "latest",`
`12`	`12`	`"@extractus/oembed-extractor": "latest"`
`13`	`13`	`}`
`14`	`14`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@`
`7`	`7`	`"start": "node index.js"`
`8`	`8`	`},`
`9`	`9`	`"dependencies": {`
`10`		`- "express": "^4.18.2",`
	`10`	`+ "express": "latest",`
`11`	`11`	`"@extractus/oembed-extractor": "latest"`
`12`	`12`	`}`
`13`	`13`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,10 +7,10 @@`
`7`	`7`	`"start": "node dist/index.js"`
`8`	`8`	`},`
`9`	`9`	`"devDependencies": {`
`10`		`- "typescript": "^5.1.6"`
	`10`	`+ "typescript": "latest"`
`11`	`11`	`},`
`12`	`12`	`"dependencies": {`
`13`		`- "express": "^4.18.2",`
	`13`	`+ "express": "latest",`
`14`	`14`	`"@extractus/oembed-extractor": "latest"`
`15`	`15`	`}`
`16`	`16`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+export const DOMParser = window.DOMParser`