Merge pull request #108 from ndaidong/v2.0.0

v2.0.0

Author: ndaidong

README.md

@@ -14,7 +14,6 @@ Extract eEmbed content from given URL.
 - [Changes with Instagram](#changes-with-instagram)
 
 
-
 ## Demo
 
 - [Give it a try!](https://ndaidong.github.io/oembed-parser-demo)
@@ -77,6 +76,29 @@ console.log(data)
 
 Return boolean. True if the URL matches with any provider in the list.
 
+#### .findProvider(String URL)
+
+Return provider which is relevant to given URL.
+
+For example:
+
+```js
+import {
+  findProvider
+} from 'oembed-parser'
+
+findProvider('https://www.facebook.com/video.php?v=999999999')
+
+// get something like below:
+
+// {
+//   fetchEndpoint: 'https://graph.facebook.com/v10.0/oembed_video',
+//   providerName: 'Facebook',
+//   providerUrl: 'https://www.facebook.com/'
+// }
+```
+
+
 #### .setProviderList(Array of provider definitions)
 
 Sets the list of providers to use, overriding the defaults.
@@ -93,17 +115,17 @@ For the expected format, see the
 List of resource providers is a clone of [oembed.com](http://oembed.com/providers.json) and available [here](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json).
 
 
-## Changes with Instagram
+## Facebook and Instagram
 
-Since October 24 2020, Facebook have deprecated their legacy urls and applied a new Facebook oEmbed endpoints. Please update your `oembed-parser` version to v1.4.2 to be able to extract Instagram links.
+Since October 24 2020, Facebook have deprecated their legacy urls and applied a new Facebook oEmbed endpoints.
+Please update your `oembed-parser` version to v1.4.2 or later to be able to extract oembed data from Instagram and Facebook.
 
-Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app. By default, `oembed-parser` build Graph API endpoint using a pre-existing access token. Althrough it should work in almost cases. However, we recommend to add your own ones.
+Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app.
 
 
-```
+```bash
 export FACEBOOK_APP_ID=your_app_id
 export FACEBOOK_CLIENT_TOKEN=your_client_token
-
 ```
 
 For more info, please refer:

eval.js

@@ -1,3 +1,6 @@
+// eval.js
+// to quickly test with a single url or file
+
 const { extract } = require('./src/main')
 
 const run = async (url) => {

index.d.ts

@@ -19,7 +19,7 @@ export interface Endpoint {
 }
 
 export interface Provider {
-    "provider_name": string;
+    "providerName": string;
     "provider_url": string;
     "endpoints": Endpoint[];
 }

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.5.2",
+  "version": "2.0.0rc1",
   "name": "oembed-parser",
   "description": "Get oEmbed data from given URL.",
   "homepage": "https://www.npmjs.com/package/oembed-parser",
@@ -9,6 +9,7 @@
   },
   "author": "@ndaidong",
   "main": "./index.js",
+  "types": "./index.d.ts",
   "engines": {
     "node": ">= 10.14.2"
   },
@@ -22,19 +23,18 @@
     "build": "tsc",
     "reset": "node reset"
   },
+  "dependencies": {
+    "got": "^11.8.3"
+  },
   "devDependencies": {
-    "jest": "^27.3.1",
-    "typescript": "^4.4.4"
+    "jest": "^27.4.3",
+    "nock": "^13.2.1"
   },
   "keywords": [
     "oembed",
     "extractor",
     "parser",
     "util"
   ],
-  "license": "MIT",
-  "types": "./index.d.ts",
-  "dependencies": {
-    "got": "^11.8.2"
-  }
+  "license": "MIT"
 }

src/config.js

@@ -0,0 +1,15 @@
+// config
+
+const fetchOptions = {
+  headers: {
+    'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0',
+    accept: 'application/json; charset=utf-8'
+  },
+  responseType: 'json',
+  timeout: 30 * 1e3,
+  redirect: 'follow'
+}
+
+module.exports = {
+  fetchOptions
+}

src/main.js

@@ -1,37 +1,28 @@
-// main
+/**
+ * oembed parser
+ * @ndaidong
+ **/
 
-const {
-  isValidURL,
-  findProvider,
-  fetchEmbed,
-  providersFromList
-} = require('./utils')
+const isValidURL = require('./utils/isValidURL')
+const fetchEmbed = require('./utils/fetchEmbed')
 
-const defaultProviderList = require('./utils/providers.json')
-let providers = providersFromList(defaultProviderList)
+const provider = require('./utils/provider')
 
 const extract = async (url, params = {}) => {
   if (!isValidURL(url)) {
     throw new Error('Invalid input URL')
   }
-  const p = findProvider(url, providers)
+  const p = provider.find(url)
   if (!p) {
     throw new Error(`No provider found with given url "${url}"`)
   }
   const data = await fetchEmbed(url, p, params)
   return data
 }
 
-const hasProvider = (url) => {
-  return findProvider(url, providers) !== null
-}
-
-const setProviderList = (list) => {
-  providers = providersFromList(list)
-}
-
 module.exports = {
   extract,
-  hasProvider,
-  setProviderList
+  hasProvider: provider.has,
+  findProvider: provider.find,
+  setProviderList: provider.set
 }

src/main.test.js

@@ -1,9 +1,12 @@
 // main
 /* eslint-env jest */
 
+const nock = require('nock')
+
 const {
   extract,
   hasProvider,
+  findProvider,
   setProviderList
 } = require('./main')
 
@@ -40,6 +43,14 @@ const InstagramKeys = [
   ...required
 ]
 
+const parseUrl = (url) => {
+  const re = new URL(url)
+  return {
+    baseUrl: `${re.protocol}//${re.host}`,
+    path: re.pathname
+  }
+}
+
 const hasProperty = (obj, key) => {
   return Object.prototype.hasOwnProperty.call(obj, key)
 }
@@ -60,9 +71,9 @@ const hasInstagramKeys = (o) => {
   return InstagramKeys.every((k) => {
     return hasProperty(o, k)
   })
-};
+}
 
-(() => {
+describe('test extract(bad url)', () => {
   const badSamples = [
     '',
     { k: 9 },
@@ -76,59 +87,124 @@ const hasInstagramKeys = (o) => {
     'https://soundcloud^(*%%$%^$$%$$*&(&)())'
   ]
 
-  const testBadOne = (url) => {
+  badSamples.forEach((url) => {
     test(`testing extract bad url "${url}"`, async () => {
       try {
         await extract(url)
       } catch (err) {
         expect(err).toBeTruthy()
       }
     })
-  }
-
-  return badSamples.map(testBadOne)
-})()
+  })
+})
 
 test('test extract YouTube link', async () => {
-  try {
-    const url = 'https://www.youtube.com/watch?v=ciS8aCrX-9s'
-    const result = await extract(url)
-    expect(hasRichKeys(result)).toBe(true)
-  } catch (err) {
-    expect(err).toBe(null)
-  }
+  const url = 'https://www.youtube.com/watch?v=ciS8aCrX-9s'
+  const provider = findProvider(url)
+  const { baseUrl, path } = parseUrl(provider.fetchEndpoint)
+  const scope = nock(baseUrl, { encodedQueryParams: true })
+  const params = new URLSearchParams({
+    url,
+    format: 'json'
+  })
+  scope.get(path)
+    .query(params)
+    .replyWithFile(200, './test-data/youtube_ciS8aCrX-9s.json', {
+      'Content-Type': 'application/json'
+    })
+  const result = await extract(url)
+  expect(hasRichKeys(result)).toBe(true)
+  expect(result.provider_name).toEqual('YouTube')
+  expect(result.type).toEqual('video')
 })
 
 test('test extract Flickr link', async () => {
-  try {
-    const url = 'https://flic.kr/p/2iYctUr'
-    const result = await extract(url)
-    expect(hasPhotoKeys(result)).toBe(true)
-  } catch (err) {
-    expect(err).toBe(null)
-  }
+  const url = 'https://flic.kr/p/2iYctUr'
+  const provider = findProvider(url)
+  const { baseUrl, path } = parseUrl(provider.fetchEndpoint)
+  const scope = nock(baseUrl, { encodedQueryParams: true })
+  const params = new URLSearchParams({
+    url,
+    format: 'json'
+  })
+  scope.get(path)
+    .query(params)
+    .replyWithFile(200, './test-data/flickr_2iYctUr.json', {
+      'Content-Type': 'application/json'
+    })
+  const result = await extract(url)
+  expect(hasPhotoKeys(result)).toBe(true)
+  expect(result.provider_name).toEqual('Flickr')
+  expect(result.type).toEqual('photo')
+  expect(result.width).toEqual(1024)
+  expect(result.height).toEqual(768)
 })
 
 test('test extract Flickr link with params', async () => {
-  try {
-    const url = 'https://flic.kr/p/2iYctUr'
-    const result = await extract(url, { maxwidth: 640, maxheight: 480 })
-    expect(hasPhotoKeys(result)).toBe(true)
-  } catch (err) {
-    expect(err).toBe(null)
-  }
+  const url = 'https://flic.kr/p/2iYctUr'
+  const provider = findProvider(url)
+  const { baseUrl, path } = parseUrl(provider.fetchEndpoint)
+  const scope = nock(baseUrl, { encodedQueryParams: true })
+  const params = new URLSearchParams({
+    url,
+    maxwidth: 640,
+    maxheight: 480,
+    format: 'json'
+  })
+  scope.get(path)
+    .query(params)
+    .replyWithFile(200, './test-data/flickr_2iYctUr_640x480.json', {
+      'Content-Type': 'application/json'
+    })
+  const result = await extract(url, { maxwidth: 640, maxheight: 480 })
+  expect(hasPhotoKeys(result)).toBe(true)
+  expect(result.provider_name).toEqual('Flickr')
+  expect(result.type).toEqual('photo')
+  expect(result.width).toBeLessThanOrEqual(640)
+  expect(result.height).toBeLessThanOrEqual(480)
 })
 
 test('test extract Instagram link', async () => {
-  try {
-    const url = 'https://www.instagram.com/p/ic7kRDqOlt/'
-    const result = await extract(url)
-    expect(hasInstagramKeys(result)).toBe(true)
-  } catch (err) {
-    // could not wait for reviewing 'Oembed Read' feature
-    // https://developers.facebook.com/docs/apps/review
-    expect(err).toBeTruthy()
-  }
+  const url = 'https://www.instagram.com/p/ic7kRDqOlt/'
+  const provider = findProvider(url)
+  const { baseUrl, path } = parseUrl(provider.fetchEndpoint)
+  const scope = nock(baseUrl, { encodedQueryParams: true })
+  const params = new URLSearchParams({
+    url,
+    format: 'json',
+    access_token: '845078789498971|8ff3ab4ddd45b8f018b35c4fb7edac62'
+  })
+  scope.get(path)
+    .query(params)
+    .replyWithFile(200, './test-data/instagram_ic7kRDqOlt.json', {
+      'Content-Type': 'application/json'
+    })
+  const result = await extract(url)
+  expect(hasInstagramKeys(result)).toBe(true)
+  expect(result.provider_name).toEqual('Instagram')
+  expect(result.type).toEqual('rich')
+})
+
+test('test extract Facebook video', async () => {
+  const url = 'https://www.facebook.com/facebook/videos/10153231379946729/'
+  const provider = findProvider(url)
+  const { baseUrl, path } = parseUrl(provider.fetchEndpoint)
+  const scope = nock(baseUrl, { encodedQueryParams: true })
+  const params = new URLSearchParams({
+    url,
+    format: 'json',
+    access_token: '845078789498971|8ff3ab4ddd45b8f018b35c4fb7edac62'
+  })
+  scope.get(path)
+    .query(params)
+    .replyWithFile(200, './test-data/facebook.json', {
+      'Content-Type': 'application/json'
+    })
+  const result = await extract(url)
+  expect(hasRichKeys(result)).toBe(true)
+  expect(result).toBeTruthy()
+  expect(result.provider_name).toEqual('Facebook')
+  expect(result.type).toEqual('video')
 })
 
 test('test .hasProvider() method', () => {