Merge pull request #111 from ndaidong/v2.0.0

v2.0.0

Author: ndaidong

README.md

@@ -1,3 +1,4 @@
+
 # oembed-parser
 
 Extract eEmbed content from given URL.
@@ -15,19 +16,28 @@ Extract eEmbed content from given URL.
 - [Example FaaS](https://us-central1-technews-251304.cloudfunctions.net/oembed-parser?url=https://www.youtube.com/watch?v=8jPQjjsBbIc)
 
 
-### Installation
+## Installation
 
 ```bash
-npm install oembed-parser
+$ npm install oembed-parser
+
+# or pnpm
+$ pnpm install oembed-parser
+
+# or yarn
+$ yarn add oembed-parser
 ```
 
-### Usage
+
+## Usage
 
 ```js
-import {
-  extract
-} from 'oembed-parser'
+const { extract } = require('oembed-parser')
 
+// es6 module syntax
+import { extract } from 'oembed-parser'
+
+// test
 const url = 'https://www.youtube.com/watch?v=8jPQjjsBbIc'
 
 extract(url).then((oembed) => {
@@ -37,96 +47,139 @@ extract(url).then((oembed) => {
 })
 ```
 
-### APIs
+## APIs
 
 #### .extract(String url [, Object params])
 
-Extract oEmbed data from specified url.
-Return: a Promise
-
-Optional argument `params` is an object with it we can set `maxwidth` and/or `maxheight` those are used to scale embed size to fit your container size. Please refer [oEmbed/Full Spec/Consumer Request](https://oembed.com/#section2) for more info.
+Load and extract oembed data.
 
-Here is how we can use `oembed-parser` in async/await style:
+Example:
 
 ```js
-import {
-  extract
-} from 'oembed-parser'
+const { extract } = require('oembed-parser')
 
 const getOembed = async (url) => {
   try {
     const oembed = await extract(url)
     return oembed
   } catch (err) {
     console.trace(err)
+  return null
   }
 }
 
 const data = getOembed('your url')
 console.log(data)
 ```
 
+Optional argument `params` is an object with it we can set `maxwidth` and/or `maxheight` those are used to scale embed size to fit your container size. Please refer [oEmbed/Full Spec/Consumer Request](https://oembed.com/#section2) for more info.
 
 #### .hasProvider(String URL)
 
-Return boolean. True if the URL matches with any provider in the list.
+Check if a URL matches with any provider in the list.
+
+Examples:
+
+```js
+const { hasProvider } = require('oembed-parser')
+
+hasProvider('https://www.youtube.com/watch?v=ciS8aCrX-9s') // return true
+hasProvider('https://trello.com/b/BO3bg7yn/notes') // return false
+```
 
 #### .findProvider(String URL)
 
-Return provider which is relevant to given URL.
+Get the provider which is relevant to given URL.
 
 For example:
 
 ```js
-import {
-  findProvider
-} from 'oembed-parser'
+const { findProvider } = require('oembed-parser')
 
 findProvider('https://www.facebook.com/video.php?v=999999999')
+```
 
-// get something like below:
+Result looks like below:
 
-// {
-//   fetchEndpoint: 'https://graph.facebook.com/v10.0/oembed_video',
-//   providerName: 'Facebook',
-//   providerUrl: 'https://www.facebook.com/'
-// }
+```json
+{
+  fetchEndpoint: 'https://graph.facebook.com/v10.0/oembed_video',
+  providerName: 'Facebook',
+  providerUrl: 'https://www.facebook.com/'
+}
 ```
 
+#### .setProviderList(Array providers)
 
-#### .setProviderList(Array of provider definitions)
-
-Sets the list of providers to use, overriding the defaults.
+Apply a list of providers to use, overriding the [default](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json).
 
 This can be useful for whitelisting only certain providers, or for adding
 custom providers.
 
-For the expected format, see the
-[default list](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json).
+Default list of resource providers is synchronized from [oembed.com](http://oembed.com/providers.json).
 
+For example:
+
+```js
+const { setProviderList } = require('oembed-parser')
+
+const providers = [
+  {
+    provider_name: 'Alpha',
+    provider_url: 'https://alpha.com',
+    endpoints: [
+      // endpoint definition here
+    ]
+  },
+  {
+    provider_name: 'Beta',
+    provider_url: 'https://beta.com',
+    endpoints: [
+      // endpoint definition here
+    ]
+  }
+]
 
-### Provider list
+setProviderList(providers)
+```
+
+#### .setRequestOptions(Object requestOptions)
+Define options to call oembed HTTP request.
 
-List of resource providers is a clone of [oembed.com](http://oembed.com/providers.json) and available [here](https://raw.githubusercontent.com/ndaidong/oembed-parser/master/src/utils/providers.json).
+`oembed-parser` is using [axios]() to send HTTP requests. Please refer [axios' request config](https://axios-http.com/docs/req_config) for more info.
+
+Default option:
+
+```js
+{
+  headers: {
+    'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0',
+    accept: 'application/json; charset=utf-8'
+  },
+  responseType: 'json',
+  responseEncoding: 'utf8',
+  timeout: 6e4,
+  maxRedirects: 3
+}
+```
 
 
 ## Facebook and Instagram
 
 Since October 24 2020, Facebook have deprecated their legacy urls and applied a new Facebook oEmbed endpoints.
-Please update your `oembed-parser` version to v1.4.2 or later to be able to extract oembed data from Instagram and Facebook.
-
-Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app.
 
+Technically, now we have to use Facebook Graph API, with the access token from a valid and live Facebook app. `oembed-parser` will try to get these values from environment variables, so please define them, for example:
 
 ```bash
 export FACEBOOK_APP_ID=your_app_id
 export FACEBOOK_CLIENT_TOKEN=your_client_token
 ```
 
-For more info, please refer:
+References:
 
+- [oEmbed Read](https://developers.facebook.com/docs/features-reference/oembed-read)
 - [Facebook oEmbed](https://developers.facebook.com/docs/plugins/oembed)
-
+- [Facebook Graph API](https://developers.facebook.com/docs/graph-api/overview)
 
 ## Test
 
@@ -140,6 +193,7 @@ npm test
 npm run eval {URL_TO_PARSE_OEMBED}
 ```
 
-# License
-
+## License
 The MIT License (MIT)
+
+---

index.d.ts

@@ -7,10 +7,14 @@
 
 export function extract(url: string, params?: any): Promise<OembedData>;
 
-export function hasProvider(url: string): boolean;
+export function hasProvider(url: string): boolean
 
 export function setProviderList(providers: Provider[]): void
 
+export function setRequestOptions(options: object): void
+
+export function getRequestOptions(): object
+
 export interface Endpoint {
     schemes?: string[];
     url: string;

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.0.0rc2",
+  "version": "2.0.0",
   "name": "oembed-parser",
   "description": "Get oEmbed data from given URL.",
   "homepage": "https://www.npmjs.com/package/oembed-parser",
@@ -24,10 +24,11 @@
     "reset": "node reset"
   },
   "dependencies": {
-    "got": "^11.8.3"
+    "axios": "^0.24.0",
+    "bellajs": "^10.0.2"
   },
   "devDependencies": {
-    "jest": "^27.4.3",
+    "jest": "^27.4.5",
     "nock": "^13.2.1"
   },
   "keywords": [

reset.js

@@ -18,6 +18,7 @@ const dirs = [
 
 const files = [
   'yarn.lock',
+  'pnpm-lock.yaml',
   'package-lock.json',
   'coverage.lcov',
   'coverage.lcov'

src/config.js

@@ -1,15 +1,23 @@
 // config
 
-const fetchOptions = {
+const { clone, copies } = require('bellajs')
+
+const requestOptions = {
   headers: {
     'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0',
     accept: 'application/json; charset=utf-8'
   },
   responseType: 'json',
-  timeout: 30 * 1e3,
-  redirect: 'follow'
+  responseEncoding: 'utf8',
+  timeout: 6e4, // 1 minute
+  maxRedirects: 3
 }
 
 module.exports = {
-  fetchOptions
+  getRequestOptions: () => {
+    return clone(requestOptions)
+  },
+  setRequestOptions: (opts) => {
+    copies(opts, requestOptions)
+  }
 }

src/main.js

@@ -8,6 +8,10 @@ const fetchEmbed = require('./utils/fetchEmbed')
 
 const provider = require('./utils/provider')
 
+const {
+  setRequestOptions
+} = require('./config')
+
 const extract = async (url, params = {}) => {
   if (!isValidURL(url)) {
     throw new Error('Invalid input URL')
@@ -24,5 +28,6 @@ module.exports = {
   extract,
   hasProvider: provider.has,
   findProvider: provider.find,
-  setProviderList: provider.set
+  setProviderList: provider.set,
+  setRequestOptions
 }

src/main.test.js

@@ -7,9 +7,14 @@ const {
   extract,
   hasProvider,
   findProvider,
-  setProviderList
+  setProviderList,
+  setRequestOptions
 } = require('./main')
 
+const {
+  getRequestOptions
+} = require('./config')
+
 const required = [
   'type',
   'version'
@@ -251,3 +256,23 @@ test('test .setProviderList() method', () => {
   expect(hasProvider('http://www.example.org/media/abcdef')).toBe(true)
   expect(hasProvider('https://www.youtube.com/watch?v=ciS8aCrX-9s')).toBe(false)
 })
+
+test('Testing setRequestOptions()', () => {
+  setRequestOptions({
+    headers: {
+      authorization: 'bearer <token>'
+    },
+    timeout: 20,
+    somethingElse: 1000
+  })
+
+  const actual = getRequestOptions()
+  const expectedHeader = {
+    authorization: 'bearer <token>',
+    'user-agent': 'Mozilla/5.0 (X11; Linux i686; rv:94.0) Gecko/20100101 Firefox/94.0',
+    accept: 'application/json; charset=utf-8'
+  }
+
+  expect(actual.headers).toEqual(expectedHeader)
+  expect(actual.timeout).toEqual(20)
+})

src/utils/retrieve.js

@@ -1,19 +1,19 @@
 // utils -> retrieve
 
-const got = require('got')
+const axios = require('axios')
 
-const { fetchOptions } = require('../config')
+const { getRequestOptions } = require('../config')
 
 module.exports = async (url) => {
   try {
-    const { headers, body } = await got(url, fetchOptions)
+    const res = await axios.get(url, getRequestOptions())
 
-    const contentType = headers['content-type'] || ''
+    const contentType = res.headers['content-type'] || ''
     if (!contentType || !contentType.includes('application/json')) {
       return null
     }
 
-    return body
+    return res.data
   } catch (err) {
     return null
   }

src/utils/retrieve.test.js

@@ -28,7 +28,7 @@ test('test retrieve() from bad source', async () => {
   const url = 'https://some.where/good/page'
   const { baseUrl, path } = parseUrl(url)
   const scope = nock(baseUrl)
-  scope.get(path).reply(500, { data: { name: 'oembed-parser' } }, {
+  scope.get(path).reply(500, '', {
     'Content-Type': 'application/json'
   })
   const result = await retrieve(url)