chore(helpers): handle break line on jsonld (microlinkhq#793)

* chore(helpers): handle break line on jsonld

* test: update snapshot

* ci(fix): run package tests

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

* test: update snapshot

Author: Kikobeats

.github/workflows/get-matrix.mjs

@@ -12,8 +12,11 @@ for (const packagePath of await readdir(packagesPath)) {
   const pkg = join(packagesPath, packagePath, 'package.json')
   const { name, scripts } = JSON.parse(await readFile(pkg))
   if (scripts && scripts.test && scripts.test !== 'exit 0') {
-    packages.push(name)
+    packages.push({
+      name,
+      filter: `./packages/${packagePath}`
+    })
   }
 }
 
-console.log(`{"package":${JSON.stringify(packages)}}`)
+console.log(JSON.stringify({ package: packages }))

.github/workflows/main.yml

@@ -42,6 +42,7 @@ jobs:
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
   test:
+    name: Test ${{ matrix.package.name }}
     if: |
       !startsWith(github.event.head_commit.message, 'chore(release):') &&
       !startsWith(github.event.head_commit.message, 'docs:') &&
@@ -64,15 +65,15 @@ jobs:
           version: latest
           run_install: true
       - name: Test
-        run: pnpm --filter "${{ matrix.package }}" exec c8 pnpm test
+        run: pnpm --filter "${{ matrix.package.filter }}" exec c8 pnpm test
       - name: Coverage
-        run: pnpm --filter "${{ matrix.package }}" exec c8 report --reporter=lcov --report-dir=coverage
+        run: pnpm --filter "${{ matrix.package.filter }}" exec c8 report --reporter=lcov --report-dir=coverage
       - name: Upload
         uses: coverallsapp/github-action@main
         with:
-          flag-name: ${{ matrix.package }}
+          flag-name: ${{ matrix.package.name }}
           parallel: true
-          file: $(pnpm --filter "${{ matrix.package }}" exec pwd)/coverage/lcov.info
+          file: $(pnpm --filter "${{ matrix.package.filter }}" exec pwd)/coverage/lcov.info
 
   finish:
     needs: test

.github/workflows/pull_request.yml

@@ -21,6 +21,7 @@ jobs:
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
   test:
+    name: Test ${{ matrix.package.name }}
     needs: matrix
     runs-on: ubuntu-latest
     strategy:
@@ -39,15 +40,15 @@ jobs:
           version: latest
           run_install: true
       - name: Test
-        run: pnpm --filter "${{ matrix.package }}" exec c8 pnpm test
+        run: pnpm --filter "${{ matrix.package.filter }}" exec c8 pnpm test
       - name: Coverage
-        run: pnpm --filter "${{ matrix.package }}" exec c8 report --reporter=lcov --report-dir=coverage
+        run: pnpm --filter "${{ matrix.package.filter }}" exec c8 report --reporter=lcov --report-dir=coverage
       - name: Upload
         uses: coverallsapp/github-action@main
         with:
-          flag-name: ${{ matrix.package }}
+          flag-name: ${{ matrix.package.name }}
           parallel: true
-          file: $(pnpm --filter "${{ matrix.package }}" exec pwd)/coverage/lcov.info
+          file: $(pnpm --filter "${{ matrix.package.filter }}" exec pwd)/coverage/lcov.info
 
   finish:
     needs: test

packages/metascraper-helpers/package.json

@@ -36,6 +36,7 @@
     "iso-639-3": "~2.2.0",
     "isostring": "0.0.1",
     "jsdom": "~27.4.0",
+    "jsonrepair": "~3.13.2",
     "lodash": "~4.17.23",
     "memoize-one": "~6.0.0",
     "microsoft-capitalize": "~1.0.6",

packages/metascraper-helpers/src/index.js

@@ -8,6 +8,7 @@ const capitalize = require('microsoft-capitalize')
 const isRelativeUrl = require('is-relative-url')
 const fileExtension = require('file-extension')
 const _normalizeUrl = require('normalize-url')
+const { jsonrepair } = require('jsonrepair')
 const smartquotes = require('smartquotes')
 const { decodeHTML } = require('entities')
 const iso6393 = require('iso-639-3/to-1')
@@ -354,19 +355,29 @@ memoizeOne.EqualityUrlAndHtmlDom = (newArgs, oldArgs) =>
 memoizeOne.EqualityFirstArgument = (newArgs, oldArgs) =>
   newArgs[0] === oldArgs[0]
 
+const parseJSON = text => {
+  try {
+    return JSON.parse(text)
+  } catch {
+    try {
+      return JSON.parse(jsonrepair(text))
+    } catch {
+      return undefined
+    }
+  }
+}
+
 const jsonld = memoizeOne(
   $ =>
     $('script[type="application/ld+json"]')
       .map((_, element) => {
-        try {
-          const el = $(element)
-          const json = JSON.parse($(el).contents().text())
-          const { '@graph': graph, ...props } = json
-          if (!graph) return json
-          return graph.map(item => ({ ...props, ...item }))
-        } catch (_) {
-          return undefined
-        }
+        const el = $(element)
+        const text = $(el).contents().text()
+        const json = parseJSON(text)
+        if (!json) return undefined
+        const { '@graph': graph, ...props } = json
+        if (!graph) return json
+        return graph.map(item => ({ ...props, ...item }))
       })
       .get()
       .filter(Boolean),

packages/metascraper-helpers/test/index.js

@@ -381,12 +381,32 @@ test('.description', t => {
 })
 
 test('.$jsonld', t => {
-  const $ = cheerio.load(`
+  {
+    const $ = cheerio.load(`
       <script type="application/ld+json">{ "offers": { "price": 119.99 }}</script>
       <script type="application/ld+json">{ "offers": { "price": "" }}</script>
       `)
-  const value = $jsonld('offers.price')($)
-  t.is(value, 119.99)
+    const value = $jsonld('offers.price')($)
+    t.is(value, 119.99)
+  }
+  {
+    const $ = cheerio.load('<script type="application/ld+json">{{</script>')
+    const value = $jsonld('offers.price')($)
+    t.is(value, undefined)
+  }
+  {
+    const $ =
+      cheerio.load(`<script type="application/ld+json">{"@context":"https://schema.org","mainEntity":{"description":"This is an example
+
+🌐 of a multiline description
+📬 to see how it is parsed
+📧 and how it is decoded"}}</script>`)
+    const value = $jsonld('mainEntity.description')($)
+    t.is(
+      value,
+      'This is an example\n\n🌐 of a multiline description\n📬 to see how it is parsed\n📧 and how it is decoded'
+    )
+  }
 })
 
 test('.lang', t => {

packages/metascraper-helpers/test/jsonld.js

@@ -51,7 +51,8 @@ test('ensure to return first value', t => {
 })
 
 test('reads multiple JSON-LD blocks', t => {
-  const $ = cheerio.load(`<script type="application/ld+json"> { "@context": "http://schema.org", "@type": "Organization", "url": "https://bykvu.com/ru", "logo": "https://bykvu.com/wp-content/themes/bykvu/img/logo.svg" } </script>
+  const $ =
+    cheerio.load(`<script type="application/ld+json"> { "@context": "http://schema.org", "@type": "Organization", "url": "https://bykvu.com/ru", "logo": "https://bykvu.com/wp-content/themes/bykvu/img/logo.svg" } </script>
         <script type="application/ld+json"> { "@context": "http://schema.org", "@type": "NewsArticle", "mainEntityOfPage": { "@type": "WebPage", "@id": "https://bykvu.com/ru/bukvy/uchenye-nazvali-depressiju-prichinoj-22-opasnyh-zabolevanij/" }, "headline": "Ученые назвали депрессию причиной 22 опасных заболеваний", "image": [ "https://bykvu.com/wp-content/themes/bykvu/includes/images/noimage_large.jpg" ], "datePublished": "2019-09-09T00:29:09+02:00", "dateModified": "2019-09-09T00:29:09+02:00", "author": { "@type": "Person", "name": "Буквы" }, "publisher": { "@type": "Organization", "name": "Буквы", "logo": { "@type": "ImageObject", "url": "https://bykvu.com/wp-content/themes/bykvu/img/apple-icon-180x180.png" } }, "description": "Ученые австралийского центра точного здравоохранения при Университете Южной Австралии выяснили, что депрессия является причиной 22 различных заболеваний." } </script>
         <script type="application/ld+json"> { "@context": "https://schema.org", "@type": "BreadcrumbList", "itemListElement": [ { "@type": "ListItem", "position": 1, "item": { "@id": "https://bykvu.com/ru", "name": "Буквы" } }, { "@type": "ListItem", "position": 2, "item": { "@id": "https://bykvu.com/ru/category/bukvy/", "name": "Новости" } }, { "@type": "ListItem", "position": 3, "item": { "@id": "https://bykvu.com/ru/bukvy/uchenye-nazvali-depressiju-prichinoj-22-opasnyh-zabolevanij/", "name": "Ученые назвали депрессию причиной 22 опасных заболеваний" } } ] } </script>`)
 
@@ -63,7 +64,6 @@ test('only caches the last invocation', t => {
   const html =
     '<script type="application/ld+json">[{"@context":"http://schema.org","@type":"NewsArticle","mainEntityOfPage":"https://www.theverge.com/2017/11/16/16667366/tesla-semi-truck-announced-price-release-date-electric-self-driving","headline":"This is the Tesla Semi truck","description":"500 miles of range and more aerodynamic than a supercar","speakable":{"@type":"SpeakableSpecification","xpath":["/html/head/title","/html/head/meta[@name=\'description\']/@content"]},"datePublished":"2017-11-16T23:47:07-05:00","dateModified":"2017-11-16T23:47:07-05:00","author":{"@type":"Person","name":"Zac Estrada"},"publisher":{"@type":"Organization","name":"The Verge","logo":{"@type":"ImageObject","url":"https://cdn.vox-cdn.com/uploads/chorus_asset/file/13668586/google_amp.0.png","width":600,"height":60}},"about":{"@type":"Event","name":"Tesla Semi Truck Event 2017","startDate":"2017-11-17T04:00:00+00:00","location":{"@type":"Place","name":"Tesla Motors factory","address":"Hawthorne, California, USA"}},"image":[{"@type":"ImageObject","url":"https://cdn.vox-cdn.com/thumbor/k8ssXKPAuRwxa1pKew982ZMgv0o=/1400x1400/filters:format(jpeg)/cdn.vox-cdn.com/uploads/chorus_asset/file/9699573/Semi_Front_Profile.jpg","width":1400,"height":1400},{"@type":"ImageObject","url":"https://cdn.vox-cdn.com/thumbor/l6nkV8CkJIdUrJIzHFWUFc1zLRM=/1400x1050/filters:format(jpeg)/cdn.vox-cdn.com/uploads/chorus_asset/file/9699573/Semi_Front_Profile.jpg","width":1400,"height":1050},{"@type":"ImageObject","url":"https://cdn.vox-cdn.com/thumbor/5Sqo6J73lBi1hwzEiKCQy6FLx3I=/1400x788/filters:format(jpeg)/cdn.vox-cdn.com/uploads/chorus_asset/file/9699573/Semi_Front_Profile.jpg","width":1400,"height":788}]}]</script>'
 
-  // Load it and process it with jsonld.
   const $ = cheerio.load(html)
   const $mutate = cheerio.load($.html())
   const json = jsonld($)
@@ -83,3 +83,17 @@ test('only caches the last invocation', t => {
   // another.
   t.deepEqual(json, jsonld(cheerio.load($.html())))
 })
+
+test('parse json with break lines', t => {
+  const $ =
+    cheerio.load(`<script type="application/ld+json">{"@context":"https://schema.org","mainEntity":{"description":"This is an example
+🌐 of a multiline description
+📬 to see how it is parsed
+📧 and how it is decoded"}}</script>`)
+  t.snapshot(jsonld($))
+})
+
+test('returns empty array if JSON-LD is invalid', t => {
+  const $ = cheerio.load('<script type="application/ld+json">{{</script>')
+  t.deepEqual(jsonld($), [])
+})

packages/metascraper-helpers/test/snapshots/jsonld.js.md

@@ -91,3 +91,19 @@ Generated by [AVA](https://avajs.dev).
         },
       },
     ]
+
+## parse json with break lines
+
+> Snapshot 1
+
+    [
+      {
+        '@context': 'https://schema.org',
+        mainEntity: {
+          description: `This is an example␊
+          🌐 of a multiline description␊
+          📬 to see how it is parsed␊
+          📧 and how it is decoded`,
+        },
+      },
+    ]

packages/metascraper-helpers/test/snapshots/jsonld.js.snap

@@ -9,7 +9,7 @@ Generated by [AVA](https://avajs.dev).
 > Snapshot 1
 
     {
-      description: null,
+      description: 'A modelo que acusa Neymar de agressão e estupro, Najila Mendes de Souza afirmou que continuou a conversa com o jogador para ter provas do estupro.',
       title: 'Modelo diz que Neymar foi “estúpido” e dá detalhes sobre suposto estupro',
     }