Update dependency adblock-rs to ^0.12.0 (#124)

* Update dependency adblock-rs to ^0.12.0

* fix path, dnf

* fix path, allowScripts

* renamed engine.serializeRaw

* fix invalid hostnames

* increase EphemeralStorage

* await browser.close for datadir cleanup

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: mschfh <37435502+mschfh@users.noreply.github.com>

Author: renovate[bot]

Makefile

@@ -43,7 +43,7 @@ bundle:
 	rm -rf $(TMP_WORKSPACE)/node_modules/core-js/client
 	rm -rf $(TMP_WORKSPACE)/node_modules/core-js/stage
 	rm -rf $(TMP_WORKSPACE)/node_modules/nan
-	find $(TMP_WORKSPACE)/node_modules/adblock-rs/js/target/release/ -type f -not -name libadblock_rs.so -delete
+	find $(TMP_WORKSPACE)/node_modules/adblock-rs/target/release/ -mindepth 1 -not -name index.js -not -name index.node -exec rm -rf {} +
 	find $(TMP_WORKSPACE)/node_modules -type f -name "*.md" -delete
 	find $(TMP_WORKSPACE)/node_modules -type d -name "test" | xargs rm -rf
 	cd $(TMP_WORKSPACE)/ && zip -r $(FUNCTION_NAME).zip *

brave/adblock.js

@@ -9,6 +9,13 @@ const adblockRsLib = require('adblock-rs')
 
 const braveDebugLib = require('./debug')
 
+// Non-network request URL schemes that adblock-rs can't parse a hostname from
+// (Request::new returns HostnameParseError). These are browser-internal or inline
+// resources, not real network requests, and show up routinely in crawl data. Drop
+// them so they don't abort the batch; anything else that fails to parse is
+// unexpected and should surface (with its value) so we can narrow it down.
+const NON_NETWORK_SCHEME_RE = /^(?:data|blob|about|javascript):/i
+
 const serializeRules = rules => {
   braveDebugLib.verbose(`Serializing ${rules.length} rules`)
   const filterSet = new adblockRsLib.FilterSet(true)
@@ -17,7 +24,7 @@ const serializeRules = rules => {
     optimize: false
   }
   const adBlockClient = new adblockRsLib.Engine(filterSet, adBlockArgs)
-  const adBlockDat = adBlockClient.serializeRaw()
+  const adBlockDat = adBlockClient.serialize()
   const adBlockDatBuffer = Buffer.from(adBlockDat)
   braveDebugLib.verbose(`Successfully serialized rules into buffer of length ${adBlockDatBuffer.byteLength}`)
   return adBlockDatBuffer
@@ -40,7 +47,22 @@ const applyBlockingRules = (adblockClient, requests) => {
     const requestType = aReport[4]
     const requestUrl = aReport[5]
 
-    const matchResult = adblockClient.check(requestUrl, frameUrl, requestType, true)
+    // Drop known non-network schemes up front: adblock-rs would throw
+    // "hostname parsing failed" on these, and they aren't real requests to block.
+    if (NON_NETWORK_SCHEME_RE.test(requestUrl)) {
+      braveDebugLib.verbose(`Dropping non-network request ${requestUrl} in frame ${frameUrl}`)
+      continue
+    }
+
+    let matchResult
+    try {
+      matchResult = adblockClient.check(requestUrl, frameUrl, requestType, true)
+    } catch (err) {
+      // Not a known non-network scheme, so this parse failure is unexpected.
+      // Re-throw with the offending values attached so Sentry shows what triggered
+      // it (the bare adblock-rs message doesn't include the URL).
+      throw new Error(`adblock check failed for requestUrl=${JSON.stringify(requestUrl)} frameUrl=${JSON.stringify(frameUrl)} requestType=${JSON.stringify(requestType)}: ${err.message}`, { cause: err })
+    }
     if (matchResult.matched === false) {
       if (matchResult.exception) {
         braveDebugLib.verbose(`Would block ${requestUrl} in frame ${frameUrl} of type ${requestType} with rule ${matchResult.filter} but excepted by ${matchResult.exception}`)

brave/lambda_actions/assemble.js

@@ -173,7 +173,7 @@ const convertRules = (rules, format) => {
   const iosFilterSet = new FilterSet(true)
   iosFilterSet.addFilters(filtersUsed)
   const engine = new Engine(iosFilterSet, { optimize: false })
-  const iosDat = engine.serializeRaw()
+  const iosDat = engine.serialize()
   const datBuffer = Buffer.from(iosDat)
 
   if (datBuffer.byteLength === 0) {

brave/lambda_actions/crawl.js

@@ -317,14 +317,16 @@ const start = async args => {
   }
 
   braveDebugLib.verbose('Wrapping up and closing puppeteer.')
+  // browser.close() can hang when Chromium wedges, so cap how long we block on it.
+  // The timeout only stops us waiting; it does not cancel close(), which keeps
+  // running in the background (along with puppeteer's own temp userDataDir cleanup),
+  // so if the environment is reused the shutdown can still finish.
   try {
-    browser.close()
+    await Promise.race([
+      browser.close(),
+      new Promise(resolve => setTimeout(resolve, 15000))
+    ])
   } catch (_) {}
-
-  // We can't wait for the browser to always close cleanly, because it often
-  // will hang indefinitely.  So we just issue the request to close and wait
-  // 10 sec.
-  setTimeout(_ => {}, 10000)
 }
 
 module.exports = {

build.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
+set -euo pipefail
+
 # This script runs in the docker container, to set up the rust toolchain
 # needed to build the adblock-rs node module.
 
-yum install -y openssl-devel nss;
+dnf install -y openssl-devel nss;
 curl https://sh.rustup.rs -sSf > /tmp/sh.rustup.rs;
 sh /tmp/sh.rustup.rs -y;
 source ~/.cargo/env;

package-lock.json

@@ -22,7 +22,7 @@
     "@aws-sdk/client-s3": "3.1071.0",
     "@aws-sdk/client-sqs": "3.1071.0",
     "@sparticuz/chromium": "^148.0.0",
-    "adblock-rs": "^0.7.18",
+    "adblock-rs": "^0.12.0",
     "fkill": "^6.2.0",
     "fs-extra": "^8.1.0",
     "glob": "^7.1.4",
@@ -36,5 +36,8 @@
   },
   "devDependencies": {
     "standard": "17.1.2"
+  },
+  "allowScripts": {
+    "adblock-rs": true
   }
-}
+}

package.json

@@ -255,6 +255,10 @@ Resources:
       CodeUri: ./build/slim-list-generator.zip
       Timeout: 600
       MemorySize: 2048
+      # Chromium (via @sparticuz/chromium) is extracted into /tmp and puppeteer
+      # creates a profile dir there too; the 512 MB default overflows (ENOSPC).
+      EphemeralStorage:
+        Size: 2048
       Layers:
         - !Sub "arn:aws:lambda:${AWS::Region}:${SentryNodeLayerArnTemplate}"
       Environment: