Expand CPM generated tules to more languages (#154)

* Add some button patterns for other languages

* Clean button text before matching

* Make sure button text is always a string

* do not match empty button texts

* Handle parsing errors

* Fix crawler rate reporting: avoid collision with built-in :rate

* more informative progress bar

* Update German button patterns

* Lint

* update Italian button patterns

* update Spanish button patterns

* Update Brazilian Portuguese button patterns

* Update Swedish button patterns

Author: muodov

collectors/CookiePopups/scrapeScript.js

@@ -312,7 +312,7 @@ function getButtonData(el) {
         .filter(b => isVisible(b) && !isDisabled(b) && b.innerText.trim());
 
     return actionableButtons.map(b => ({
-        text: b.innerText,
+        text: b.innerText ?? b.textContent ?? '',
         selector: getUniqueSelector(b),
     }));
 }

post-processing/detect-cookie-popups.js

@@ -169,7 +169,7 @@ async function main() {
     });
 
     const pages = fs.readdirSync(crawlDir).filter(name => name.endsWith('.json') && name !== 'metadata.json');
-    const progressBar = process.env.IS_CI ? null : new ProgressBar('[:bar] :percent ETA :etas :page', {
+    const progressBar = process.env.IS_CI ? null : new ProgressBar('[:bar] :current/:total :percent ETA :etas rate :rate/s :page', {
         complete: chalk.green('='),
         incomplete: ' ',
         total: pages.length,
@@ -189,8 +189,17 @@ async function main() {
             console.log(`${index + 1}/${pages.length} : ${page}`);
         }
         const filePath = path.join(crawlDir, page);
-        const contents = await fs.promises.readFile(filePath, 'utf-8');
-        const data = JSON.parse(contents.toString());
+
+        let contents;
+        let data;
+        try {
+            contents = await fs.promises.readFile(filePath, 'utf-8');
+            data = JSON.parse(contents.toString());
+        } catch (error) {
+            console.error(`Error reading or parsing file ${page}:`, error.message);
+            progressBar?.tick({ page });
+            return;
+        }
 
         if (!data.data || !data.data.cookiepopups) {
             progressBar?.tick({ page });