feat: detect adult website from domain

Author: atinux

src/index.js

@@ -5,7 +5,7 @@ const { createHash } = require('crypto')
 const { URL } = require('url')
 const { join } = require('path')
 const { tmpdir } = require('os')
-const { isCrawlable, puppeteerArgs, puppeteerViewport } = require('./utils')
+const { isCrawlable, puppeteerArgs, puppeteerViewport, isAdultContentDomain } = require('./utils')
 const { hasVue, getVueMeta, getFramework, getPlugins, getUI, getNuxtMeta, getNuxtModules } = require('./detectors')
 const consola = require('consola')
 
@@ -141,11 +141,15 @@ async function analyze (originalUrl, options = {}) {
     }
     infos.meta.siteName = await page.$eval('head > meta[property="og:site_name"]', element => element.content).catch(() => '')
 
-    const rtaLabel = await page.$eval('head > meta[name="RATING"]', element => element.content).catch(() => '')
-    if (rtaLabel.trim() === 'RTA-5042-1996-1400-1577-RTA') {
+    // Is adult website?
+    const rtaLabel = await page.$eval('head > meta[name="rating"]', element => element.content).catch(() => '')
+    if (['adult', 'RTA-5042-1996-1400-1577-RTA'].includes(rtaLabel.trim())) {
       infos.meta.isAdultContent = true
+    } else {
+      infos.meta.isAdultContent = await isAdultContentDomain(domain)
     }
 
+
     // Get page language
     const matches = html.match(new RegExp('<html[^>]*[: ]lang="([a-z]{2}((-|_)[A-Z]{2})?)"', 'i'));
     if (matches && matches.length) {

src/utils.js

@@ -1,3 +1,7 @@
+const { Resolver } = require('dns').promises
+const fetch = require('node-fetch')
+const resolver = new Resolver()
+
 const asArray = (value) => value instanceof Array ? value : [value]
 exports.asArray = asArray
 
@@ -112,3 +116,21 @@ exports.puppeteerViewport = {
   isMobile: false,
   width: 1920,
 }
+
+const openDNSIPs = [/67\.215\.65.\d{3}/gi, /146\.112\.61\.\d{3}/gi]
+
+function isOpenDNS(ip) {
+  return openDNSIPs.some(function (regexp) {
+    return regexp.test(ip)
+  })
+}
+
+exports.isAdultContentDomain = async function (domain) {
+  resolver.setServers(['208.67.222.123'])
+  const adultIPs = await resolver.resolve(domain, 'A')
+  if (isOpenDNS(adultIPs[0])) {
+    return true
+  } else {
+    return false
+  }
+}