Feat: Add device fingerprint

Author: coder-hxl

src/api.ts

@@ -14,7 +14,8 @@ import {
   logError,
   logSuccess,
   logWarn,
-  mkdirDirSync
+  mkdirDirSync,
+  random
 } from './utils'
 
 import {
@@ -216,7 +217,8 @@ function loaderCommonConfig(
   // 1.detailTargets
   crawlConfig.detailTargets.forEach((detail) => {
     // detail > advanced > app
-    const { url, timeout, proxy, maxRetry, priority, headers } = detail
+    const { url, timeout, proxy, maxRetry, priority, headers, fingerprint } =
+      detail
 
     // 1.1.baseUrl
     if (!isUndefined(xCrawlConfig.baseUrl)) {
@@ -259,6 +261,75 @@ function loaderCommonConfig(
     if (isUndefined(headers)) {
       detail.headers = advancedConfig.headers
     }
+
+    // 1.7.fingerprint(公共部分)
+    if (fingerprint) {
+      const { userAgent, ua, platform, mobile, acceptLanguage } = fingerprint
+      let headers = detail.headers
+
+      if (!headers) {
+        detail.headers = headers = {}
+      }
+
+      // 1.user-agent
+      if (userAgent) {
+        headers['user-agent'] = userAgent
+      }
+
+      // 2.sec-ch-ua
+      if (ua) {
+        headers['sec-ch-ua'] = ua
+      }
+
+      // 3.sec-ch-platform
+      if (platform) {
+        headers['sec-ch-platform'] = platform
+      }
+
+      // 4.sec-ch-mobile
+      if (mobile) {
+        headers['sec-ch-mobile'] = mobile
+      }
+
+      // 4.accept-language
+      if (acceptLanguage) {
+        headers['accept-language'] = acceptLanguage
+      }
+    } else if (isUndefined(fingerprint) && advancedConfig.fingerprint) {
+      const { userAgents, uas, platforms, mobiles, acceptLanguages } =
+        advancedConfig.fingerprint
+      let headers = detail.headers
+
+      if (!headers) {
+        detail.headers = headers = {}
+      }
+
+      // 1.user-agent
+      if (userAgents) {
+        headers['user-agent'] = userAgents[random(userAgents.length)]
+      }
+
+      // 2.sec-ch-ua
+      if (uas) {
+        headers['sec-ch-ua'] = uas[random(uas.length)]
+      }
+
+      // 3.sec-ch-platform
+      if (platforms) {
+        headers['sec-ch-platform'] = platforms[random(platforms.length)]
+      }
+
+      // 4.sec-ch-mobile
+      if (mobiles) {
+        headers['sec-ch-mobile'] = mobiles[random(mobiles.length)]
+      }
+
+      // 4.accept-language
+      if (acceptLanguages) {
+        headers['accept-language'] =
+          acceptLanguages[random(acceptLanguages.length)]
+      }
+    }
   })
 
   // 2.intervalTime
@@ -274,6 +345,30 @@ function loaderCommonConfig(
   crawlConfig.onCrawlItemComplete = advancedConfig.onCrawlItemComplete
 }
 
+function loaderPageDetailFingerprint(
+  detail: CrawlPageDetailConfig,
+  fingerprint: {
+    maxWidth: number
+    minWidth?: number
+    maxHeight: number
+    minHidth?: number
+  }
+) {
+  const { maxWidth, minWidth, maxHeight, minHidth } = fingerprint
+
+  // 1.width / height
+  const width = maxWidth === minWidth ? maxWidth : random(maxWidth, minWidth)
+  const height =
+    maxHeight === minHidth ? maxHeight : random(maxHeight, minHidth)
+  const viewport = detail.viewport
+  if (!viewport) {
+    detail.viewport = { width, height }
+  } else {
+    viewport.width = width
+    viewport.height = height
+  }
+}
+
 /* Create Config */
 /*
   每个创建配置函数的返回值都是类似于对应的进阶版(类似 CrawlAdvancedConfig)配置
@@ -318,21 +413,26 @@ function createCrawlPageConfig(
   loaderCommonConfig(xCrawlConfig, advancedConfig, crawlPageConfig)
 
   // 装载单独配置
-  const haveAdvancedCookies = !isUndefined(advancedConfig.cookies)
-  const haveAdvancedViewport = !isUndefined(advancedConfig.viewport)
   crawlPageConfig.detailTargets.forEach((detail) => {
     // detail > advanced  > xCrawl
-    const { cookies, viewport } = detail
+    const { cookies, viewport, fingerprint } = detail
 
     // 1.cookies
-    if (isUndefined(cookies) && haveAdvancedCookies) {
+    if (isUndefined(cookies) && advancedConfig.cookies) {
       detail.cookies = advancedConfig.cookies
     }
 
     // 2.viewport
-    if (isUndefined(viewport) && haveAdvancedViewport) {
+    if (isUndefined(viewport) && advancedConfig.viewport) {
       detail.viewport = advancedConfig.viewport
     }
+
+    // 3.fingerprint
+    if (fingerprint) {
+      loaderPageDetailFingerprint(detail, fingerprint)
+    } else if (isUndefined(fingerprint) && advancedConfig.fingerprint) {
+      loaderPageDetailFingerprint(detail, advancedConfig.fingerprint)
+    }
   })
 
   return crawlPageConfig as CrawlPageConfig
@@ -657,6 +757,8 @@ export function createCrawlPage(xCrawlConfig: LoaderXCrawlConfig) {
     const { detailTargets, intervalTime, onCrawlItemComplete } =
       createCrawlPageConfig(xCrawlConfig, config)
 
+    log(detailTargets)
+
     const extraConfig: ExtraPageConfig = {
       errorPageMap: new Map(),
       browser: browser!,

src/request.ts

@@ -41,8 +41,8 @@ function parseHeaders(
 ) {
   const rawHeaders = rawConfig.headers ?? {}
   const headers: AnyObject = {
-    'User-Agent':
-      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
+    'user-agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
     ...rawHeaders
   }
 

src/types/api.ts

@@ -35,6 +35,18 @@ export type PageCookies =
   | Protocol.Network.CookieParam
   | Protocol.Network.CookieParam[]
 
+export type Platform =
+  | 'Android'
+  | 'Chrome OS'
+  | 'Chromium OS'
+  | 'iOS'
+  | 'Linux'
+  | 'macOS'
+  | 'Windows'
+  | 'Unknown'
+
+export type Mobile = '?0' | '?1'
+
 // API crawl config
 // Common
 export interface CrawlCommonConfig {
@@ -50,6 +62,17 @@ export interface CrawlPageDetailConfig extends CrawlCommonConfig {
   cookies?: PageCookies | null
   priority?: number
   viewport?: Viewport | null
+  fingerprint?: {
+    maxWidth: number
+    minWidth: number
+    maxHeight: number
+    minHidth: number
+    userAgent?: string
+    ua?: string
+    platform?: Platform
+    mobile?: Mobile
+    acceptLanguage?: string
+  } | null
 }
 
 export interface CrawlDataDetailConfig extends CrawlCommonConfig {
@@ -59,6 +82,13 @@ export interface CrawlDataDetailConfig extends CrawlCommonConfig {
   params?: AnyObject
   data?: any
   priority?: number
+  fingerprint?: {
+    userAgent?: string
+    ua?: string
+    platform?: Platform
+    mobile?: Mobile
+    acceptLanguage?: string
+  } | null
 }
 
 export interface CrawlFileDetailConfig extends CrawlCommonConfig {
@@ -68,12 +98,30 @@ export interface CrawlFileDetailConfig extends CrawlCommonConfig {
   storeDir?: string | null
   fileName?: string
   extension?: string | null
+  fingerprint?: {
+    userAgent?: string
+    ua?: string
+    platform?: Platform
+    mobile?: Mobile
+    acceptLanguage?: string
+  } | null
 }
 
 // 2.Advanced
 export interface CrawlPageAdvancedConfig extends CrawlCommonConfig {
   targets: (string | CrawlPageDetailConfig)[]
   intervalTime?: IntervalTime
+  fingerprint?: {
+    maxWidth: number
+    minWidth?: number
+    maxHeight: number
+    minHidth?: number
+    userAgents?: string[]
+    uas?: string[]
+    platforms?: Platform[]
+    mobiles?: Mobile[]
+    acceptLanguages?: string[]
+  }
 
   headers?: AnyObject
   cookies?: PageCookies
@@ -85,6 +133,13 @@ export interface CrawlPageAdvancedConfig extends CrawlCommonConfig {
 export interface CrawlDataAdvancedConfig<T> extends CrawlCommonConfig {
   targets: (string | CrawlDataDetailConfig)[]
   intervalTime?: IntervalTime
+  fingerprint?: {
+    userAgents?: string[]
+    uas?: string[]
+    platforms?: Platform[]
+    mobiles?: Mobile[]
+    acceptLanguages?: string[]
+  }
 
   headers?: AnyObject
 
@@ -94,6 +149,13 @@ export interface CrawlDataAdvancedConfig<T> extends CrawlCommonConfig {
 export interface CrawlFileAdvancedConfig extends CrawlCommonConfig {
   targets: (string | CrawlFileDetailConfig)[]
   intervalTime?: IntervalTime
+  fingerprint?: {
+    userAgents?: string[]
+    uas?: string[]
+    platforms?: Platform[]
+    mobiles?: Mobile[]
+    acceptLanguages?: string[]
+  }
 
   headers?: AnyObject
   storeDir?: string