Add: Complete priority request function

Author: coder-hxl

src/api.ts

@@ -5,6 +5,7 @@ import puppeteer, { Browser, HTTPResponse, Page, Protocol } from 'puppeteer'
 
 import { ControllerConfig, controller } from './controller'
 import { request } from './request'
+import { quickSort } from './sort'
 import {
   isArray,
   isObject,
@@ -41,7 +42,6 @@ import {
   LoaderFileRequestConfig
 } from './types/api'
 import { LoaderXCrawlBaseConfig } from './types'
-import { quickSort } from './sort'
 
 async function crawlRequestSingle(
   controllerConfig: ControllerConfig<
@@ -109,7 +109,7 @@ function loaderCommonConfig(
 ) {
   // 1.requestConfigs
   loaderConfig.requestConfigs = requestObjecs.map((requestConfig) => {
-    let { url, timeout, proxy, maxRetry } = requestConfig
+    let { url, timeout, proxy, maxRetry, priority } = requestConfig
 
     // 1.1.baseUrl
     if (!isUndefined(baseConfig.baseUrl)) {
@@ -141,7 +141,12 @@ function loaderCommonConfig(
       }
     }
 
-    return { url, timeout, proxy, maxRetry }
+    // 1.5.priority
+    if (isUndefined(priority)) {
+      priority = 0
+    }
+
+    return { url, timeout, proxy, maxRetry, priority }
   })
 
   // 2.intervalTime

src/batchCrawlHandle.ts

@@ -1,6 +1,11 @@
 import { isNumber, isUndefined, log, logNumber, random, sleep } from './utils'
 
-import type { IntervalTime } from './types/api'
+import type {
+  IntervalTime,
+  LoaderDataRequestConfig,
+  LoaderFileRequestConfig,
+  LoaderPageRequestConfig
+} from './types/api'
 import type { ControllerConfig } from './controller'
 
 async function useSleepByBatch(
@@ -26,7 +31,14 @@ async function useSleepByBatch(
   }
 }
 
-export async function asyncBatchCrawl<T, V, C>(
+export async function asyncBatchCrawl<
+  T extends
+    | LoaderPageRequestConfig
+    | LoaderDataRequestConfig
+    | LoaderFileRequestConfig,
+  V,
+  C
+>(
   controllerConfigs: ControllerConfig<T, V>[],
   intervalTime: IntervalTime | undefined,
   crawlSingleFnExtraConfig: C,
@@ -73,7 +85,14 @@ export async function asyncBatchCrawl<T, V, C>(
   await Promise.all(crawlQueue)
 }
 
-export async function syncBatchCrawl<T, V, C>(
+export async function syncBatchCrawl<
+  T extends
+    | LoaderPageRequestConfig
+    | LoaderDataRequestConfig
+    | LoaderFileRequestConfig,
+  V,
+  C
+>(
   controllerConfigs: ControllerConfig<T, V>[],
   intervalTime: IntervalTime | undefined,
   crawlSingleFnExtraConfig: C,

src/controller.ts

@@ -1,8 +1,20 @@
 import { asyncBatchCrawl, syncBatchCrawl } from './batchCrawlHandle'
-import { IntervalTime } from './types/api'
+import { priorityQueueMergeSort } from './sort'
+import {
+  IntervalTime,
+  LoaderDataRequestConfig,
+  LoaderFileRequestConfig,
+  LoaderPageRequestConfig
+} from './types/api'
 import { log, logError, logNumber, logSuccess, logWarn } from './utils'
 
-export interface ControllerConfig<T, V> {
+export interface ControllerConfig<
+  T extends
+    | LoaderPageRequestConfig
+    | LoaderDataRequestConfig
+    | LoaderFileRequestConfig,
+  V
+> {
   id: number
   isSuccess: boolean
   crawlCount: number
@@ -12,7 +24,14 @@ export interface ControllerConfig<T, V> {
   crawlSingleRes: V | null
 }
 
-export async function controller<T extends { maxRetry: number }, V, C>(
+export async function controller<
+  T extends
+    | LoaderPageRequestConfig
+    | LoaderDataRequestConfig
+    | LoaderFileRequestConfig,
+  V,
+  C
+>(
   name: 'page' | 'data' | 'file',
   mode: 'async' | 'sync',
   requestConfigs: T[],
@@ -23,8 +42,21 @@ export async function controller<T extends { maxRetry: number }, V, C>(
     crawlSingleFnExtraConfig: C
   ) => Promise<V>
 ): Promise<ControllerConfig<T, V>[]> {
+  // 是否使用优先爬取
+  const isPriorityCrawl = !requestConfigs.every(
+    (item) => item.priority === requestConfigs[0].priority
+  )
+  const targetRequestConfigs = isPriorityCrawl
+    ? priorityQueueMergeSort(
+        requestConfigs.map((item) => ({
+          ...item,
+          valueOf: () => item.priority
+        }))
+      )
+    : requestConfigs
+
   // 通过映射生成新的配置数组
-  const controllerConfigs: ControllerConfig<T, V>[] = requestConfigs.map(
+  const controllerConfigs: ControllerConfig<T, V>[] = targetRequestConfigs.map(
     (requestConfig, index) => ({
       id: index + 1,
       isSuccess: false,

src/sort.ts

@@ -45,4 +45,33 @@ export function quickSort<T extends any[]>(arr: T): T {
   return arr
 }
 
-// console.log(quickSort([7, 3, 6, 4, 9, 2, 1, 5]))
+export function priorityQueueMergeSort<T extends any[]>(arr: T): T {
+  if (arr.length === 1) return arr
+
+  const mid = Math.floor(arr.length / 2)
+  const newLeftArr = priorityQueueMergeSort(arr.slice(0, mid))
+  const newRightArr = priorityQueueMergeSort(arr.slice(mid))
+
+  const newArr = [] as any as T
+  let i = 0
+  let j = 0
+  while (i < newLeftArr.length && j < newRightArr.length) {
+    if (newLeftArr[i] >= newRightArr[j]) {
+      newArr.push(newLeftArr[i])
+      i++
+    } else {
+      newArr.push(newRightArr[j])
+      j++
+    }
+  }
+
+  if (i < newLeftArr.length) {
+    newArr.push(...newLeftArr.slice(i))
+  }
+
+  if (j < newRightArr.length) {
+    newArr.push(...newRightArr.splice(j))
+  }
+
+  return newArr
+}

src/types/api.ts

@@ -7,6 +7,7 @@ import { AnyObject, MapTypeObject } from './common'
 type LoaderHasConfig = {
   timeout: number
   maxRetry: number
+  priority: number
 }
 
 export type LoaderPageRequestConfig = PageRequestConfig & LoaderHasConfig
@@ -71,6 +72,7 @@ export interface PageRequestConfig {
   proxy?: string
   cookies?: PageRequestConfigCookies
   maxRetry?: number
+  priority?: number
 }
 
 export interface DataRequestConfig {
@@ -82,6 +84,7 @@ export interface DataRequestConfig {
   timeout?: number
   proxy?: string
   maxRetry?: number
+  priority?: number
 }
 
 export interface FileRequestConfig {
@@ -90,6 +93,7 @@ export interface FileRequestConfig {
   timeout?: number
   proxy?: string
   maxRetry?: number
+  priority?: number
 }
 
 // CrawlConfig