fix headless browser bugs: lots of chrome child processes keep growing as page resources are not effectively freed

Author: blogdaren

Examples/start.php

@@ -391,18 +391,18 @@ function startAppDownloader()
     };
 
     //使用无头浏览器回调或者直接使用无头浏览器相关API
-    $downloader->onHeadlessBrowserOpenPage = function($downloader, $browser, $page, $url){
+    /*$downloader->onHeadlessBrowserOpenPage = function($downloader, $browser, $page, $url){
         //注意：灵活设计特定类型的返回值有助于对付各种复杂的应用场景
         //1. 返回false， 会触发中断后续的业务逻辑；
         //2. 返回string，会触发中断后续的业务逻辑，一般多用于返回页面的HTML；
         //3. 返回array， 会继续执行后续的业务逻辑，一般多用于返回无头浏览器选项参数；
         //4. 返回其他，  会继续执行后续的业务逻辑，相当于是什么也没有发生；
 
         //注意：一般无需调用如下几行代码，因为爬山虎内部默认会自动调用无头API做同样的工作.
-        //$page->navigate($url)->waitForNavigation('firstMeaningfulPaint');
-        //$html = $page->getHtml();
-        //return $html;
-    };
+        $page->navigate($url)->waitForNavigation('firstMeaningfulPaint');
+        $html = $page->getHtml();
+        return $html;
+    };*/
 }
 
 

src/Downloader.php

@@ -841,30 +841,30 @@ public function useHeadlessBrowser($args = [])
         $options['track_request_args'] = !empty($args['track_request_args']) ? true : false;
         $merged_options = $this->headlessBrowser->getMergedOptions($options);
 
+        //maybe issue request by user callback
         try{
-            $this->headlessBrowser->getBrowserFactoryInstance()->setOptions($merged_options);
-            $browser = $this->headlessBrowser->getBrowserInstance();
-            $page = $this->headlessBrowser->getPage();
+            if(property_exists($this, 'onHeadlessBrowserOpenPage')){
+                $this->headlessBrowser->getBrowserFactoryInstance()->setOptions($merged_options);
+                $browser = $this->headlessBrowser->getBrowserInstance();
+                $page = $this->headlessBrowser->getPage();
+                $returning = $this->triggerUserCallback('onHeadlessBrowserOpenPage', $this, $browser, $page, $args['url']);
+                if(false === $returning){
+                    $page->close();
+                    return Tool::throwback('-353', $this->langConfig['downloader_download_task_no'] . '(NaN)', $extra);
+                }elseif(is_string($returning)){
+                    $page->close();
+                    $extra = ['content' => $returning];
+                    return Tool::throwback('0', $this->langConfig['downloader_download_task_yes'], $extra);
+                }else{
+                    $page->close();
+                    is_array($returning) && $merged_options = array_merge($merged_options, $returning);
+                }
+            }
         }catch(\Throwable $e){
             $msg = $e->getMessage();
             return Tool::throwback('-352', $this->langConfig['headless_browser_exception'] . " $msg ", $extra);
         }
 
-        $returning = $this->triggerUserCallback('onHeadlessBrowserOpenPage', $this, $browser, $page, $args['url']);
-
-        //maybe issue request by user callback
-        if(false === $returning){
-            $page->close();
-            return Tool::throwback('-353', $this->langConfig['downloader_download_task_no'] . '(NaN)', $extra);
-        }elseif(is_string($returning)){
-            $page->close();
-            $extra = ['content' => $returning];
-            return Tool::throwback('0', $this->langConfig['downloader_download_task_yes'], $extra);
-        }else{
-            $page->close();
-            is_array($returning) && $merged_options = array_merge($merged_options, $returning);
-        }
-
         //maybe issue request by API directly
         try{
             $content = $this->headlessBrowser->request($args['method'], $args['url'], $merged_options);

src/Kernel/Library/Helper/Tool.php

@@ -1383,6 +1383,31 @@ public static function executeShell($command = '')
         }
     }
 
+    /**
+     * @brief    convertRawHeaderToArray    
+     *
+     * @param    string  $header
+     *
+     * @return   array
+     */
+    public static function convertRawHeaderToArray($header = '')
+    {
+        if(!is_string($header)) return [];
+
+        $_header = [];
+        $header = explode("\n", $header);
+        foreach($header as $k => $v)
+        {
+            $tmp = explode(":", $v);
+            if(empty($tmp[0]) || empty($tmp[1])) continue;
+            if(!empty($tmp[0]) && preg_match("/(CONNECT|GET|POST|PUT|DELETE)/s", $tmp[0])) continue; 
+            $k1 = trim($tmp[0]);
+            $v1 = trim($tmp[1]);
+            $_header[$k1] = $v1;
+        }
+
+        return $_header;
+    }
 }
 
 

src/Kernel/Middleware/HeadlessBrowser/Chrome.php

@@ -173,6 +173,19 @@ public function getBrowserInstance()
         return self::$_browser;
     }
 
+    /**
+     * @brief    destroy    
+     *
+     * @return   void
+     */
+    public function destroy()
+    {
+        if(empty(self::$_browser) || !is_object(self::$_browser)) return;
+
+        self::$_browser->close();
+        self::$_browser = null;
+    }
+
     /**
      * @brief    get default arguments    
      *
@@ -251,9 +264,15 @@ public function request($method, $url, $args = [])
 
         //issue http request
         $page = self::getPage();
-        $page->navigate($url)->waitForNavigation($page_event, $navigate_timeout);
-        $html = $page->getHtml();
-        $page->close();
+
+        try{
+            $page->navigate($url)->waitForNavigation($page_event, $navigate_timeout);
+            $html = $page->getHtml();
+            $page->close();
+        }catch(\Throwable $e){
+            $page->close();
+            throw new \Exception($e->getMessage());
+        }
 
         return $html;
     }
@@ -290,12 +309,19 @@ public function setOptions($options = [])
     /**
      * @brief    set http header
      *
-     * @param    array  $headers
+     * @param    array|string  $headers
      *
      * @return   object
      */
     public function setHeaders($headers = [])
     {
+        if(!is_string($headers) && !is_array($headers)) return $this;
+
+        if(is_string($headers))
+        {
+            $headers = Tool::convertRawHeaderToArray($headers);
+        }
+
         !empty($headers) && self::$_config['headers'] = $headers;
 
         return $this;

src/Kernel/Middleware/HttpClient/Guzzle.php

@@ -358,12 +358,19 @@ public function setOptions($options = [])
     /**
      * @brief    set http header
      *
-     * @param    array  $headers
+     * @param    array|string  $headers
      *
      * @return   object
      */
     public function setHeaders($headers = [])
     {
+        if(!is_string($headers) && !is_array($headers)) return $this;
+
+        if(is_string($headers))
+        {
+            $headers = Tool::convertRawHeaderToArray($headers);
+        }
+
         !empty($headers) && self::$_config['headers'] = $headers;
 
         return $this;

src/Kernel/PHPCreeper.php

@@ -40,7 +40,7 @@ class PHPCreeper extends Worker
      *
      * @var string
      */
-    public const CURRENT_VERSION = '1.9.6';
+    public const CURRENT_VERSION = '1.9.7';
 
     /**
      * engine name