Merge pull request #408 from qiniu/feat/middleware-and-backup-domains

add http client middleware and getting region hosts with backup domains

Author: bachue

src/Qiniu/Config.php

@@ -32,6 +32,10 @@ final class Config
     private $regionCache;
     // UC Host
     private $ucHost;
+    // backup UC Hosts
+    private $backupUcHosts;
+    // backup UC Hosts max retry time
+    public $backupUcHostsRetryTimes;
 
     // 构造函数
     public function __construct(Region $z = null)
@@ -41,11 +45,17 @@ public function __construct(Region $z = null)
         $this->useCdnDomains = false;
         $this->regionCache = array();
         $this->ucHost = Config::UC_HOST;
+        $this->backupUcHosts = array(
+            "kodo-config.qiniuapi.com",
+            "api.qiniu.com"
+        );
+        $this->backupUcHostsRetryTimes = 2;
     }
 
-    public function setUcHost($ucHost)
+    public function setUcHost($ucHost, $backupUcHosts = array())
     {
         $this->ucHost = $ucHost;
+        $this->backupUcHosts = $backupUcHosts;
     }
 
     public function getUcHost()
@@ -59,6 +69,21 @@ public function getUcHost()
         return $scheme . $this->ucHost;
     }
 
+    public function appendBackupUcHosts($hosts)
+    {
+        $this->backupUcHosts = array_merge($this->backupUcHosts, $hosts);
+    }
+
+    public function prependBackupUcHosts($hosts)
+    {
+        $this->backupUcHosts = array_merge($hosts, $this->backupUcHosts);
+    }
+
+    public function getBackupUcHosts()
+    {
+        return $this->backupUcHosts;
+    }
+
     public function getUpHost($accessKey, $bucket)
     {
         $region = $this->getRegion($accessKey, $bucket);
@@ -308,7 +333,13 @@ private function getRegion($accessKey, $bucket)
             return $regionCache;
         }
 
-        $region = Zone::queryZone($accessKey, $bucket, $this->getUcHost());
+        $region = Zone::queryZone(
+            $accessKey,
+            $bucket,
+            $this->getUcHost(),
+            $this->getBackupUcHosts(),
+            $this->backupUcHostsRetryTimes
+        );
         if (is_array($region)) {
             list($region, $err) = $region;
             if ($err != null) {
@@ -332,7 +363,13 @@ private function getRegionV2($accessKey, $bucket)
             return array($regionCache, null);
         }
 
-        $region = Zone::queryZone($accessKey, $bucket, $this->getUcHost());
+        $region = Zone::queryZone(
+            $accessKey,
+            $bucket,
+            $this->getUcHost(),
+            $this->getBackupUcHosts(),
+            $this->backupUcHostsRetryTimes
+        );
         if (is_array($region)) {
             list($region, $err) = $region;
             return array($region, $err);

src/Qiniu/Http/Client.php

@@ -2,6 +2,7 @@
 namespace Qiniu\Http;
 
 use Qiniu\Config;
+use Qiniu\Http\Middleware;
 
 final class Client
 {
@@ -14,7 +15,7 @@ final class Client
     public static function get($url, array $headers = array(), $opt = null)
     {
         $request = new Request('GET', $url, $headers, null, $opt);
-        return self::sendRequest($request);
+        return self::sendRequestWithMiddleware($request);
     }
 
     /**
@@ -119,6 +120,19 @@ private static function userAgent()
         return $ua;
     }
 
+    /**
+     * @param Request $request
+     * @return Response
+     */
+    public static function sendRequestWithMiddleware($request)
+    {
+        $middlewares = $request->opt->middlewares;
+        $handle = Middleware\compose($middlewares, function ($req) {
+            return Client::sendRequest($req);
+        });
+        return $handle($request);
+    }
+
     /**
      * @param Request $request
      * @return Response

src/Qiniu/Http/Middleware/Middleware.php

@@ -0,0 +1,31 @@
+<?php
+namespace Qiniu\Http\Middleware;
+
+use Qiniu\Http\Request;
+use Qiniu\Http\Response;
+
+interface Middleware
+{
+    /**
+     * @param Request $request
+     * @param callable(Request): Response $next
+     * @return Response
+     */
+    public function send($request, $next);
+}
+
+/**
+ * @param array<Middleware> $middlewares
+ * @param callable(Request): Response $handler
+ * @return callable(Request): Response
+ */
+function compose($middlewares, $handler)
+{
+    $next = $handler;
+    foreach (array_reverse($middlewares) as $middleware) {
+        $next = function ($request) use ($middleware, $next) {
+            return $middleware->send($request, $next);
+        };
+    }
+    return $next;
+}

src/Qiniu/Http/Middleware/RetryDomainsMiddleware.php

@@ -0,0 +1,76 @@
+<?php
+namespace Qiniu\Http\Middleware;
+
+use Qiniu\Http\Request;
+use Qiniu\Http\Response;
+
+class RetryDomainsMiddleware implements Middleware
+{
+    /**
+     * @var array<string> backup domains.
+     */
+    private $backupDomains;
+
+    /**
+     * @var numeric max retry times for each backup domains.
+     */
+    private $maxRetryTimes;
+
+    /**
+     * @var callable args response and request; returns bool; If true will retry with backup domains.
+     */
+    private $retryCondition;
+
+    /**
+     * @param array<string> $backupDomains
+     * @param numeric $maxRetryTimes
+     */
+    public function __construct($backupDomains, $maxRetryTimes = 2, $retryCondition = null)
+    {
+        $this->backupDomains = $backupDomains;
+        $this->maxRetryTimes = $maxRetryTimes;
+        $this->retryCondition = $retryCondition;
+    }
+
+    private function shouldRetry($resp, $req)
+    {
+        if (is_callable($this->retryCondition)) {
+            return call_user_func($this->retryCondition, $resp, $req);
+        }
+
+        return !$resp || $resp->needRetry();
+    }
+
+    /**
+     * @param Request $request
+     * @param callable(Request): Response $next
+     * @return Response
+     */
+    public function send($request, $next)
+    {
+        $response = null;
+        $urlComponents = parse_url($request->url);
+
+        foreach (array_merge(array($urlComponents["host"]), $this->backupDomains) as $backupDomain) {
+            $urlComponents["host"] = $backupDomain;
+            $request->url = \Qiniu\unparse_url($urlComponents);
+            $retriedTimes = 0;
+
+            while ($retriedTimes < $this->maxRetryTimes) {
+                $response = $next($request);
+
+                $retriedTimes += 1;
+
+                if (!$this->shouldRetry($response, $request)) {
+                    return $response;
+                }
+            }
+        }
+
+        if (!$response) {
+            $response = $next($request);
+        }
+
+        return $response;
+    }
+}

src/Qiniu/Http/Request.php

@@ -3,10 +3,26 @@
 
 final class Request
 {
+    /**
+     * @var string
+     */
     public $url;
+
+    /**
+     * @var array<string, string>
+     */
     public $headers;
+
+    /**
+     * @var mixed|null
+     */
     public $body;
+
+    /**
+     * @var string
+     */
     public $method;
+
     /**
      * @var RequestOptions
      */

src/Qiniu/Http/RequestOptions.php

@@ -2,6 +2,8 @@
 
 namespace Qiniu\Http;
 
+use Qiniu\Http\Middleware\Middleware;
+
 final class RequestOptions
 {
 
@@ -30,16 +32,23 @@ final class RequestOptions
      */
     public $timeout_ms;
 
+    /**
+     * @var array<Middleware>
+     */
+    public $middlewares;
+
     public function __construct(
         $connection_timeout = null,
         $connection_timeout_ms = null,
         $timeout = null,
-        $timeout_ms = null
+        $timeout_ms = null,
+        $middlewares = array()
     ) {
         $this->connection_timeout = $connection_timeout;
         $this->connection_timeout_ms = $connection_timeout_ms;
         $this->timeout = $timeout;
         $this->timeout_ms = $timeout_ms;
+        $this->middlewares = $middlewares;
     }
 
     public function getCurlOpt()

src/Qiniu/Http/Response.php

@@ -199,10 +199,18 @@ public function ok()
 
     public function needRetry()
     {
-        $code = $this->statusCode;
-        if ($code < 0 || ($code / 100 === 5 and $code !== 579) || $code === 996) {
-            return true;
+        if ($this->statusCode > 0 && $this->statusCode < 500) {
+            return false;
         }
+
+        // https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code
+        if (in_array($this->statusCode, array(
+            501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701
+        ))) {
+            return false;
+        }
+
+        return true;
     }
 
     private static function isJson($headers)

src/Qiniu/Region.php

@@ -3,6 +3,8 @@
 
 use Qiniu\Http\Client;
 use Qiniu\Http\Error;
+use Qiniu\Http\Middleware\RetryDomainsMiddleware;
+use Qiniu\Http\RequestOptions;
 
 class Region
 {
@@ -169,16 +171,23 @@ public static function regionSeoul()
     }
 
     /*
-     * GET /v2/query?ak=<ak>&bucket=<bucket>
+     * GET /v4/query?ak=<ak>&bucket=<bucket>
      **/
-    public static function queryRegion($ak, $bucket, $ucHost = null)
+    public static function queryRegion($ak, $bucket, $ucHost = null, $backupUcHosts = array(), $retryTimes = 2)
     {
         $region = new Region();
         if (!$ucHost) {
             $ucHost = "https://" . Config::UC_HOST;
         }
         $url = $ucHost . '/v4/query' . "?ak=$ak&bucket=$bucket";
-        $ret = Client::Get($url);
+        $reqOpt = new RequestOptions();
+        $reqOpt->middlewares = array(
+            new RetryDomainsMiddleware(
+                $backupUcHosts,
+                $retryTimes
+            )
+        );
+        $ret = Client::Get($url, array(), $reqOpt);
         if (!$ret->ok()) {
             return array(null, new Error($url, $ret));
         }

src/Qiniu/Zone.php

@@ -50,8 +50,8 @@ public static function qvmZonez1()
         return parent::qvmRegionHuabei();
     }
 
-    public static function queryZone($ak, $bucket, $ucHost = null)
+    public static function queryZone($ak, $bucket, $ucHost = null, $backupUcHosts = array(), $retryTimes = 2)
     {
-        return parent::queryRegion($ak, $bucket, $ucHost);
+        return parent::queryRegion($ak, $bucket, $ucHost, $backupUcHosts, $retryTimes);
     }
 }

src/Qiniu/functions.php

@@ -302,4 +302,35 @@ function ucwords($str, $delimiters)
             return \ucwords($str, $delimiters);
         }
     }
+
+    /**
+     * 将 parse_url 的结果转换回字符串
+     * TODO: add unit test
+     *
+     * @param $parsed_url - parse_url 的结果
+     * @return string
+     */
+    function unparse_url($parsed_url)
+    {
+
+        $scheme   = isset($parsed_url['scheme']) ? $parsed_url['scheme'] . '://' : '';
+
+        $host     = isset($parsed_url['host']) ? $parsed_url['host'] : '';
+
+        $port     = isset($parsed_url['port']) ? ':' . $parsed_url['port'] : '';
+
+        $user     = isset($parsed_url['user']) ? $parsed_url['user'] : '';
+
+        $pass     = isset($parsed_url['pass']) ? ':' . $parsed_url['pass']  : '';
+
+        $pass     = ($user || $pass) ? "$pass@" : '';
+
+        $path     = isset($parsed_url['path']) ? $parsed_url['path'] : '';
+
+        $query    = isset($parsed_url['query']) ? '?' . $parsed_url['query'] : '';
+
+        $fragment = isset($parsed_url['fragment']) ? '#' . $parsed_url['fragment'] : '';
+
+        return "$scheme$user$pass$host$port$path$query$fragment";
+    }
 }