Skip to content

Commit bf24f8b

Browse files
ilyakharevilyakharev
authored
Merge pull request #87 from ydb-platform/retry-nodes
Fix exception on re-create server nodes
2 parents 309be3f + 24093b1 commit bf24f8b

File tree

11 files changed

+176
-58
lines changed

11 files changed

+176
-58
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
* fixed exception on re-create server nodes
12
* fixed key name in createTable function
23
* added simple std looger
34

‎src/Discovery.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ class Discovery
4141
*/
4242
public function __construct(Ydb $ydb, LoggerInterface $logger = null)
4343
{
44+
$this->ydb = $ydb;
45+
4446
$this->client = new ServiceClient($ydb->endpoint(), [
4547
'credentials' => $ydb->iam()->getCredentials(),
4648
]);

‎src/Logger/SimpleStdLogger.php‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,17 @@ class SimpleStdLogger implements \Psr\Log\LoggerInterface
2424
self::EMERGENCY => 'EMERGENCY',
2525
];
2626

27+
protected static function getLevelName(int $level): string
28+
{
29+
if (!isset(static::$levels[$level])) {
30+
throw new InvalidArgumentException('Level "'.$level.'" is not defined, use one of: '.implode(', ', array_keys(static::$levels)));
31+
}
32+
33+
return static::$levels[$level];
34+
}
35+
2736
protected $level;
37+
2838
public function __construct(int $level)
2939
{
3040
$this->level = $level;
@@ -37,44 +47,44 @@ public function emergency($message, array $context = []): void
3747

3848
public function alert($message, array $context = []): void
3949
{
40-
$this->log(self::EMERGENCY, $message, $context);
50+
$this->log(self::ALERT, $message, $context);
4151
}
4252

4353
public function critical($message, array $context = []): void
4454
{
45-
$this->log(self::EMERGENCY, $message, $context);
55+
$this->log(self::CRITICAL, $message, $context);
4656
}
4757

4858
public function error($message, array $context = []): void
4959
{
50-
$this->log(self::EMERGENCY, $message, $context);
60+
$this->log(self::ERROR, $message, $context);
5161
}
5262

5363
public function warning($message, array $context = []): void
5464
{
55-
$this->log(self::EMERGENCY, $message, $context);
65+
$this->log(self::WARNING, $message, $context);
5666
}
5767

5868
public function notice($message, array $context = []): void
5969
{
60-
$this->log(self::EMERGENCY, $message, $context);
70+
$this->log(self::NOTICE, $message, $context);
6171
}
6272

6373
public function info($message, array $context = []): void
6474
{
65-
$this->log(self::EMERGENCY, $message, $context);
75+
$this->log(self::INFO, $message, $context);
6676
}
6777

6878
public function debug($message, array $context = []): void
6979
{
70-
$this->log(self::EMERGENCY, $message, $context);
80+
$this->log(self::DEBUG, $message, $context);
7181
}
7282

7383
public function log($level, $message, array $context = []): void
7484
{
7585
if ($level>$this->level) return;
7686
fwrite(STDERR,
77-
date("d/m/y H:i")." ".$level. " ".$message." ".json_encode($context)
87+
date("d/m/y H:i:s")." ".self::getLevelName($level). " ".$message." ".json_encode($context)."\n"
7888
);
7989
}
8090
}

‎src/Operations.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ class Operations
3636
*/
3737
public function __construct(Ydb $ydb, LoggerInterface $logger = null)
3838
{
39+
$this->ydb = $ydb;
40+
3941
$this->client = new ServiceClient($ydb->endpoint(), [
4042
'credentials' => $ydb->iam()->getCredentials(),
4143
]);

‎src/Retry/Retry.php‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,8 @@
44

55
use Closure;
66
use YdbPlatform\Ydb\Exception;
7-
use YdbPlatform\Ydb\Exceptions\Grpc\DeadlineExceededException;
8-
use YdbPlatform\Ydb\Exceptions\NonRetryableException;
97
use YdbPlatform\Ydb\Exceptions\RetryableException;
10-
use YdbPlatform\Ydb\Exceptions\Ydb\AbortedException;
11-
use YdbPlatform\Ydb\Exceptions\Ydb\BadSessionException;
12-
use YdbPlatform\Ydb\Exceptions\Ydb\SessionBusyException;
13-
use YdbPlatform\Ydb\Exceptions\Ydb\UnavailableException;
14-
use YdbPlatform\Ydb\Exceptions\Ydb\UndeterminedException;
8+
use Psr\Log\LoggerInterface;
159

1610
class Retry
1711
{
@@ -22,9 +16,14 @@ class Retry
2216

2317
protected $fastBackOff;
2418
protected $noBackOff;
19+
/**
20+
* @var LoggerInterface
21+
*/
22+
protected $logger;
2523

26-
public function __construct()
24+
public function __construct(LoggerInterface $logger)
2725
{
26+
$this->logger = $logger;
2827
$this->timeoutMs = 2000;
2928
$this->fastBackOff = new Backoff(6, 5);
3029
$this->slowBackOff = new Backoff(6, 1000);
@@ -79,15 +78,19 @@ public function retry(Closure $closure, bool $idempotent)
7978
$retryCount = 0;
8079
$lastException = null;
8180
while (microtime(true) < $startTime + $this->timeoutMs / 1000) {
81+
$this->logger->debug("YDB: Run user function. Retry count: $retryCount. Ms: ".(microtime(true) - $startTime));
8282
try {
8383
return $closure();
8484
} catch (Exception $e) {
85+
$this->logger->warning("YDB: Received exception: ".$e->getMessage());
8586
if (!$this->canRetry($e, $idempotent)){
86-
throw $e;
87+
$lastException = $e;
88+
break;
8789
}
8890
$retryCount++;
89-
$this->retryDelay($retryCount, $this->backoffType($e));
9091
$lastException = $e;
92+
$delay = $this->retryDelay($retryCount, $this->backoffType($e))*1000;
93+
usleep($delay);
9194
}
9295
}
9396
throw $lastException;
@@ -99,18 +102,20 @@ public function retry(Closure $closure, bool $idempotent)
99102
*/
100103
protected function backoffType(string $e): Backoff
101104
{
102-
return in_array($e, self::$immediatelyBackoff)?$this->noBackOff:
103-
(in_array($e, self::$fastBackoff)?$this->fastBackOff:$this->slowBackOff);
105+
return in_array($e, self::$immediatelyBackoff) ? $this->noBackOff :
106+
(in_array($e, self::$fastBackoff) ? $this->fastBackOff : $this->slowBackOff);
104107
}
105108

106-
protected function alwaysRetry(string $exception){
109+
protected function alwaysRetry(string $exception)
110+
{
107111
return in_array($exception, self::$alwaysRetry);
108112
}
109113

110114
protected function canRetry(Exception $e, bool $idempotent)
111115
{
112-
return is_a($e, RetryableException::class)&&($this->alwaysRetry(get_class($e)) || $idempotent);
116+
return is_a($e, RetryableException::class) && ($this->alwaysRetry(get_class($e)) || $idempotent);
113117
}
118+
114119
private static $immediatelyBackoff = [
115120
\YdbPlatform\Ydb\Exceptions\Grpc\AbortedException::class,
116121
\YdbPlatform\Ydb\Exceptions\Ydb\BadSessionException::class,

‎src/Scheme.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ class Scheme
3939
*/
4040
public function __construct(Ydb $ydb, LoggerInterface $logger = null)
4141
{
42+
$this->ydb = $ydb;
43+
4244
$this->client = new ServiceClient($ydb->endpoint(), [
4345
'credentials' => $ydb->iam()->getCredentials(),
4446
]);

‎src/Scripting.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ class Scripting
3636
*/
3737
public function __construct(Ydb $ydb, LoggerInterface $logger = null)
3838
{
39+
$this->ydb = $ydb;
40+
3941
$this->client = new ServiceClient($ydb->endpoint(), [
4042
'credentials' => $ydb->iam()->getCredentials(),
4143
]);

‎src/Session.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ class Session
7676
*/
7777
public function __construct(Table $table, $session_id)
7878
{
79+
$this->ydb = $table->ydb();
80+
7981
$this->table = $table;
8082

8183
$this->session_id = $session_id;

‎src/Table.php‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,19 @@ class Table
6161
*/
6262
private $retry;
6363

64+
/**
65+
* @var Ydb
66+
*/
67+
protected $ydb;
68+
6469
/**
6570
* @param Ydb $ydb
6671
* @param LoggerInterface|null $logger
6772
*/
6873
public function __construct(Ydb $ydb, LoggerInterface $logger = null, Retry &$retry)
6974
{
75+
$this->ydb = $ydb;
76+
7077
$this->client = new ServiceClient($ydb->endpoint(), [
7178
'credentials' => $ydb->iam()->getCredentials(),
7279
]);
@@ -142,6 +149,14 @@ public function session()
142149
return $session;
143150
}
144151

152+
/**
153+
* @return Ydb
154+
*/
155+
public function ydb()
156+
{
157+
return $this->ydb;
158+
}
159+
145160
/**
146161
* @return Session|null
147162
*/
@@ -468,8 +483,7 @@ public function retrySession(Closure $userFunc, bool $idempotent = false, RetryP
468483

469484
public function retryTransaction(Closure $userFunc, bool $idempotent = false, RetryParams $params = null){
470485

471-
return $this->retry->withParams($params)->retry(function () use ($params, $idempotent, $userFunc){
472-
$this->retrySession(function (Session $session) use ($userFunc) {
486+
return $this->retrySession(function (Session $session) use ($userFunc) {
473487
try{
474488
$session->beginTransaction();
475489
$result = $userFunc($session);
@@ -482,7 +496,6 @@ public function retryTransaction(Closure $userFunc, bool $idempotent = false, Re
482496
throw $exception;
483497
}
484498
}, $idempotent, $params);
485-
}, $idempotent);
486499

487500
}
488501

‎src/Traits/RequestTrait.php‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use YdbPlatform\Ydb\Issue;
88
use YdbPlatform\Ydb\Exception;
99
use YdbPlatform\Ydb\QueryResult;
10+
use YdbPlatform\Ydb\Ydb;
1011

1112
trait RequestTrait
1213
{
@@ -30,6 +31,16 @@ trait RequestTrait
3031
*/
3132
protected $last_request_try_count = 0;
3233

34+
/**
35+
* @var Ydb
36+
*/
37+
protected $ydb;
38+
39+
/**
40+
* @var int
41+
*/
42+
protected $lastDiscovery = 0;
43+
3344
/**
3445
* Make a request to the service with the given method.
3546
*
@@ -41,6 +52,8 @@ trait RequestTrait
4152
*/
4253
protected function doRequest($service, $method, array $data = [])
4354
{
55+
$this->checkDiscovery();
56+
4457
$this->meta['x-ydb-auth-ticket'] = [$this->credentials->token()];
4558

4659
$this->saveLastRequest($service, $method, $data);
@@ -82,7 +95,7 @@ protected function doRequest($service, $method, array $data = [])
8295
if (method_exists($call, 'wait')) {
8396
list($response, $status) = $call->wait();
8497

85-
$this->checkGrpcStatus($service, $method, $status);
98+
$this->handleGrpcStatus($service, $method, $status);
8699

87100
return $this->processResponse($service, $method, $response, $resultClass);
88101
}
@@ -101,6 +114,10 @@ protected function doRequest($service, $method, array $data = [])
101114
*/
102115
protected function doStreamRequest($service, $method, $data = [])
103116
{
117+
$this->checkDiscovery();
118+
119+
$this->meta['x-ydb-auth-ticket'] = [$this->credentials->token()];
120+
104121
if (method_exists($this, 'take')) {
105122
$this->take();
106123
}
@@ -149,10 +166,25 @@ protected function doStreamRequest($service, $method, $data = [])
149166
* @param object $status
150167
* @throws Exception
151168
*/
152-
protected function checkGrpcStatus($service, $method, $status)
169+
protected function handleGrpcStatus($service, $method, $status)
153170
{
154171
if (isset($status->code) && $status->code !== 0) {
155-
$message = 'YDB ' . $service . ' ' . $method . ' (status code GRPC_' . $status->code . '): ' . ($status->details ?? 'no details');
172+
$message = 'YDB ' . $service . ' ' . $method . ' (status code GRPC_'.
173+
(isset(self::$grpcExceptions[$status->code])?self::$grpcNames[$status->code]:$status->code)
174+
.' ' . $status->code . '): ' . ($status->details ?? 'no details');
175+
$this->logger->error($message);
176+
if ($this->ydb->needDiscovery()){
177+
try{
178+
$this->ydb->discover();
179+
}catch (\Exception $e){}
180+
}
181+
$endpoint = $this->ydb->endpoint();
182+
if ($this->ydb->needDiscovery()){
183+
$endpoint = $this->ydb->cluster()->all()[array_rand($this->ydb->cluster()->all())]->endpoint();
184+
}
185+
$this->client = new $this->client($endpoint,[
186+
'credentials' => $this->ydb->iam()->getCredentials()
187+
]);
156188
if (isset(self::$grpcExceptions[$status->code])) {
157189
throw new self::$grpcExceptions[$status->code]($message);
158190
} else {
@@ -272,6 +304,17 @@ protected function resetLastRequest()
272304
$this->last_request_try_count = 0;
273305
}
274306

307+
protected function checkDiscovery(){
308+
if ($this->ydb->needDiscovery() && time()-$this->lastDiscovery>$this->ydb->discoveryInterval()){
309+
try{
310+
$this->lastDiscovery = time();
311+
$this->ydb->discover();
312+
} catch (\Exception $e){
313+
314+
}
315+
}
316+
}
317+
275318
private static $ydbExceptions = [
276319
StatusCode::STATUS_CODE_UNSPECIFIED => \YdbPlatform\Ydb\Exceptions\Ydb\StatusCodeUnspecified::class,
277320
StatusCode::BAD_REQUEST => \YdbPlatform\Ydb\Exceptions\Ydb\BadRequestException::class,
@@ -313,4 +356,22 @@ protected function resetLastRequest()
313356
16 => \YdbPlatform\Ydb\Exceptions\Grpc\UnauthenticatedException::class
314357
];
315358

359+
private static $grpcNames = [
360+
1 => "CANCELLED",
361+
2 => "UNKNOWN",
362+
3 => "INVALID_ARGUMENT",
363+
4 => "DEADLINE_EXCEEDED",
364+
5 => "NOT_FOUND",
365+
6 => "ALREADY_EXISTS",
366+
7 => "PERMISSION_DENIED",
367+
8 => "RESOURCE_EXHAUSTED",
368+
9 => "FAILED_PRECONDITION",
369+
10 => "ABORTED",
370+
11 => "OUT_OF_RANGE",
371+
12 => "UNIMPLEMENTED",
372+
13 => "INTERNAL",
373+
14 => "UNAVAILABLE",
374+
15 => "DATA_LOSS",
375+
16 => "UNAUTHENTICATED"
376+
];
316377
}

0 commit comments

Comments
 (0)