Merge pull request #184 from ByteInternet/more-robust-dns-check

more robust waitForAvailability

Author: vdloo

ci/test/magento/deploy_brancher.php

@@ -19,6 +19,7 @@
 
 $productionStage = $configuration->addStage('test', 'banaan.store');
 $productionStage->addBrancherServer('hndeployintegr8')
+    ->setBrancherTimeout(3000)
     ->setLabels(['gitref='. (\getenv('GITHUB_SHA') ?: 'unknown')]);
 
 return $configuration;

composer.json

@@ -17,7 +17,7 @@
         "doctrine/annotations": "^1.6",
         "guzzlehttp/guzzle": "^7.5",
         "hypernode/api-client": "^0.5",
-        "hypernode/deploy-configuration": "^3.5",
+        "hypernode/deploy-configuration": "^3.6",
         "php-di/php-di": "^7.0",
         "psr/log": "^1.0",
         "symfony/console": "^5.4",

composer.lock

@@ -107,14 +107,20 @@ public function createForHypernode(string $hypernode, array $data = []): string
      *
      * @param string $brancherHypernode Name of the brancher Hypernode
      * @param int $timeout Maximum time to wait for availability
+     * @param int $reachabilityCheckCount Number of consecutive successful checks required
+     * @param int $reachabilityCheckInterval Seconds between reachability checks
      * @return void
      * @throws CreateBrancherHypernodeFailedException
      * @throws HypernodeApiClientException
      * @throws HypernodeApiServerException
      * @throws TimeoutException
      */
-    public function waitForAvailability(string $brancherHypernode, int $timeout = 1500): void
-    {
+    public function waitForAvailability(
+        string $brancherHypernode,
+        int $timeout = 1500,
+        int $reachabilityCheckCount = 6,
+        int $reachabilityCheckInterval = 10
+    ): void {
         $latest = microtime(true);
         $timeElapsed = 0;
         $resolved = false;
@@ -175,7 +181,7 @@ public function waitForAvailability(string $brancherHypernode, int $timeout = 15
             );
         }
 
-        $resolved = false;
+        $consecutiveSuccesses = 0;
         while ($timeElapsed < $timeout) {
             $now = microtime(true);
             $timeElapsed += $now - $latest;
@@ -184,12 +190,37 @@ public function waitForAvailability(string $brancherHypernode, int $timeout = 15
             $connection = @fsockopen(sprintf("%s.hypernode.io", $brancherHypernode), 22);
             if ($connection) {
                 fclose($connection);
-                $resolved = true;
-                break;
+                $consecutiveSuccesses++;
+                $this->log->info(
+                    sprintf(
+                        'Brancher Hypernode %s reachability check %d/%d succeeded.',
+                        $brancherHypernode,
+                        $consecutiveSuccesses,
+                        $reachabilityCheckCount
+                    )
+                );
+
+                if ($consecutiveSuccesses >= $reachabilityCheckCount) {
+                    break;
+                }
+                sleep($reachabilityCheckInterval);
+            } else {
+                if ($consecutiveSuccesses > 0) {
+                    $this->log->info(
+                        sprintf(
+                            'Brancher Hypernode %s reachability check failed, resetting counter (was at %d/%d).',
+                            $brancherHypernode,
+                            $consecutiveSuccesses,
+                            $reachabilityCheckCount
+                        )
+                    );
+                }
+                $consecutiveSuccesses = 0;
+                sleep($reachabilityCheckInterval);
             }
         }
 
-        if (!$resolved) {
+        if ($consecutiveSuccesses < $reachabilityCheckCount) {
             throw new TimeoutException(
                 sprintf('Timed out waiting for brancher Hypernode %s to become reachable', $brancherHypernode)
             );

src/Brancher/BrancherHypernodeManager.php

@@ -35,6 +35,10 @@ class DeployRunner
     public const TASK_BUILD = 'build';
     public const TASK_DEPLOY = 'deploy';
 
+    public const DEFAULT_BRANCHER_TIMEOUT = 1500;
+    public const DEFAULT_BRANCHER_REACHABILITY_CHECK_COUNT = 6;
+    public const DEFAULT_BRANCHER_REACHABILITY_CHECK_INTERVAL = 10;
+
     private TaskFactory $taskFactory;
     private InputInterface $input;
     private LoggerInterface $log;
@@ -274,6 +278,9 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche
         if ($isBrancher && $parentApp) {
             $settings = $serverOptions[Server::OPTION_HN_BRANCHER_SETTINGS] ?? [];
             $labels = $serverOptions[Server::OPTION_HN_BRANCHER_LABELS] ?? [];
+            $timeout = $serverOptions[Server::OPTION_HN_BRANCHER_TIMEOUT] ?? self::DEFAULT_BRANCHER_TIMEOUT;
+            $reachabilityCheckCount = $serverOptions[Server::OPTION_HN_BRANCHER_REACHABILITY_CHECK_COUNT] ?? self::DEFAULT_BRANCHER_REACHABILITY_CHECK_COUNT;
+            $reachabilityCheckInterval = $serverOptions[Server::OPTION_HN_BRANCHER_REACHABILITY_CHECK_INTERVAL] ?? self::DEFAULT_BRANCHER_REACHABILITY_CHECK_INTERVAL;
 
             $this->log->info(sprintf('Creating an brancher Hypernode based on %s.', $parentApp));
             if ($settings) {
@@ -299,7 +306,12 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche
 
             try {
                 $this->log->info('Waiting for brancher Hypernode to become available...');
-                $this->brancherHypernodeManager->waitForAvailability($brancherApp);
+                $this->brancherHypernodeManager->waitForAvailability(
+                    $brancherApp,
+                    $timeout,
+                    $reachabilityCheckCount,
+                    $reachabilityCheckInterval
+                );
                 $this->log->info('Brancher Hypernode has become available!');
             } catch (CreateBrancherHypernodeFailedException | TimeoutException $e) {
                 if (in_array($brancherApp, $this->brancherHypernodesRegistered)) {