diff --git a/src/Brancher/BrancherHypernodeManager.php b/src/Brancher/BrancherHypernodeManager.php index 9f96175..cf75a66 100644 --- a/src/Brancher/BrancherHypernodeManager.php +++ b/src/Brancher/BrancherHypernodeManager.php @@ -14,6 +14,11 @@ class BrancherHypernodeManager { + /** + * Timeout in seconds for initial SSH check when reusing a Brancher. + */ + private const REUSED_SSH_CHECK_TIMEOUT_SECONDS = 10; + private LoggerInterface $log; private HypernodeClient $hypernodeClient; @@ -102,6 +107,69 @@ public function createForHypernode(string $hypernode, array $data = []): string return $this->hypernodeClient->brancherApp->create($hypernode, $data); } + /** + * Check if brancher Hypernode is reachable via SSH. + * + * @param string $brancherHypernode Name of the brancher Hypernode + * @param int $timeout Maximum time to wait for reachability + * @param int $reachabilityCheckCount Number of consecutive successful checks required + * @param int $reachabilityCheckInterval Seconds between reachability checks + * @return bool True if reachable, false if timed out + */ + private function checkSshReachability( + string $brancherHypernode, + int $timeout, + int $reachabilityCheckCount, + int $reachabilityCheckInterval + ): bool { + $startTime = microtime(true); + $consecutiveSuccesses = 0; + + while ((microtime(true) - $startTime) < $timeout) { + $connection = @fsockopen(sprintf("%s.hypernode.io", $brancherHypernode), 22); + if ($connection) { + fclose($connection); + $consecutiveSuccesses++; + $this->log->info( + sprintf( + 'Brancher Hypernode %s reachability check %d/%d succeeded.', + $brancherHypernode, + $consecutiveSuccesses, + $reachabilityCheckCount + ) + ); + + if ($consecutiveSuccesses >= $reachabilityCheckCount) { + return true; + } + + // Only sleep if there's enough time remaining for another check + if ((microtime(true) - $startTime + $reachabilityCheckInterval) < $timeout) { + sleep($reachabilityCheckInterval); + } + } else { + if ($consecutiveSuccesses > 0) { + $this->log->info( + sprintf( + 'Brancher Hypernode %s reachability check failed, resetting counter (was at %d/%d).', + $brancherHypernode, + $consecutiveSuccesses, + $reachabilityCheckCount + ) + ); + } + $consecutiveSuccesses = 0; + + // Only sleep if there's enough time remaining for another check + if ((microtime(true) - $startTime + $reachabilityCheckInterval) < $timeout) { + sleep($reachabilityCheckInterval); + } + } + } + + return false; + } + /** * Wait for brancher Hypernode to become available. * @@ -121,6 +189,81 @@ public function waitForAvailability( int $reachabilityCheckCount = 6, int $reachabilityCheckInterval = 10 ): void { + $this->waitForAvailabilityInternal($brancherHypernode, $timeout, $reachabilityCheckCount, $reachabilityCheckInterval, false); + } + + /** + * Wait for reused brancher Hypernode to become available. + * For reused Branchers, first checks SSH connectivity before checking logbook flows, + * as older Branchers may not have recent logbook entries. + * + * @param string $brancherHypernode Name of the brancher Hypernode + * @param int $timeout Maximum time to wait for availability + * @param int $reachabilityCheckCount Number of consecutive successful checks required + * @param int $reachabilityCheckInterval Seconds between reachability checks + * @return void + * @throws CreateBrancherHypernodeFailedException + * @throws HypernodeApiClientException + * @throws HypernodeApiServerException + * @throws TimeoutException + */ + public function waitForReusedAvailability( + string $brancherHypernode, + int $timeout = 1500, + int $reachabilityCheckCount = 6, + int $reachabilityCheckInterval = 10 + ): void { + $this->waitForAvailabilityInternal($brancherHypernode, $timeout, $reachabilityCheckCount, $reachabilityCheckInterval, true); + } + + /** + * Internal method to wait for brancher Hypernode to become available. + * + * @param string $brancherHypernode Name of the brancher Hypernode + * @param int $timeout Maximum time to wait for availability + * @param int $reachabilityCheckCount Number of consecutive successful checks required + * @param int $reachabilityCheckInterval Seconds between reachability checks + * @param bool $checkSshFirst For reused Branchers, check SSH first before logbook + * @return void + * @throws CreateBrancherHypernodeFailedException + * @throws HypernodeApiClientException + * @throws HypernodeApiServerException + * @throws TimeoutException + */ + private function waitForAvailabilityInternal( + string $brancherHypernode, + int $timeout, + int $reachabilityCheckCount, + int $reachabilityCheckInterval, + bool $checkSshFirst + ): void { + // For reused Branchers, first check if SSH is already reachable + if ($checkSshFirst) { + $this->log->info( + sprintf( + 'Checking if reused brancher Hypernode %s is already reachable via SSH...', + $brancherHypernode + ) + ); + + if ($this->checkSshReachability($brancherHypernode, self::REUSED_SSH_CHECK_TIMEOUT_SECONDS, $reachabilityCheckCount, $reachabilityCheckInterval)) { + $this->log->info( + sprintf( + 'Reused brancher Hypernode %s is already reachable!', + $brancherHypernode + ) + ); + return; + } + + $this->log->info( + sprintf( + 'Reused brancher Hypernode %s is not yet reachable, proceeding with full availability wait...', + $brancherHypernode + ) + ); + } + $latest = microtime(true); $timeElapsed = 0; $resolved = false; @@ -181,46 +324,8 @@ public function waitForAvailability( ); } - $consecutiveSuccesses = 0; - while ($timeElapsed < $timeout) { - $now = microtime(true); - $timeElapsed += $now - $latest; - $latest = $now; - - $connection = @fsockopen(sprintf("%s.hypernode.io", $brancherHypernode), 22); - if ($connection) { - fclose($connection); - $consecutiveSuccesses++; - $this->log->info( - sprintf( - 'Brancher Hypernode %s reachability check %d/%d succeeded.', - $brancherHypernode, - $consecutiveSuccesses, - $reachabilityCheckCount - ) - ); - - if ($consecutiveSuccesses >= $reachabilityCheckCount) { - break; - } - sleep($reachabilityCheckInterval); - } else { - if ($consecutiveSuccesses > 0) { - $this->log->info( - sprintf( - 'Brancher Hypernode %s reachability check failed, resetting counter (was at %d/%d).', - $brancherHypernode, - $consecutiveSuccesses, - $reachabilityCheckCount - ) - ); - } - $consecutiveSuccesses = 0; - sleep($reachabilityCheckInterval); - } - } - - if ($consecutiveSuccesses < $reachabilityCheckCount) { + $remainingTimeout = $timeout - $timeElapsed; + if (!$this->checkSshReachability($brancherHypernode, $remainingTimeout, $reachabilityCheckCount, $reachabilityCheckInterval)) { throw new TimeoutException( sprintf('Timed out waiting for brancher Hypernode %s to become reachable', $brancherHypernode) ); diff --git a/src/DeployRunner.php b/src/DeployRunner.php index 4bb2b15..eed23e3 100644 --- a/src/DeployRunner.php +++ b/src/DeployRunner.php @@ -296,8 +296,10 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche $data = $settings; $data['labels'] = $labels; + $isReused = false; if ($reuseBrancher && $brancherApp = $this->brancherHypernodeManager->reuseExistingBrancherHypernode($parentApp, $labels)) { $this->log->info(sprintf('Found existing brancher Hypernode, name is %s.', $brancherApp)); + $isReused = true; } else { $brancherApp = $this->brancherHypernodeManager->createForHypernode($parentApp, $data); $this->log->info(sprintf('Successfully requested brancher Hypernode, name is %s.', $brancherApp)); @@ -306,12 +308,21 @@ private function maybeConfigureBrancherServer(Server $server, bool $reuseBranche try { $this->log->info('Waiting for brancher Hypernode to become available...'); - $this->brancherHypernodeManager->waitForAvailability( - $brancherApp, - $timeout, - $reachabilityCheckCount, - $reachabilityCheckInterval - ); + if ($isReused) { + $this->brancherHypernodeManager->waitForReusedAvailability( + $brancherApp, + $timeout, + $reachabilityCheckCount, + $reachabilityCheckInterval + ); + } else { + $this->brancherHypernodeManager->waitForAvailability( + $brancherApp, + $timeout, + $reachabilityCheckCount, + $reachabilityCheckInterval + ); + } $this->log->info('Brancher Hypernode has become available!'); } catch (CreateBrancherHypernodeFailedException | TimeoutException $e) { if (in_array($brancherApp, $this->brancherHypernodesRegistered)) {