(MAINT) debug ssh more

Author: davidmalloncares

.github/workflows/test-install-matrix.yaml

@@ -60,6 +60,10 @@ jobs:
             echo 'Host *'                      >  $HOME/.ssh/config
             echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
             echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            echo '    StrictHostKeyChecking no' >> $HOME/.ssh/config
+            echo '    UserKnownHostsFile /dev/null' >> $HOME/.ssh/config
+            echo '    ConnectTimeout 30'       >> $HOME/.ssh/config
+            echo '    ConnectionAttempts 10'   >> $HOME/.ssh/config
             bundle exec rake spec_prep
           echo ::endgroup::
           echo ::group::provision
@@ -75,6 +79,9 @@ jobs:
           echo ::endgroup::
           echo ::group::update every 'ssh:' tag in ./inventory.yaml file to add 'native-ssh: true' under install_test_cluster and indent correctly
             sed -i -e '/ssh:/a\        native-ssh: true' ./inventory.yaml || true; echo
+            # Also add additional SSH options for container environments
+            sed -i -e '/ssh:/a\        connect-timeout: 30' ./inventory.yaml || true; echo
+            sed -i -e '/ssh:/a\        host-key-check: false' ./inventory.yaml || true; echo
           echo ::endgroup::
           echo ::group::list contents of ./inventory.yaml
             ls -l ./inventory.yaml || true; echo
@@ -96,13 +103,50 @@ jobs:
             # Check if containers are running
             docker ps
             
-            # Try to connect manually to each container
+            # Check SSH processes in containers
             for container in $(docker ps --format "table {{.Names}}" | tail -n +2); do
               echo "Testing SSH to container: $container"
-              timeout 30 docker exec $container systemctl status sshd || echo "SSH service check failed for $container"
-              timeout 30 docker exec $container ps aux | grep sshd || echo "No sshd processes found in $container"
+              echo "SSH processes in $container:"
+              docker exec $container ps aux | grep sshd || echo "No sshd processes found in $container"
+              
+              # Test SSH connectivity directly
+              echo "Testing direct SSH connection to $container:"
+              container_ip=$(docker inspect $container | jq -r '.[0].NetworkSettings.IPAddress')
+              echo "Container IP: $container_ip"
+              
+              # Get the mapped SSH port
+              ssh_port=$(docker port $container 22 | cut -d: -f2)
+              echo "SSH port mapping: localhost:$ssh_port -> $container:22"
+              
+              # Test SSH connection with timeout
+              timeout 10 ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -p $ssh_port root@localhost 'echo "SSH connection successful"' || echo "SSH connection failed to $container"
             done
-          echo ::endgroup::            
+          echo ::endgroup::
+      - name: Wait for SSH to be ready
+        run: |
+          echo ::group::wait_for_ssh
+            # Wait for SSH to be available on all containers via Bolt
+            for i in {1..12}; do
+              echo "Attempt $i: Testing Bolt SSH connectivity..."
+              if bundle exec bolt command run 'echo "Bolt SSH test successful"' \
+                --inventoryfile ./inventory.yaml \
+                --targets all \
+                --connect-timeout 10 \
+                --no-host-key-check \
+                --transport ssh; then
+                echo "All containers are accessible via Bolt SSH!"
+                break
+              fi
+              if [ $i -eq 12 ]; then
+                echo "Containers failed to become accessible after 12 attempts"
+                echo "Final inventory check:"
+                cat ./inventory.yaml
+                exit 1
+              fi
+              echo "Waiting 10 seconds before retry..."
+              sleep 10
+            done
+          echo ::endgroup::        
       - name: Install PE on test cluster
         timeout-minutes: 120
         run: |