Skip to content

Commit c9e4c04

Browse files
jochen-testingbotjochen-testingbot
committed
improve monitoring
1 parent d2186d6 commit c9e4c04

File tree

2 files changed

+86
-21
lines changed

2 files changed

+86
-21
lines changed

‎src/main/java/ssh/CustomConnectionMonitor.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@ public class CustomConnectionMonitor {
1616
private final App app;
1717
private Timer timer;
1818
private boolean retrying = false;
19-
private long currentRetryDelay = 5000;
20-
private final int maxRetries = 3;
19+
private int retryAttempts = 0;
20+
private static final long CURRENT_RETRY_DELAY = 5000;
21+
private static final int MAX_RETRIES = 30;
2122

2223
public CustomConnectionMonitor(SSHTunnel tunnel, App app) {
2324
this.tunnel = tunnel;
@@ -37,12 +38,11 @@ public void connectionLost(Throwable reason) {
3738
if (!this.retrying) {
3839
this.retrying = true;
3940
timer = new Timer("Reconnect-" + tunnel.getConnectionId());
40-
timer.schedule(new ReconnectTask(), currentRetryDelay);
41+
timer.schedule(new ReconnectTask(), CURRENT_RETRY_DELAY);
4142
}
4243
}
4344

44-
class ReconnectTask extends TimerTask {
45-
private int retryAttempts = 0;
45+
private class ReconnectTask extends TimerTask {
4646

4747
@Override
4848
public void run() {
@@ -51,15 +51,14 @@ public void run() {
5151

5252
Logger.getLogger(CustomConnectionMonitor.class.getName()).log(Level.INFO,
5353
String.format("[%s] Attempting to re-establish SSH Connection (attempt %d, delay %dms)",
54-
tunnel.getConnectionId(), retryAttempts, currentRetryDelay));
54+
tunnel.getConnectionId(), retryAttempts, CURRENT_RETRY_DELAY));
5555

5656
tunnel.stop();
5757
tunnel.connect();
5858

5959
if (tunnel.isAuthenticated()) {
6060
// Successful reconnection
6161
retrying = false;
62-
retryAttempts = 0;
6362
timer.cancel();
6463

6564
app.getHttpProxy().start();
@@ -68,6 +67,7 @@ public void run() {
6867
Logger.getLogger(CustomConnectionMonitor.class.getName()).log(Level.INFO,
6968
String.format("[%s] Successfully re-established SSH Connection after %d attempts",
7069
tunnel.getConnectionId(), retryAttempts));
70+
CustomConnectionMonitor.this.retryAttempts = 0;
7171
return;
7272
}
7373

@@ -78,15 +78,15 @@ public void run() {
7878
}
7979

8080
// Check if we should continue retrying
81-
if (retryAttempts >= maxRetries) {
81+
if (retryAttempts >= MAX_RETRIES) {
8282
Logger.getLogger(CustomConnectionMonitor.class.getName()).log(Level.WARNING,
8383
String.format("[%s] Giving up retrying after %d attempts. Creating a new Tunnel Connection.",
8484
tunnel.getConnectionId(), retryAttempts));
8585

8686
// Give up connecting to this tunnel VM, try another one
8787
timer.cancel();
8888
retrying = false;
89-
retryAttempts = 0;
89+
CustomConnectionMonitor.this.retryAttempts = 0;
9090

9191
app.stop();
9292
try {
@@ -98,11 +98,11 @@ public void run() {
9898
} else {
9999
timer.cancel();
100100
timer = new Timer("Reconnect-" + tunnel.getConnectionId());
101-
timer.schedule(new ReconnectTask(), currentRetryDelay);
101+
timer.schedule(new ReconnectTask(), CURRENT_RETRY_DELAY);
102102

103103
Logger.getLogger(CustomConnectionMonitor.class.getName()).log(Level.INFO,
104104
String.format("[%s] Will retry in %dms (attempt %d)",
105-
tunnel.getConnectionId(), currentRetryDelay, retryAttempts + 1));
105+
tunnel.getConnectionId(), CURRENT_RETRY_DELAY, retryAttempts + 1));
106106
}
107107
}
108108
}

‎src/main/java/ssh/SSHTunnel.java‎

Lines changed: 75 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.jcraft.jsch.Session;
55
import com.jcraft.jsch.JSchException;
66
import com.testingbot.tunnel.App;
7+
78
import java.io.IOException;
89
import java.util.Timer;
910
import java.util.TimerTask;
@@ -12,7 +13,6 @@
1213
import java.util.logging.Logger;
1314

1415
/**
15-
*
1616
* @author TestingBot
1717
*/
1818
public class SSHTunnel {
@@ -23,9 +23,10 @@ public class SSHTunnel {
2323
private final String connectionId;
2424
private Timer keepAliveTimer;
2525
private Timer connectionMonitorTimer;
26-
private boolean authenticated = false;
26+
private Timer portForwardingMonitorTimer;
2727
private boolean shuttingDown = false;
28-
private CustomConnectionMonitor connectionMonitor;
28+
private final CustomConnectionMonitor connectionMonitor;
29+
private boolean portForwardingEstablished = false;
2930

3031
public SSHTunnel(App app, String server) throws Exception {
3132
/* Create a connection instance */
@@ -58,9 +59,9 @@ public final void connect() throws Exception {
5859
}
5960

6061
// Authentication is done during connect() with JSch
61-
this.authenticated = session.isConnected();
62-
63-
if (!this.authenticated) {
62+
boolean authenticated = session.isConnected();
63+
64+
if (!authenticated) {
6465
Logger.getLogger(SSHTunnel.class.getName()).log(Level.SEVERE,
6566
String.format("[%s] Failed authenticating to the tunnel. Please make sure you are supplying correct login credentials.", connectionId));
6667
throw new Exception("Authentication failed");
@@ -69,10 +70,14 @@ public final void connect() throws Exception {
6970
// Start keep-alive timer with configurable interval
7071
keepAliveTimer = new Timer("KeepAlive-" + connectionId);
7172
keepAliveTimer.schedule(new KeepAliveTask(), 30000, 30000);
72-
73+
7374
// Start connection monitoring timer
7475
connectionMonitorTimer = new Timer("ConnectionMonitor-" + connectionId);
7576
connectionMonitorTimer.schedule(new ConnectionMonitorTask(), 10000, 10000);
77+
78+
// Start port forwarding monitoring timer
79+
portForwardingMonitorTimer = new Timer("PortForwardingMonitor-" + connectionId);
80+
portForwardingMonitorTimer.schedule(new PortForwardingMonitorTask(), 15000, 15000);
7681
}
7782

7883
public void stop(boolean quitting) {
@@ -81,13 +86,16 @@ public void stop(boolean quitting) {
8186
}
8287

8388
public void stop() {
84-
Logger.getLogger(SSHTunnel.class.getName()).log(Level.INFO, String.format("[%s] Stopping secure tunnel",connectionId));
89+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.INFO, String.format("[%s] Stopping secure tunnel", connectionId));
8590
if (keepAliveTimer != null) {
8691
keepAliveTimer.cancel();
8792
}
8893
if (connectionMonitorTimer != null) {
8994
connectionMonitorTimer.cancel();
9095
}
96+
if (portForwardingMonitorTimer != null) {
97+
portForwardingMonitorTimer.cancel();
98+
}
9199

92100
if (session != null && session.isConnected()) {
93101
session.disconnect();
@@ -100,10 +108,12 @@ public void createPortForwarding() {
100108
String hubHost = "hub.testingbot.com";
101109
session.setPortForwardingL(app.getSSHPort(), hubHost, app.getHubPort());
102110

111+
portForwardingEstablished = true;
103112
Logger.getLogger(SSHTunnel.class.getName()).log(Level.INFO,
104113
String.format("[%s] Port forwarding established: %s:2010 -> localhost:%d, localhost:%d -> %s:%d",
105114
connectionId, server, app.getJettyPort(), app.getSSHPort(), hubHost, app.getHubPort()));
106115
} catch (JSchException ex) {
116+
portForwardingEstablished = false;
107117
Logger.getLogger(SSHTunnel.class.getName()).log(Level.SEVERE,
108118
String.format("[%s] Could not setup port forwarding. Please make sure we can make an outbound connection to port 2010.", connectionId), ex);
109119
}
@@ -134,15 +144,15 @@ public void run() {
134144
long roundTripTime = System.currentTimeMillis() - startTime;
135145

136146
Logger.getLogger(SSHTunnel.class.getName()).log(Level.FINE,
137-
String.format("[%s] Keep-alive sent, RTT: %dms", connectionId, roundTripTime));
147+
String.format("[%s] Keep-alive sent, RTT: %dms", connectionId, roundTripTime));
138148
}
139149
} catch (Exception ex) {
140150
Logger.getLogger(SSHTunnel.class.getName()).log(Level.WARNING,
141151
String.format("[%s] Keep-alive failed: %s", connectionId, ex.getMessage()));
142152
}
143153
}
144154
}
145-
155+
146156
class ConnectionMonitorTask extends TimerTask {
147157
@Override
148158
public void run() {
@@ -156,5 +166,60 @@ public void run() {
156166
}
157167
}
158168
}
169+
170+
class PortForwardingMonitorTask extends TimerTask {
171+
@Override
172+
public void run() {
173+
try {
174+
if (session != null && session.isConnected() && !shuttingDown) {
175+
// Check if port forwarding is still active by testing the forwarded ports
176+
String[] forwardedPorts = session.getPortForwardingL();
177+
boolean localForwardingActive = false;
178+
179+
if (forwardedPorts != null) {
180+
for (String port : forwardedPorts) {
181+
if (port.contains(String.valueOf(app.getSSHPort()))) {
182+
localForwardingActive = true;
183+
break;
184+
}
185+
}
186+
}
187+
188+
if (!localForwardingActive && portForwardingEstablished) {
189+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.WARNING,
190+
String.format("[%s] Local port forwarding lost, attempting to restart", connectionId));
191+
restartPortForwarding();
192+
}
193+
}
194+
} catch (Exception ex) {
195+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.WARNING,
196+
String.format("[%s] Port forwarding monitoring failed: %s", connectionId, ex.getMessage()));
197+
}
198+
}
199+
}
200+
201+
private void restartPortForwarding() {
202+
try {
203+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.INFO,
204+
String.format("[%s] Restarting port forwarding", connectionId));
205+
206+
// Clear existing port forwarding
207+
portForwardingEstablished = false;
208+
209+
// Re-establish port forwarding
210+
createPortForwarding();
211+
212+
if (portForwardingEstablished) {
213+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.INFO,
214+
String.format("[%s] Port forwarding successfully restarted", connectionId));
215+
} else {
216+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.SEVERE,
217+
String.format("[%s] Failed to restart port forwarding", connectionId));
218+
}
219+
} catch (Exception ex) {
220+
Logger.getLogger(SSHTunnel.class.getName()).log(Level.SEVERE,
221+
String.format("[%s] Error during port forwarding restart: %s", connectionId, ex.getMessage()), ex);
222+
}
223+
}
159224
}
160225

0 commit comments

Comments
 (0)