Skip to content

Commit 23ad267

Browse files
trevjtrevj
committed
retry SSH connections with exponential backoff, for firefox
1 parent a193f6e commit 23ad267

File tree

2 files changed

+90
-79
lines changed

2 files changed

+90
-79
lines changed

‎src/cloud/install/installer.ts‎

Lines changed: 84 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import arraybuffers = require('../../arraybuffers/arraybuffers');
77
import linefeeder = require('../../net/linefeeder');
88
import logging = require('../../logging/logging');
9+
import promises = require('../../promises/promises');
910
import queue = require('../../handler/queue');
1011

1112
// https://github.com/borisyankov/DefinitelyTyped/blob/master/ssh2/ssh2-tests.ts
@@ -26,6 +27,10 @@ const PROGRESS_PREFIX = 'CLOUD_INSTALL_PROGRESS';
2627
// Prefix for status updates.
2728
const STATUS_PREFIX = 'CLOUD_INSTALL_STATUS';
2829

30+
// Retry timing for SSH connection establishment.
31+
const INITIAL_CONNECTION_INTERVAL_MS = 500;
32+
const MAX_CONNECTION_INTERVAL_MS = 10000;
33+
2934
// Installs uProxy on a server, via SSH.
3035
// The process is as close as possible to a manual install
3136
// so that we have fewer paths to test.
@@ -50,86 +55,90 @@ class CloudInstaller {
5055
debug: undefined
5156
};
5257

53-
const connection = new Client();
54-
return new Promise<string>((F, R) => {
55-
connection.on('ready', () => {
56-
log.debug('logged into server');
57-
58-
const stdoutRaw = new queue.Queue<ArrayBuffer, void>();
59-
const stdout = new linefeeder.LineFeeder(stdoutRaw);
60-
stdout.setSyncHandler((line: string) => {
61-
log.debug('STDOUT: %1', line);
62-
// Search for the URL anywhere in the line so we will
63-
// continue to work in the face of minor changes
64-
// to the install script.
65-
if (line.indexOf(INVITATION_PREFIX) === 0) {
66-
const inviteJson = line.substring(INVITATION_PREFIX.length);
67-
try {
68-
F(JSON.parse(inviteJson));
69-
} catch (e) {
58+
let numAttempts = 0;
59+
return promises.retryWithExponentialBackoff(() => {
60+
log.debug('connection attempt %1...', (++numAttempts));
61+
return new Promise<string>((F, R) => {
62+
const connection = new Client();
63+
connection.on('ready', () => {
64+
log.debug('logged into server');
65+
66+
const stdoutRaw = new queue.Queue<ArrayBuffer, void>();
67+
const stdout = new linefeeder.LineFeeder(stdoutRaw);
68+
stdout.setSyncHandler((line: string) => {
69+
log.debug('STDOUT: %1', line);
70+
// Search for the URL anywhere in the line so we will
71+
// continue to work in the face of minor changes
72+
// to the install script.
73+
if (line.indexOf(INVITATION_PREFIX) === 0) {
74+
const inviteJson = line.substring(INVITATION_PREFIX.length);
75+
try {
76+
F(JSON.parse(inviteJson));
77+
} catch (e) {
78+
R({
79+
message: 'could not parse invite: ' + inviteJson
80+
});
81+
}
82+
} else if (line.indexOf(PROGRESS_PREFIX) === 0) {
83+
const tokens = line.split(' ');
84+
if (tokens.length < 2) {
85+
log.warn('could not parse progress update');
86+
} else {
87+
const progress = parseInt(tokens[1], 10);
88+
this.dispatchEvent_('progress', progress);
89+
}
90+
} else if (line.indexOf(STATUS_PREFIX) === 0) {
91+
this.dispatchEvent_('status', line);
92+
}
93+
});
94+
95+
const stderrRaw = new queue.Queue<ArrayBuffer, void>();
96+
const stderr = new linefeeder.LineFeeder(stderrRaw);
97+
stderr.setSyncHandler((line: string) => {
98+
log.error('STDERR: %1', line);
99+
});
100+
101+
connection.exec(INSTALL_COMMAND, (e: Error, stream: ssh2.Channel) => {
102+
if (e) {
103+
connection.end();
70104
R({
71-
message: 'could not parse invite: ' + inviteJson
105+
message: 'could not execute command: ' + e.message
72106
});
107+
return;
73108
}
74-
} else if (line.indexOf(PROGRESS_PREFIX) === 0) {
75-
const tokens = line.split(' ');
76-
if (tokens.length < 2) {
77-
log.warn('could not parse progress update');
78-
} else {
79-
const progress = parseInt(tokens[1], 10);
80-
this.dispatchEvent_('progress', progress);
81-
}
82-
} else if (line.indexOf(STATUS_PREFIX) === 0) {
83-
this.dispatchEvent_('status', line);
84-
}
85-
});
86-
87-
const stderrRaw = new queue.Queue<ArrayBuffer, void>();
88-
const stderr = new linefeeder.LineFeeder(stderrRaw);
89-
stderr.setSyncHandler((line: string) => {
90-
log.error('STDERR: %1', line);
91-
});
92-
93-
connection.exec(INSTALL_COMMAND, (e: Error, stream: ssh2.Channel) => {
94-
if (e) {
95-
connection.end();
96-
R({
97-
message: 'could not execute command: ' + e.message
98-
});
99-
return;
100-
}
101-
stream.on('end', () => {
102-
stdout.flush();
103-
stderr.flush();
104-
connection.end();
105-
R({
106-
message: 'invitation URL not found'
109+
stream.on('end', () => {
110+
stdout.flush();
111+
stderr.flush();
112+
connection.end();
113+
R({
114+
message: 'invitation URL not found'
115+
});
116+
}).on('data', (data: Buffer) => {
117+
stdoutRaw.handle(arraybuffers.bufferToArrayBuffer(data));
118+
}).stderr.on('data', (data: Buffer) => {
119+
stderrRaw.handle(arraybuffers.bufferToArrayBuffer(data));
107120
});
108-
}).on('data', (data:Buffer) => {
109-
stdoutRaw.handle(arraybuffers.bufferToArrayBuffer(data));
110-
}).stderr.on('data', (data: Buffer) => {
111-
stderrRaw.handle(arraybuffers.bufferToArrayBuffer(data));
112121
});
113-
});
114-
}).on('error', (e: Error) => {
115-
// This occurs when:
116-
// - user supplies the wrong username or password
117-
// - host cannot be reached, e.g. non-existant hostname
118-
R({
119-
message: 'could not login: ' + e.message
120-
});
121-
}).on('end', () => {
122-
log.debug('connection end');
123-
R({
124-
message: 'connection end without invitation URL'
125-
});
126-
}).on('close', (hadError: boolean) => {
127-
log.debug('connection close, with%1 error', (hadError ? '' : 'out'));
128-
R({
129-
message: 'connection close without invitation URL'
130-
});
131-
}).connect(connectConfig);
132-
});
122+
}).on('error', (e: Error) => {
123+
// This occurs when:
124+
// - user supplies the wrong username or password
125+
// - host cannot be reached, e.g. non-existant hostname
126+
R({
127+
message: 'could not login: ' + e.message
128+
});
129+
}).on('end', () => {
130+
log.debug('connection end');
131+
R({
132+
message: 'connection end without invitation URL'
133+
});
134+
}).on('close', (hadError: boolean) => {
135+
log.debug('connection close, with%1 error', (hadError ? '' : 'out'));
136+
R({
137+
message: 'connection close without invitation URL'
138+
});
139+
}).connect(connectConfig);
140+
});
141+
}, MAX_CONNECTION_INTERVAL_MS, INITIAL_CONNECTION_INTERVAL_MS);
133142
}
134143
}
135144

‎src/cloud/social/provider.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,9 @@ const STORAGE_KEY = 'cloud-social-contacts';
2626
// Timeout for establishing an SSH connection.
2727
const CONNECT_TIMEOUT_MS = 10000;
2828

29-
// Maximum number of times to try establishing an SSH connection.
30-
const MAX_CONNECT_ATTEMPTS = 3;
29+
// Retry timing for SSH connection establishment.
30+
const INITIAL_CONNECTION_INTERVAL_MS = 500;
31+
const MAX_CONNECTION_INTERVAL_MS = 10000;
3132

3233
// Credentials for accessing a cloud instance.
3334
// The serialised, base64 form is distributed amongst users.
@@ -218,8 +219,9 @@ export class CloudSocialProvider {
218219
});
219220
};
220221

221-
this.clients_[invite.host] = promises.retry(connect,
222-
MAX_CONNECT_ATTEMPTS).then((connection:Connection) => {
222+
this.clients_[invite.host] = promises.retryWithExponentialBackoff(connect,
223+
MAX_CONNECTION_INTERVAL_MS, INITIAL_CONNECTION_INTERVAL_MS).then(
224+
(connection:Connection) => {
223225
log.info('connected to zork on %1', invite.host);
224226

225227
// Fetch the banner, if available, then emit an instance message.

0 commit comments

Comments
 (0)