From 03bcf6939b1324f80a7023bb4e2373df10d64183 Mon Sep 17 00:00:00 2001 From: "Marcel M. Cary" Date: Mon, 9 Nov 2020 11:59:57 -0800 Subject: [PATCH 1/3] Add connection lifecycle debugging info for ssh I'd like to confirm my understanding of when SSH and dial connections are established with the docker-modem agent for SSH. Call "debug" a key points when connections are established and ended. --- lib/ssh.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/ssh.js b/lib/ssh.js index f650413..7ccbde4 100644 --- a/lib/ssh.js +++ b/lib/ssh.js @@ -1,14 +1,19 @@ var Client = require('ssh2').Client, http = require('http'); +var debug = require('debug')('modem.ssh'); module.exports = function(opt) { var conn = new Client(); var agent = new http.Agent(); agent.createConnection = function(options, fn) { + debug('createConnection'); conn.once('ready', function() { + debug('ready'); conn.exec('docker system dial-stdio', function(err, stream) { + debug("dialed") if (err) { + debug('error'); conn.end(); agent.destroy(); return; @@ -17,13 +22,17 @@ module.exports = function(opt) { fn(null, stream); stream.once('close', () => { + debug('close'); conn.end(); agent.destroy(); }); }); }).connect(opt); - conn.once('end', () => agent.destroy()); + conn.once('end', () => { + debug('end') + agent.destroy() + }); }; return agent; From 689a9d5c0b1d925cc4b8bc14a9445e6c62795d6d Mon Sep 17 00:00:00 2001 From: "Marcel M. Cary" Date: Mon, 9 Nov 2020 12:01:08 -0800 Subject: [PATCH 2/3] Customize docker-modem ssh agent's connection lifetime When implicitly using the docker-modem agent for SSH by specifying "protocol" as a connection option, a new agent is created for each request. However, it is not suitable when passed as the "agent" option to Modem(), in which case the same agent will be reused across requests. This is because the docker-modem ssh agent shares one SSH client for all connections of the HTTP agent whose createConnection method it overrides. However, the agent appears to be capable of handling multiple exec calls in parallel. This means that when one stream closes, the client will be ended, disrupting any other connections still open. This is fine for short-lived sequential HTTP requests made with the agent, but if we create and attach to a docker container and then make another request (for example, to start it), the third request will disconnect the attached streams from the attach request. Mon, 09 Nov 2020 19:33:40 GMT modem.ssh1 createConnection 2020-11-09T19:33:40.013Z modem Sending: { path: /containers/create?Image=ubuntu&AttachStdin=true&AttachStdout=true&AttachStderr=true&Tty=true&Cmd=%2Fbin%2Fbash&OpenStdin=true, ... } Mon, 09 Nov 2020 19:33:40 GMT modem.ssh1 ready Mon, 09 Nov 2020 19:33:40 GMT modem.ssh1 dialed 2020-11-09T19:33:40.251Z modem Received: {"Id":"d13f119a1a1e60b9b54d6685e3c8e6451221b68fa9baee7f5d9c9b0ac06e52b6","Warnings":[]} Mon, 09 Nov 2020 19:33:40 GMT modem.ssh1 close Mon, 09 Nov 2020 19:33:40 GMT modem.ssh1 end # sleep after request to createContainer Mon, 09 Nov 2020 19:33:42 GMT modem.ssh2 createConnection 2020-11-09T19:33:42.258Z modem Sending: { path: /containers/d13f119a1a1e60b9b54d6685e3c8e6451221b68fa9baee7f5d9c9b0ac06e52b6/attach?stream=true&stdout=true&stderr=true&stdin=true, ... } Mon, 09 Nov 2020 19:33:42 GMT modem.ssh2 ready Mon, 09 Nov 2020 19:33:42 GMT modem.ssh2 dialed # sleep after request to "attach": no "end" or "close" logged because # "attach" holds the connection open Mon, 09 Nov 2020 19:33:44 GMT modem.ssh3 createConnection 2020-11-09T19:33:44.417Z modem Sending: { path: /containers/d13f119a1a1e60b9b54d6685e3c8e6451221b68fa9baee7f5d9c9b0ac06e52b6/start, ... } # both streams are closed (the one for "attach" and for "start") Mon, 09 Nov 2020 19:33:44 GMT modem.ssh2 end Mon, 09 Nov 2020 19:33:44 GMT modem.ssh3 end Mon, 09 Nov 2020 19:33:44 GMT modem.ssh3 ready Mon, 09 Nov 2020 19:33:44 GMT modem.ssh3 dialed 2020-11-09T19:33:44.964Z modem Received: Mon, 09 Nov 2020 19:33:44 GMT modem.ssh3 close Make the ssh connection local to the http agent, which creates a new, separate ssh connection each time the http agent requests a new connection. This allows "attach" to stay attached. --- lib/ssh.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/lib/ssh.js b/lib/ssh.js index 7ccbde4..b19b0bf 100644 --- a/lib/ssh.js +++ b/lib/ssh.js @@ -3,11 +3,11 @@ var Client = require('ssh2').Client, var debug = require('debug')('modem.ssh'); module.exports = function(opt) { - var conn = new Client(); var agent = new http.Agent(); agent.createConnection = function(options, fn) { debug('createConnection'); + var conn = new Client(); conn.once('ready', function() { debug('ready'); conn.exec('docker system dial-stdio', function(err, stream) { @@ -15,7 +15,6 @@ module.exports = function(opt) { if (err) { debug('error'); conn.end(); - agent.destroy(); return; } @@ -24,15 +23,9 @@ module.exports = function(opt) { stream.once('close', () => { debug('close'); conn.end(); - agent.destroy(); }); }); }).connect(opt); - - conn.once('end', () => { - debug('end') - agent.destroy() - }); }; return agent; From 5bdaec1ed139028208074a14d65a6dc2fbbc23ae Mon Sep 17 00:00:00 2001 From: "Marcel M. Cary" Date: Mon, 9 Nov 2020 12:02:55 -0800 Subject: [PATCH 3/3] Establish multiple docker connections over one SSH connection SSH is capable of multiplexing multiple streams over one ssh-authenticated connection, but we are establishing a new connection for each http agent connection. Connect over SSH only once for the modem instance and create new docker connections by calling dial as many times as needed to open new multiplexed streams. In other words, create this situation: PID USER RSS ARGS 24566 root 8940 \_ sshd: root@notty 24620 root 57600 | \_ docker system dial-stdio 24621 root 56936 | \_ docker system dial-stdio 24622 root 57152 | \_ docker system dial-stdio not this one: 23901 root 8884 \_ sshd: root@notty 23954 root 58268 | \_ docker system dial-stdio 23911 root 8796 \_ sshd: root@notty 23978 root 58020 | \_ docker system dial-stdio 23924 root 8844 \_ sshd: root@notty 23991 root 58180 | \_ docker system dial-stdio Trace: modem.ssh createConnection +0ms modem Sending: { path: /containers/create?Image=ubuntu&AttachStdin=true&AttachStdout=true&AttachStderr=true&Tty=true&Cmd=%2Fbin%2Fbash&OpenStdin=true, ... } +0ms modem.ssh ready +52ms modem.ssh dialed +80ms modem Received: {"Id":"9d435ecdff50659bc7b602119c2e364ee11e9110bffc3af239d5a21623efec2a","Warnings":[]} + +254ms modem.ssh createConnection +129ms modem Sending: { path: /containers/9d435ecdff50659bc7b602119c2e364ee11e9110bffc3af239d5a21623efec2a/attach?stream=true&stdout=true&stderr=true&stdin=true, ... } +2ms modem.ssh close +2ms modem.ssh dialed +8ms modem.ssh createConnection +44ms modem Sending: { path: /containers/9d435ecdff50659bc7b602119c2e364ee11e9110bffc3af239d5a21623efec2a/start, ... } +55ms modem.ssh dialed +2ms modem Received: +407ms modem.ssh close +434ms Performance when connecting to a host on the other side of the world (a worst-case scenario) improves from 16-20 sec to 6-8 sec to make these three calls plus a kill and wait (5 calls total). --- lib/ssh.js | 55 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/ssh.js b/lib/ssh.js index b19b0bf..c320e8a 100644 --- a/lib/ssh.js +++ b/lib/ssh.js @@ -4,29 +4,44 @@ var debug = require('debug')('modem.ssh'); module.exports = function(opt) { var agent = new http.Agent(); + var conn = new Client(); + var ready = false; agent.createConnection = function(options, fn) { - debug('createConnection'); - var conn = new Client(); - conn.once('ready', function() { - debug('ready'); - conn.exec('docker system dial-stdio', function(err, stream) { - debug("dialed") - if (err) { - debug('error'); - conn.end(); - return; - } - - fn(null, stream); - - stream.once('close', () => { - debug('close'); - conn.end(); - }); - }); - }).connect(opt); + debug('createConnection') + if (ready) { + dial(conn, fn); + } else { + conn.once('ready', function() { + ready = true; + debug('ready'); + dial(conn, fn); + }).connect(opt); + } }; + agent.destroy_without_ssh = agent.destroy; + agent.destroy = function() { + conn.end(); + agent.destroy_without_ssh(); + } + return agent; }; + +function dial(conn, fn) { + conn.exec('docker system dial-stdio', function(err, stream) { + debug('dialed') + if (err) { + debug('error'); + return; + } + + fn(null, stream); + + stream.once('close', () => { + debug('close') + stream.end(); + }); + }); +}