Merge pull request #2680 from aiordache/replace_paramiko

Shell out to local SSH client as alternative to a paramiko connection

Author: aiordache

Dockerfile

@@ -2,6 +2,10 @@ ARG PYTHON_VERSION=2.7
 
 FROM python:${PYTHON_VERSION}
 
+# Add SSH keys and set permissions
+COPY tests/ssh-keys /root/.ssh
+RUN chmod -R 600 /root/.ssh
+
 RUN mkdir /src
 WORKDIR /src
 

Jenkinsfile

@@ -3,6 +3,7 @@
 def imageNameBase = "dockerbuildbot/docker-py"
 def imageNamePy2
 def imageNamePy3
+def imageDindSSH
 def images = [:]
 
 def buildImage = { name, buildargs, pyTag ->
@@ -13,7 +14,7 @@ def buildImage = { name, buildargs, pyTag ->
     img = docker.build(name, buildargs)
     img.push()
   }
-  images[pyTag] = img.id
+  if (pyTag?.trim()) images[pyTag] = img.id
 }
 
 def buildImages = { ->
@@ -23,7 +24,9 @@ def buildImages = { ->
 
       imageNamePy2 = "${imageNameBase}:py2-${gitCommit()}"
       imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}"
+      imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}"
 
+      buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "")
       buildImage(imageNamePy2, "-f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 .", "py2.7")
       buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.7 .", "py3.7")
     }
@@ -81,22 +84,37 @@ def runTests = { Map settings ->
         def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}"
         try {
           sh """docker network create ${testNetwork}"""
-          sh """docker run -d --name  ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\
-            docker:${dockerVersion}-dind dockerd -H tcp://0.0.0.0:2375
+          sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\
+            ${imageDindSSH} dockerd -H tcp://0.0.0.0:2375
           """
-          sh """docker run \\
+          sh """docker run --rm \\
             --name ${testContainerName} \\
             -e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\
             -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\
             --network ${testNetwork} \\
             --volumes-from ${dindContainerName} \\
             ${testImage} \\
-            py.test -v -rxs --cov=docker tests/
+            py.test -v -rxs --cov=docker --ignore=tests/ssh tests/
+          """
+          sh """docker stop ${dindContainerName}"""
+          
+          // start DIND container with SSH
+          sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\
+            ${imageDindSSH} dockerd --experimental"""
+          sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """
+          // run SSH tests only
+          sh """docker run --rm \\
+            --name ${testContainerName} \\
+            -e "DOCKER_HOST=ssh://${dindContainerName}:22" \\
+            -e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\
+            --network ${testNetwork} \\
+            --volumes-from ${dindContainerName} \\
+            ${testImage} \\
+            py.test -v -rxs --cov=docker tests/ssh
           """
         } finally {
           sh """
-            docker stop ${dindContainerName} ${testContainerName}
-            docker rm -vf ${dindContainerName} ${testContainerName}
+            docker stop ${dindContainerName}
             docker network rm ${testNetwork}
           """
         }

Makefile

@@ -1,3 +1,6 @@
+TEST_API_VERSION ?= 1.39
+TEST_ENGINE_VERSION ?= 19.03.13
+
 .PHONY: all
 all: test
 
@@ -10,6 +13,10 @@ clean:
 build:
 	docker build -t docker-sdk-python -f tests/Dockerfile --build-arg PYTHON_VERSION=2.7 --build-arg APT_MIRROR .
 
+.PHONY: build-dind-ssh
+build-dind-ssh:
+	docker build -t docker-dind-ssh -f tests/Dockerfile-ssh-dind --build-arg ENGINE_VERSION=${TEST_ENGINE_VERSION} --build-arg API_VERSION=${TEST_API_VERSION} --build-arg APT_MIRROR .
+
 .PHONY: build-py3
 build-py3:
 	docker build -t docker-sdk-python3 -f tests/Dockerfile --build-arg APT_MIRROR .
@@ -41,9 +48,6 @@ integration-test: build
 integration-test-py3: build-py3
 	docker run -t --rm -v /var/run/docker.sock:/var/run/docker.sock docker-sdk-python3 py.test -v tests/integration/${file}
 
-TEST_API_VERSION ?= 1.39
-TEST_ENGINE_VERSION ?= 19.03.12
-
 .PHONY: setup-network
 setup-network:
 	docker network inspect dpy-tests || docker network create dpy-tests
@@ -69,6 +73,29 @@ integration-dind-py3: build-py3 setup-network
 		--network dpy-tests docker-sdk-python3 py.test tests/integration/${file}
 	docker rm -vf dpy-dind-py3
 
+.PHONY: integration-ssh-py2
+integration-ssh-py2: build-dind-ssh build setup-network
+	docker rm -vf dpy-dind-py2 || :
+	docker run -d --network dpy-tests --name dpy-dind-py2 --privileged\
+		docker-dind-ssh dockerd --experimental
+	# start SSH daemon
+	docker exec dpy-dind-py2 sh -c "/usr/sbin/sshd"
+	docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py2" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\
+		--network dpy-tests docker-sdk-python py.test tests/ssh/${file}
+	docker rm -vf dpy-dind-py2
+
+.PHONY: integration-ssh-py3
+integration-ssh-py3: build-dind-ssh build-py3 setup-network
+	docker rm -vf dpy-dind-py3 || :
+	docker run -d --network dpy-tests --name dpy-dind-py3 --privileged\
+		docker-dind-ssh dockerd --experimental
+	# start SSH daemon
+	docker exec dpy-dind-py3 sh -c "/usr/sbin/sshd"
+	docker run -t --rm --env="DOCKER_HOST=ssh://dpy-dind-py3" --env="DOCKER_TEST_API_VERSION=${TEST_API_VERSION}"\
+		--network dpy-tests docker-sdk-python3 py.test tests/ssh/${file}
+	docker rm -vf dpy-dind-py3
+
+
 .PHONY: integration-dind-ssl
 integration-dind-ssl: build-dind-certs build build-py3
 	docker rm -vf dpy-dind-certs dpy-dind-ssl || :

docker/api/client.py

@@ -89,6 +89,9 @@ class APIClient(
         user_agent (str): Set a custom user agent for requests to the server.
         credstore_env (dict): Override environment variables when calling the
             credential store process.
+        use_ssh_client (bool): If set to `True`, an ssh connection is made
+            via shelling out to the ssh client. Ensure the ssh client is
+            installed and configured on the host.
     """
 
     __attrs__ = requests.Session.__attrs__ + ['_auth_configs',
@@ -100,7 +103,7 @@ class APIClient(
     def __init__(self, base_url=None, version=None,
                  timeout=DEFAULT_TIMEOUT_SECONDS, tls=False,
                  user_agent=DEFAULT_USER_AGENT, num_pools=None,
-                 credstore_env=None):
+                 credstore_env=None, use_ssh_client=False):
         super(APIClient, self).__init__()
 
         if tls and not base_url:
@@ -161,7 +164,8 @@ def __init__(self, base_url=None, version=None,
         elif base_url.startswith('ssh://'):
             try:
                 self._custom_adapter = SSHHTTPAdapter(
-                    base_url, timeout, pool_connections=num_pools
+                    base_url, timeout, pool_connections=num_pools,
+                    shell_out=use_ssh_client
                 )
             except NameError:
                 raise DockerException(

docker/client.py

@@ -35,6 +35,9 @@ class DockerClient(object):
         user_agent (str): Set a custom user agent for requests to the server.
         credstore_env (dict): Override environment variables when calling the
             credential store process.
+        use_ssh_client (bool): If set to `True`, an ssh connection is made
+            via shelling out to the ssh client. Ensure the ssh client is
+            installed and configured on the host.
     """
     def __init__(self, *args, **kwargs):
         self.api = APIClient(*args, **kwargs)
@@ -70,6 +73,9 @@ def from_env(cls, **kwargs):
                 from. Default: the value of ``os.environ``
             credstore_env (dict): Override environment variables when calling
                 the credential store process.
+            use_ssh_client (bool): If set to `True`, an ssh connection is
+                made via shelling out to the ssh client. Ensure the ssh
+                client is installed and configured on the host.
 
         Example:
 
@@ -81,8 +87,12 @@ def from_env(cls, **kwargs):
         """
         timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS)
         version = kwargs.pop('version', None)
+        use_ssh_client = kwargs.pop('use_ssh_client', False)
         return cls(
-            timeout=timeout, version=version, **kwargs_from_env(**kwargs)
+            timeout=timeout,
+            version=version,
+            use_ssh_client=use_ssh_client,
+            **kwargs_from_env(**kwargs)
         )
 
     # Resources