Merge upstream branch 'master' into feature/logs_since

Signed-off-by: Viacheslav Boiko <[email protected]>

Author: Viacheslav Boiko

Makefile

@@ -11,25 +11,35 @@ build:
 build-py3:
 	docker build -t docker-py3 -f Dockerfile-py3 .
 
-test: flake8 unit-test unit-test-py3 integration-dind
+build-dind-certs:
+	docker build -t dpy-dind-certs -f tests/Dockerfile-dind-certs .
+
+test: flake8 unit-test unit-test-py3 integration-dind integration-dind-ssl
 
 unit-test: build
-	docker run docker-py py.test tests/test.py tests/utils_test.py
+	docker run docker-py py.test tests/unit
 
 unit-test-py3: build-py3
-	docker run docker-py3 py.test tests/test.py tests/utils_test.py
+	docker run docker-py3 py.test tests/unit
 
 integration-test: build
-	docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test -rxs tests/integration_test.py
+	docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py py.test tests/integration
 
 integration-test-py3: build-py3
-	docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test -rxs tests/integration_test.py
+	docker run -v /var/run/docker.sock:/var/run/docker.sock docker-py3 py.test tests/integration
 
 integration-dind: build build-py3
-	docker run -d --name dpy-dind --privileged dockerswarm/dind:1.8.1 docker -d -H tcp://0.0.0.0:2375
-	docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py py.test -rxs tests/integration_test.py
-	docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3 py.test -rxs tests/integration_test.py
+	docker run -d --name dpy-dind --env="DOCKER_HOST=tcp://localhost:2375" --privileged dockerswarm/dind:1.8.1 docker -d -H tcp://0.0.0.0:2375
+	docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py py.test tests/integration
+	docker run --env="DOCKER_HOST=tcp://docker:2375" --link=dpy-dind:docker docker-py3 py.test tests/integration
 	docker rm -vf dpy-dind
 
+integration-dind-ssl: build-dind-certs build build-py3
+	docker run -d --name dpy-dind-certs dpy-dind-certs
+	docker run -d --env="DOCKER_HOST=tcp://localhost:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --volumes-from dpy-dind-certs --name dpy-dind-ssl -v /tmp --privileged dockerswarm/dind:1.8.1 docker daemon --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/server-cert.pem --tlskey=/certs/server-key.pem -H tcp://0.0.0.0:2375
+	docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --link=dpy-dind-ssl:docker docker-py py.test tests/integration_test.py
+	docker run --volumes-from dpy-dind-ssl --env="DOCKER_HOST=tcp://docker:2375" --env="DOCKER_TLS_VERIFY=1" --env="DOCKER_CERT_PATH=/certs" --link=dpy-dind-ssl:docker docker-py3 py.test tests/integration_test.py
+	docker rm -vf dpy-dind-ssl dpy-dind-certs
+
 flake8: build
-	docker run docker-py flake8 docker tests
+	docker run docker-py flake8 docker tests

docker/api/__init__.py

@@ -5,3 +5,4 @@
 from .exec_api import ExecApiMixin
 from .image import ImageApiMixin
 from .volume import VolumeApiMixin
+from .network import NetworkApiMixin

docker/api/container.py

@@ -76,6 +76,12 @@ def containers(self, quiet=False, all=False, trunc=False, latest=False,
 
     @utils.check_resource
     def copy(self, container, resource):
+        if utils.version_gte(self._version, '1.20'):
+            warnings.warn(
+                'Client.copy() is deprecated for API version >= 1.20, '
+                'please use get_archive() instead',
+                DeprecationWarning
+            )
         res = self._post_json(
             self._url("/containers/{0}/copy".format(container)),
             data={"Resource": resource},
@@ -146,6 +152,21 @@ def export(self, container):
         self._raise_for_status(res)
         return res.raw
 
+    @utils.check_resource
+    @utils.minimum_version('1.20')
+    def get_archive(self, container, path):
+        params = {
+            'path': path
+        }
+        url = self._url('/containers/{0}/archive', container)
+        res = self._get(url, params=params, stream=True)
+        self._raise_for_status(res)
+        encoded_stat = res.headers.get('x-docker-container-path-stat')
+        return (
+            res.raw,
+            utils.decode_json_header(encoded_stat) if encoded_stat else None
+        )
+
     @utils.check_resource
     def inspect_container(self, container):
         return self._result(
@@ -226,6 +247,15 @@ def port(self, container, private_port):
 
         return h_ports
 
+    @utils.check_resource
+    @utils.minimum_version('1.20')
+    def put_archive(self, container, path, data):
+        params = {'path': path}
+        url = self._url('/containers/{0}/archive', container)
+        res = self._put(url, params=params, data=data)
+        self._raise_for_status(res)
+        return res.status_code == 200
+
     @utils.check_resource
     def remove_container(self, container, v=False, link=False, force=False):
         params = {'v': v, 'link': link, 'force': force}

docker/api/exec_api.py

@@ -1,5 +1,3 @@
-import shlex
-
 import six
 
 from .. import errors
@@ -20,7 +18,7 @@ def exec_create(self, container, cmd, stdout=True, stderr=True, tty=False,
                 'User-specific exec is not supported in API < 1.19'
             )
         if isinstance(cmd, six.string_types):
-            cmd = shlex.split(str(cmd))
+            cmd = utils.split_command(cmd)
 
         data = {
             'Container': container,

docker/api/network.py

@@ -0,0 +1,55 @@
+import json
+
+from ..utils import check_resource, minimum_version
+
+
+class NetworkApiMixin(object):
+    @minimum_version('1.21')
+    def networks(self, names=None, ids=None):
+        filters = {}
+        if names:
+            filters['name'] = names
+        if ids:
+            filters['id'] = ids
+
+        params = {'filters': json.dumps(filters)}
+
+        url = self._url("/networks")
+        res = self._get(url, params=params)
+        return self._result(res, json=True)
+
+    @minimum_version('1.21')
+    def create_network(self, name, driver=None):
+        data = {
+            'name': name,
+            'driver': driver,
+        }
+        url = self._url("/networks/create")
+        res = self._post_json(url, data=data)
+        return self._result(res, json=True)
+
+    @minimum_version('1.21')
+    def remove_network(self, net_id):
+        url = self._url("/networks/{0}", net_id)
+        res = self._delete(url)
+        self._raise_for_status(res)
+
+    @minimum_version('1.21')
+    def inspect_network(self, net_id):
+        url = self._url("/networks/{0}", net_id)
+        res = self._get(url)
+        return self._result(res, json=True)
+
+    @check_resource
+    @minimum_version('1.21')
+    def connect_container_to_network(self, container, net_id):
+        data = {"container": container}
+        url = self._url("/networks/{0}/connect", net_id)
+        self._post_json(url, data=data)
+
+    @check_resource
+    @minimum_version('1.21')
+    def disconnect_container_from_network(self, container, net_id):
+        data = {"container": container}
+        url = self._url("/networks/{0}/disconnect", net_id)
+        self._post_json(url, data=data)

docker/api/volume.py

@@ -12,7 +12,7 @@ def volumes(self, filters=None):
 
     @utils.minimum_version('1.21')
     def create_volume(self, name, driver=None, driver_opts=None):
-        url = self._url('/volumes')
+        url = self._url('/volumes/create')
         if driver_opts is not None and not isinstance(driver_opts, dict):
             raise TypeError('driver_opts must be a dictionary')
 

docker/auth/auth.py

@@ -13,7 +13,6 @@
 #    limitations under the License.
 
 import base64
-import fileinput
 import json
 import logging
 import os
@@ -102,7 +101,7 @@ def decode_auth(auth):
 
 def encode_header(auth):
     auth_json = json.dumps(auth).encode('ascii')
-    return base64.b64encode(auth_json)
+    return base64.urlsafe_b64encode(auth_json)
 
 
 def parse_auth(entries):
@@ -132,78 +131,79 @@ def parse_auth(entries):
     return conf
 
 
+def find_config_file(config_path=None):
+    environment_path = os.path.join(
+        os.environ.get('DOCKER_CONFIG'),
+        os.path.basename(DOCKER_CONFIG_FILENAME)
+    ) if os.environ.get('DOCKER_CONFIG') else None
+
+    paths = [
+        config_path,  # 1
+        environment_path,  # 2
+        os.path.join(os.path.expanduser('~'), DOCKER_CONFIG_FILENAME),  # 3
+        os.path.join(
+            os.path.expanduser('~'), LEGACY_DOCKER_CONFIG_FILENAME
+        )  # 4
+    ]
+
+    for path in paths:
+        if path and os.path.exists(path):
+            return path
+    return None
+
+
 def load_config(config_path=None):
     """
     Loads authentication data from a Docker configuration file in the given
     root directory or if config_path is passed use given path.
+    Lookup priority:
+        explicit config_path parameter > DOCKER_CONFIG environment variable >
+        ~/.docker/config.json > ~/.dockercfg
     """
-    conf = {}
-    data = None
-
-    # Prefer ~/.docker/config.json.
-    config_file = config_path or os.path.join(os.path.expanduser('~'),
-                                              DOCKER_CONFIG_FILENAME)
-
-    log.debug("Trying {0}".format(config_file))
-
-    if os.path.exists(config_file):
-        try:
-            with open(config_file) as f:
-                for section, data in six.iteritems(json.load(f)):
-                    if section != 'auths':
-                        continue
-                    log.debug("Found 'auths' section")
-                    return parse_auth(data)
-            log.debug("Couldn't find 'auths' section")
-        except (IOError, KeyError, ValueError) as e:
-            # Likely missing new Docker config file or it's in an
-            # unknown format, continue to attempt to read old location
-            # and format.
-            log.debug(e)
-            pass
-    else:
-        log.debug("File doesn't exist")
-
-    config_file = config_path or os.path.join(os.path.expanduser('~'),
-                                              LEGACY_DOCKER_CONFIG_FILENAME)
 
-    log.debug("Trying {0}".format(config_file))
+    config_file = find_config_file(config_path)
 
-    if not os.path.exists(config_file):
-        log.debug("File doesn't exist - returning empty config")
+    if not config_file:
+        log.debug("File doesn't exist")
         return {}
 
-    log.debug("Attempting to parse as JSON")
     try:
         with open(config_file) as f:
-            return parse_auth(json.load(f))
-    except Exception as e:
+            data = json.load(f)
+            if data.get('auths'):
+                log.debug("Found 'auths' section")
+                return parse_auth(data['auths'])
+            else:
+                log.debug("Couldn't find 'auths' section")
+                f.seek(0)
+                return parse_auth(json.load(f))
+    except (IOError, KeyError, ValueError) as e:
+        # Likely missing new Docker config file or it's in an
+        # unknown format, continue to attempt to read old location
+        # and format.
         log.debug(e)
-        pass
 
-    # If that fails, we assume the configuration file contains a single
-    # authentication token for the public registry in the following format:
-    #
-    # auth = AUTH_TOKEN
-    # email = [email protected]
     log.debug("Attempting to parse legacy auth file format")
     try:
         data = []
-        for line in fileinput.input(config_file):
-            data.append(line.strip().split(' = ')[1])
-        if len(data) < 2:
-            # Not enough data
-            raise errors.InvalidConfigFile(
-                'Invalid or empty configuration file!')
+        with open(config_file) as f:
+            for line in f.readlines():
+                data.append(line.strip().split(' = ')[1])
+            if len(data) < 2:
+                # Not enough data
+                raise errors.InvalidConfigFile(
+                    'Invalid or empty configuration file!'
+                )
 
         username, password = decode_auth(data[0])
-        conf[INDEX_NAME] = {
-            'username': username,
-            'password': password,
-            'email': data[1],
-            'serveraddress': INDEX_URL,
+        return {
+            INDEX_NAME: {
+                'username': username,
+                'password': password,
+                'email': data[1],
+                'serveraddress': INDEX_URL,
+            }
         }
-        return conf
     except Exception as e:
         log.debug(e)
         pass

docker/client.py

@@ -39,7 +39,8 @@ class Client(
         api.DaemonApiMixin,
         api.ExecApiMixin,
         api.ImageApiMixin,
-        api.VolumeApiMixin):
+        api.VolumeApiMixin,
+        api.NetworkApiMixin):
     def __init__(self, base_url=None, version=None,
                  timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False):
         super(Client, self).__init__()
@@ -108,6 +109,9 @@ def _post(self, url, **kwargs):
     def _get(self, url, **kwargs):
         return self.get(url, **self._set_request_timeout(kwargs))
 
+    def _put(self, url, **kwargs):
+        return self.put(url, **self._set_request_timeout(kwargs))
+
     def _delete(self, url, **kwargs):
         return self.delete(url, **self._set_request_timeout(kwargs))
 
@@ -184,6 +188,8 @@ def _get_raw_response_socket(self, response):
         self._raise_for_status(response)
         if six.PY3:
             sock = response.raw._fp.fp.raw
+            if self.base_url.startswith("https://"):
+                sock = sock._sock
         else:
             sock = response.raw._fp.fp._sock
         try:
@@ -240,10 +246,7 @@ def _multiplexed_response_stream_helper(self, response):
         # Disable timeout on the underlying socket to prevent
         # Read timed out(s) for long running processes
         socket = self._get_raw_response_socket(response)
-        if six.PY3:
-            socket._sock.settimeout(None)
-        else:
-            socket.settimeout(None)
+        self._disable_socket_timeout(socket)
 
         while True:
             header = response.raw.read(constants.STREAM_HEADER_SIZE_BYTES)
@@ -272,6 +275,19 @@ def _stream_raw_result(self, response):
         for out in response.iter_content(chunk_size=1, decode_unicode=True):
             yield out
 
+    def _disable_socket_timeout(self, socket):
+        """ Depending on the combination of python version and whether we're
+        connecting over http or https, we might need to access _sock, which
+        may or may not exist; or we may need to just settimeout on socket
+         itself, which also may or may not have settimeout on it.
+
+        To avoid missing the correct one, we try both.
+        """
+        if hasattr(socket, "settimeout"):
+            socket.settimeout(None)
+        if hasattr(socket, "_sock") and hasattr(socket._sock, "settimeout"):
+            socket._sock.settimeout(None)
+
     def _get_result(self, container, stream, res):
         cont = self.inspect_container(container)
         return self._get_result_tty(stream, res, cont['Config']['Tty'])