Skip to content

Commit bb1c528

Browse files
Skazza94Skazza94
authored
Add max_pool_size parameter (#2699)
* Add max_pool_size parameter Signed-off-by: Mariano Scazzariello <[email protected]> * Add client version to tests Signed-off-by: Mariano Scazzariello <[email protected]> * Fix parameter position Signed-off-by: Mariano Scazzariello <[email protected]>
1 parent 8002222 commit bb1c528

File tree

7 files changed

+195
-17
lines changed

7 files changed

+195
-17
lines changed

‎docker/api/client.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99

1010
from .. import auth
1111
from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH,
12-
DEFAULT_TIMEOUT_SECONDS, DEFAULT_USER_AGENT,
13-
IS_WINDOWS_PLATFORM, MINIMUM_DOCKER_API_VERSION,
14-
STREAM_HEADER_SIZE_BYTES)
12+
DEFAULT_MAX_POOL_SIZE, DEFAULT_TIMEOUT_SECONDS,
13+
DEFAULT_USER_AGENT, IS_WINDOWS_PLATFORM,
14+
MINIMUM_DOCKER_API_VERSION, STREAM_HEADER_SIZE_BYTES)
1515
from ..errors import (DockerException, InvalidVersion, TLSParameterError,
1616
create_api_error_from_http_exception)
1717
from ..tls import TLSConfig
@@ -92,6 +92,8 @@ class APIClient(
9292
use_ssh_client (bool): If set to `True`, an ssh connection is made
9393
via shelling out to the ssh client. Ensure the ssh client is
9494
installed and configured on the host.
95+
max_pool_size (int): The maximum number of connections
96+
to save in the pool.
9597
"""
9698

9799
__attrs__ = requests.Session.__attrs__ + ['_auth_configs',
@@ -103,7 +105,8 @@ class APIClient(
103105
def __init__(self, base_url=None, version=None,
104106
timeout=DEFAULT_TIMEOUT_SECONDS, tls=False,
105107
user_agent=DEFAULT_USER_AGENT, num_pools=None,
106-
credstore_env=None, use_ssh_client=False):
108+
credstore_env=None, use_ssh_client=False,
109+
max_pool_size=DEFAULT_MAX_POOL_SIZE):
107110
super(APIClient, self).__init__()
108111

109112
if tls and not base_url:
@@ -139,7 +142,8 @@ def __init__(self, base_url=None, version=None,
139142

140143
if base_url.startswith('http+unix://'):
141144
self._custom_adapter = UnixHTTPAdapter(
142-
base_url, timeout, pool_connections=num_pools
145+
base_url, timeout, pool_connections=num_pools,
146+
max_pool_size=max_pool_size
143147
)
144148
self.mount('http+docker://', self._custom_adapter)
145149
self._unmount('http://', 'https://')
@@ -153,7 +157,8 @@ def __init__(self, base_url=None, version=None,
153157
)
154158
try:
155159
self._custom_adapter = NpipeHTTPAdapter(
156-
base_url, timeout, pool_connections=num_pools
160+
base_url, timeout, pool_connections=num_pools,
161+
max_pool_size=max_pool_size
157162
)
158163
except NameError:
159164
raise DockerException(
@@ -165,7 +170,7 @@ def __init__(self, base_url=None, version=None,
165170
try:
166171
self._custom_adapter = SSHHTTPAdapter(
167172
base_url, timeout, pool_connections=num_pools,
168-
shell_out=use_ssh_client
173+
max_pool_size=max_pool_size, shell_out=use_ssh_client
169174
)
170175
except NameError:
171176
raise DockerException(

‎docker/client.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from .api.client import APIClient
2-
from .constants import DEFAULT_TIMEOUT_SECONDS
2+
from .constants import (DEFAULT_TIMEOUT_SECONDS, DEFAULT_MAX_POOL_SIZE)
33
from .models.configs import ConfigCollection
44
from .models.containers import ContainerCollection
55
from .models.images import ImageCollection
@@ -38,6 +38,8 @@ class DockerClient(object):
3838
use_ssh_client (bool): If set to `True`, an ssh connection is made
3939
via shelling out to the ssh client. Ensure the ssh client is
4040
installed and configured on the host.
41+
max_pool_size (int): The maximum number of connections
42+
to save in the pool.
4143
"""
4244
def __init__(self, *args, **kwargs):
4345
self.api = APIClient(*args, **kwargs)
@@ -67,6 +69,8 @@ def from_env(cls, **kwargs):
6769
version (str): The version of the API to use. Set to ``auto`` to
6870
automatically detect the server's version. Default: ``auto``
6971
timeout (int): Default timeout for API calls, in seconds.
72+
max_pool_size (int): The maximum number of connections
73+
to save in the pool.
7074
ssl_version (int): A valid `SSL version`_.
7175
assert_hostname (bool): Verify the hostname of the server.
7276
environment (dict): The environment to read environment variables
@@ -86,10 +90,12 @@ def from_env(cls, **kwargs):
8690
https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
8791
"""
8892
timeout = kwargs.pop('timeout', DEFAULT_TIMEOUT_SECONDS)
93+
max_pool_size = kwargs.pop('max_pool_size', DEFAULT_MAX_POOL_SIZE)
8994
version = kwargs.pop('version', None)
9095
use_ssh_client = kwargs.pop('use_ssh_client', False)
9196
return cls(
9297
timeout=timeout,
98+
max_pool_size=max_pool_size,
9399
version=version,
94100
use_ssh_client=use_ssh_client,
95101
**kwargs_from_env(**kwargs)

‎docker/constants.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
# For more details see: https://github.com/docker/docker-py/issues/2246
3737
DEFAULT_NUM_POOLS_SSH = 9
3838

39+
DEFAULT_MAX_POOL_SIZE = 10
40+
3941
DEFAULT_DATA_CHUNK_SIZE = 1024 * 2048
4042

4143
DEFAULT_SWARM_ADDR_POOL = ['10.0.0.0/8']

‎docker/transport/npipeconn.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,15 @@ class NpipeHTTPAdapter(BaseHTTPAdapter):
7373

7474
__attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['npipe_path',
7575
'pools',
76-
'timeout']
76+
'timeout',
77+
'max_pool_size']
7778

7879
def __init__(self, base_url, timeout=60,
79-
pool_connections=constants.DEFAULT_NUM_POOLS):
80+
pool_connections=constants.DEFAULT_NUM_POOLS,
81+
max_pool_size=constants.DEFAULT_MAX_POOL_SIZE):
8082
self.npipe_path = base_url.replace('npipe://', '')
8183
self.timeout = timeout
84+
self.max_pool_size = max_pool_size
8285
self.pools = RecentlyUsedContainer(
8386
pool_connections, dispose_func=lambda p: p.close()
8487
)
@@ -91,7 +94,8 @@ def get_connection(self, url, proxies=None):
9194
return pool
9295

9396
pool = NpipeHTTPConnectionPool(
94-
self.npipe_path, self.timeout
97+
self.npipe_path, self.timeout,
98+
maxsize=self.max_pool_size
9599
)
96100
self.pools[url] = pool
97101

‎docker/transport/sshconn.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,11 +184,12 @@ def _get_conn(self, timeout):
184184
class SSHHTTPAdapter(BaseHTTPAdapter):
185185

186186
__attrs__ = requests.adapters.HTTPAdapter.__attrs__ + [
187-
'pools', 'timeout', 'ssh_client', 'ssh_params'
187+
'pools', 'timeout', 'ssh_client', 'ssh_params', 'max_pool_size'
188188
]
189189

190190
def __init__(self, base_url, timeout=60,
191191
pool_connections=constants.DEFAULT_NUM_POOLS,
192+
max_pool_size=constants.DEFAULT_MAX_POOL_SIZE,
192193
shell_out=True):
193194
self.ssh_client = None
194195
if not shell_out:
@@ -197,6 +198,7 @@ def __init__(self, base_url, timeout=60,
197198
base_url = base_url.lstrip('ssh://')
198199
self.host = base_url
199200
self.timeout = timeout
201+
self.max_pool_size = max_pool_size
200202
self.pools = RecentlyUsedContainer(
201203
pool_connections, dispose_func=lambda p: p.close()
202204
)
@@ -219,6 +221,7 @@ def get_connection(self, url, proxies=None):
219221
pool = SSHConnectionPool(
220222
ssh_client=self.ssh_client,
221223
timeout=self.timeout,
224+
maxsize=self.max_pool_size,
222225
host=self.host
223226
)
224227
self.pools[url] = pool

‎docker/transport/unixconn.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,18 @@ class UnixHTTPAdapter(BaseHTTPAdapter):
7474

7575
__attrs__ = requests.adapters.HTTPAdapter.__attrs__ + ['pools',
7676
'socket_path',
77-
'timeout']
77+
'timeout',
78+
'max_pool_size']
7879

7980
def __init__(self, socket_url, timeout=60,
80-
pool_connections=constants.DEFAULT_NUM_POOLS):
81+
pool_connections=constants.DEFAULT_NUM_POOLS,
82+
max_pool_size=constants.DEFAULT_MAX_POOL_SIZE):
8183
socket_path = socket_url.replace('http+unix://', '')
8284
if not socket_path.startswith('/'):
8385
socket_path = '/' + socket_path
8486
self.socket_path = socket_path
8587
self.timeout = timeout
88+
self.max_pool_size = max_pool_size
8689
self.pools = RecentlyUsedContainer(
8790
pool_connections, dispose_func=lambda p: p.close()
8891
)
@@ -95,7 +98,8 @@ def get_connection(self, url, proxies=None):
9598
return pool
9699

97100
pool = UnixHTTPConnectionPool(
98-
url, self.socket_path, self.timeout
101+
url, self.socket_path, self.timeout,
102+
maxsize=self.max_pool_size
99103
)
100104
self.pools[url] = pool
101105

‎tests/unit/client_test.py‎

Lines changed: 156 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import docker
66
import pytest
77
from docker.constants import (
8-
DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS)
8+
DEFAULT_DOCKER_API_VERSION, DEFAULT_TIMEOUT_SECONDS,
9+
DEFAULT_MAX_POOL_SIZE, IS_WINDOWS_PLATFORM
10+
)
911
from docker.utils import kwargs_from_env
1012

1113
from . import fake_api
@@ -15,8 +17,8 @@
1517
except ImportError:
1618
import mock
1719

18-
1920
TEST_CERT_DIR = os.path.join(os.path.dirname(__file__), 'testdata/certs')
21+
POOL_SIZE = 20
2022

2123

2224
class ClientTest(unittest.TestCase):
@@ -76,6 +78,84 @@ def test_call_containers(self):
7678
assert "'ContainerCollection' object is not callable" in s
7779
assert "docker.APIClient" in s
7880

81+
@pytest.mark.skipif(
82+
IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux'
83+
)
84+
@mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool")
85+
def test_default_pool_size_unix(self, mock_obj):
86+
client = docker.DockerClient(
87+
version=DEFAULT_DOCKER_API_VERSION
88+
)
89+
mock_obj.return_value.urlopen.return_value.status = 200
90+
client.ping()
91+
92+
base_url = "{base_url}/v{version}/_ping".format(
93+
base_url=client.api.base_url,
94+
version=client.api._version
95+
)
96+
97+
mock_obj.assert_called_once_with(base_url,
98+
"/var/run/docker.sock",
99+
60,
100+
maxsize=DEFAULT_MAX_POOL_SIZE
101+
)
102+
103+
@pytest.mark.skipif(
104+
not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows'
105+
)
106+
@mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool")
107+
def test_default_pool_size_win(self, mock_obj):
108+
client = docker.DockerClient(
109+
version=DEFAULT_DOCKER_API_VERSION
110+
)
111+
mock_obj.return_value.urlopen.return_value.status = 200
112+
client.ping()
113+
114+
mock_obj.assert_called_once_with("//./pipe/docker_engine",
115+
60,
116+
maxsize=DEFAULT_MAX_POOL_SIZE
117+
)
118+
119+
@pytest.mark.skipif(
120+
IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux'
121+
)
122+
@mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool")
123+
def test_pool_size_unix(self, mock_obj):
124+
client = docker.DockerClient(
125+
version=DEFAULT_DOCKER_API_VERSION,
126+
max_pool_size=POOL_SIZE
127+
)
128+
mock_obj.return_value.urlopen.return_value.status = 200
129+
client.ping()
130+
131+
base_url = "{base_url}/v{version}/_ping".format(
132+
base_url=client.api.base_url,
133+
version=client.api._version
134+
)
135+
136+
mock_obj.assert_called_once_with(base_url,
137+
"/var/run/docker.sock",
138+
60,
139+
maxsize=POOL_SIZE
140+
)
141+
142+
@pytest.mark.skipif(
143+
not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows'
144+
)
145+
@mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool")
146+
def test_pool_size_win(self, mock_obj):
147+
client = docker.DockerClient(
148+
version=DEFAULT_DOCKER_API_VERSION,
149+
max_pool_size=POOL_SIZE
150+
)
151+
mock_obj.return_value.urlopen.return_value.status = 200
152+
client.ping()
153+
154+
mock_obj.assert_called_once_with("//./pipe/docker_engine",
155+
60,
156+
maxsize=POOL_SIZE
157+
)
158+
79159

80160
class FromEnvTest(unittest.TestCase):
81161

@@ -112,3 +192,77 @@ def test_from_env_without_timeout_uses_default(self):
112192
client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
113193

114194
assert client.api.timeout == DEFAULT_TIMEOUT_SECONDS
195+
196+
@pytest.mark.skipif(
197+
IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux'
198+
)
199+
@mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool")
200+
def test_default_pool_size_from_env_unix(self, mock_obj):
201+
client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
202+
mock_obj.return_value.urlopen.return_value.status = 200
203+
client.ping()
204+
205+
base_url = "{base_url}/v{version}/_ping".format(
206+
base_url=client.api.base_url,
207+
version=client.api._version
208+
)
209+
210+
mock_obj.assert_called_once_with(base_url,
211+
"/var/run/docker.sock",
212+
60,
213+
maxsize=DEFAULT_MAX_POOL_SIZE
214+
)
215+
216+
@pytest.mark.skipif(
217+
not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows'
218+
)
219+
@mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool")
220+
def test_default_pool_size_from_env_win(self, mock_obj):
221+
client = docker.from_env(version=DEFAULT_DOCKER_API_VERSION)
222+
mock_obj.return_value.urlopen.return_value.status = 200
223+
client.ping()
224+
225+
mock_obj.assert_called_once_with("//./pipe/docker_engine",
226+
60,
227+
maxsize=DEFAULT_MAX_POOL_SIZE
228+
)
229+
230+
@pytest.mark.skipif(
231+
IS_WINDOWS_PLATFORM, reason='Unix Connection Pool only on Linux'
232+
)
233+
@mock.patch("docker.transport.unixconn.UnixHTTPConnectionPool")
234+
def test_pool_size_from_env_unix(self, mock_obj):
235+
client = docker.from_env(
236+
version=DEFAULT_DOCKER_API_VERSION,
237+
max_pool_size=POOL_SIZE
238+
)
239+
mock_obj.return_value.urlopen.return_value.status = 200
240+
client.ping()
241+
242+
base_url = "{base_url}/v{version}/_ping".format(
243+
base_url=client.api.base_url,
244+
version=client.api._version
245+
)
246+
247+
mock_obj.assert_called_once_with(base_url,
248+
"/var/run/docker.sock",
249+
60,
250+
maxsize=POOL_SIZE
251+
)
252+
253+
@pytest.mark.skipif(
254+
not IS_WINDOWS_PLATFORM, reason='Npipe Connection Pool only on Windows'
255+
)
256+
@mock.patch("docker.transport.npipeconn.NpipeHTTPConnectionPool")
257+
def test_pool_size_from_env_win(self, mock_obj):
258+
client = docker.from_env(
259+
version=DEFAULT_DOCKER_API_VERSION,
260+
max_pool_size=POOL_SIZE
261+
)
262+
mock_obj.return_value.urlopen.return_value.status = 200
263+
client.ping()
264+
265+
mock_obj.assert_called_once_with("//./pipe/docker_engine",
266+
60,
267+
maxsize=POOL_SIZE
268+
)

0 commit comments

Comments
 (0)