Skip to content

Commit b72fb1e

Browse files
committed
Merge branch 'little-dude-master'
2 parents 6663881 + 16c2809 commit b72fb1e

File tree

13 files changed

+371
-69
lines changed

13 files changed

+371
-69
lines changed

docker/api/client.py

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from ..tls import TLSConfig
3333
from ..transport import SSLAdapter, UnixAdapter
3434
from ..utils import utils, check_resource, update_headers, config
35-
from ..utils.socket import frames_iter, socket_raw_iter
35+
from ..utils.socket import frames_iter, consume_socket_output, demux_adaptor
3636
from ..utils.json_stream import json_stream
3737
try:
3838
from ..transport import NpipeAdapter
@@ -381,19 +381,23 @@ def _stream_raw_result(self, response, chunk_size=1, decode=True):
381381
for out in response.iter_content(chunk_size, decode):
382382
yield out
383383

384-
def _read_from_socket(self, response, stream, tty=False):
384+
def _read_from_socket(self, response, stream, tty=True, demux=False):
385385
socket = self._get_raw_response_socket(response)
386386

387-
gen = None
388-
if tty is False:
389-
gen = frames_iter(socket)
387+
gen = frames_iter(socket, tty)
388+
389+
if demux:
390+
# The generator will output tuples (stdout, stderr)
391+
gen = (demux_adaptor(*frame) for frame in gen)
390392
else:
391-
gen = socket_raw_iter(socket)
393+
# The generator will output strings
394+
gen = (data for (_, data) in gen)
392395

393396
if stream:
394397
return gen
395398
else:
396-
return six.binary_type().join(gen)
399+
# Wait for all the frames, concatenate them, and return the result
400+
return consume_socket_output(gen, demux=demux)
397401

398402
def _disable_socket_timeout(self, socket):
399403
""" Depending on the combination of python version and whether we're

docker/api/container.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
class ContainerApiMixin(object):
1414
@utils.check_resource('container')
1515
def attach(self, container, stdout=True, stderr=True,
16-
stream=False, logs=False):
16+
stream=False, logs=False, demux=False):
1717
"""
1818
Attach to a container.
1919
@@ -28,11 +28,15 @@ def attach(self, container, stdout=True, stderr=True,
2828
stream (bool): Return container output progressively as an iterator
2929
of strings, rather than a single string.
3030
logs (bool): Include the container's previous output.
31+
demux (bool): Keep stdout and stderr separate.
3132
3233
Returns:
33-
By default, the container's output as a single string.
34+
By default, the container's output as a single string (two if
35+
``demux=True``: one for stdout and one for stderr).
3436
35-
If ``stream=True``, an iterator of output strings.
37+
If ``stream=True``, an iterator of output strings. If
38+
``demux=True``, two iterators are returned: one for stdout and one
39+
for stderr.
3640
3741
Raises:
3842
:py:class:`docker.errors.APIError`
@@ -54,8 +58,7 @@ def attach(self, container, stdout=True, stderr=True,
5458
response = self._post(u, headers=headers, params=params, stream=True)
5559

5660
output = self._read_from_socket(
57-
response, stream, self._check_is_tty(container)
58-
)
61+
response, stream, self._check_is_tty(container), demux=demux)
5962

6063
if stream:
6164
return CancellableStream(output, response)

docker/api/exec_api.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ def exec_resize(self, exec_id, height=None, width=None):
118118

119119
@utils.check_resource('exec_id')
120120
def exec_start(self, exec_id, detach=False, tty=False, stream=False,
121-
socket=False):
121+
socket=False, demux=False):
122122
"""
123123
Start a previously set up exec instance.
124124
@@ -130,11 +130,14 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False,
130130
stream (bool): Stream response data. Default: False
131131
socket (bool): Return the connection socket to allow custom
132132
read/write operations.
133+
demux (bool): Return stdout and stderr separately
133134
134135
Returns:
135-
(generator or str): If ``stream=True``, a generator yielding
136-
response chunks. If ``socket=True``, a socket object for the
137-
connection. A string containing response data otherwise.
136+
137+
(generator or str or tuple): If ``stream=True``, a generator
138+
yielding response chunks. If ``socket=True``, a socket object for
139+
the connection. A string containing response data otherwise. If
140+
``demux=True``, stdout and stderr are separated.
138141
139142
Raises:
140143
:py:class:`docker.errors.APIError`
@@ -162,4 +165,4 @@ def exec_start(self, exec_id, detach=False, tty=False, stream=False,
162165
return self._result(res)
163166
if socket:
164167
return self._get_raw_response_socket(res)
165-
return self._read_from_socket(res, stream, tty)
168+
return self._read_from_socket(res, stream, tty=tty, demux=demux)

docker/models/containers.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ def diff(self):
144144

145145
def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False,
146146
privileged=False, user='', detach=False, stream=False,
147-
socket=False, environment=None, workdir=None):
147+
socket=False, environment=None, workdir=None, demux=False):
148148
"""
149149
Run a command inside this container. Similar to
150150
``docker exec``.
@@ -166,6 +166,7 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False,
166166
the following format ``["PASSWORD=xxx"]`` or
167167
``{"PASSWORD": "xxx"}``.
168168
workdir (str): Path to working directory for this exec session
169+
demux (bool): Return stdout and stderr separately
169170
170171
Returns:
171172
(ExecResult): A tuple of (exit_code, output)
@@ -187,7 +188,8 @@ def exec_run(self, cmd, stdout=True, stderr=True, stdin=False, tty=False,
187188
workdir=workdir
188189
)
189190
exec_output = self.client.api.exec_start(
190-
resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket
191+
resp['Id'], detach=detach, tty=tty, stream=stream, socket=socket,
192+
demux=demux
191193
)
192194
if socket or stream:
193195
return ExecResult(None, exec_output)

docker/utils/socket.py

Lines changed: 83 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212
NpipeSocket = type(None)
1313

1414

15+
STDOUT = 1
16+
STDERR = 2
17+
18+
1519
class SocketError(Exception):
1620
pass
1721

@@ -51,28 +55,43 @@ def read_exactly(socket, n):
5155
return data
5256

5357

54-
def next_frame_size(socket):
58+
def next_frame_header(socket):
5559
"""
56-
Returns the size of the next frame of data waiting to be read from socket,
57-
according to the protocol defined here:
60+
Returns the stream and size of the next frame of data waiting to be read
61+
from socket, according to the protocol defined here:
5862
59-
https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/attach-to-a-container
63+
https://docs.docker.com/engine/api/v1.24/#attach-to-a-container
6064
"""
6165
try:
6266
data = read_exactly(socket, 8)
6367
except SocketError:
64-
return -1
68+
return (-1, -1)
69+
70+
stream, actual = struct.unpack('>BxxxL', data)
71+
return (stream, actual)
72+
6573

66-
_, actual = struct.unpack('>BxxxL', data)
67-
return actual
74+
def frames_iter(socket, tty):
75+
"""
76+
Return a generator of frames read from socket. A frame is a tuple where
77+
the first item is the stream number and the second item is a chunk of data.
78+
79+
If the tty setting is enabled, the streams are multiplexed into the stdout
80+
stream.
81+
"""
82+
if tty:
83+
return ((STDOUT, frame) for frame in frames_iter_tty(socket))
84+
else:
85+
return frames_iter_no_tty(socket)
6886

6987

70-
def frames_iter(socket):
88+
def frames_iter_no_tty(socket):
7189
"""
72-
Returns a generator of frames read from socket
90+
Returns a generator of data read from the socket when the tty setting is
91+
not enabled.
7392
"""
7493
while True:
75-
n = next_frame_size(socket)
94+
(stream, n) = next_frame_header(socket)
7695
if n < 0:
7796
break
7897
while n > 0:
@@ -84,17 +103,67 @@ def frames_iter(socket):
84103
# We have reached EOF
85104
return
86105
n -= data_length
87-
yield result
106+
yield (stream, result)
88107

89108

90-
def socket_raw_iter(socket):
109+
def frames_iter_tty(socket):
91110
"""
92-
Returns a generator of data read from the socket.
93-
This is used for non-multiplexed streams.
111+
Return a generator of data read from the socket when the tty setting is
112+
enabled.
94113
"""
95114
while True:
96115
result = read(socket)
97116
if len(result) == 0:
98117
# We have reached EOF
99118
return
100119
yield result
120+
121+
122+
def consume_socket_output(frames, demux=False):
123+
"""
124+
Iterate through frames read from the socket and return the result.
125+
126+
Args:
127+
128+
demux (bool):
129+
If False, stdout and stderr are multiplexed, and the result is the
130+
concatenation of all the frames. If True, the streams are
131+
demultiplexed, and the result is a 2-tuple where each item is the
132+
concatenation of frames belonging to the same stream.
133+
"""
134+
if demux is False:
135+
# If the streams are multiplexed, the generator returns strings, that
136+
# we just need to concatenate.
137+
return six.binary_type().join(frames)
138+
139+
# If the streams are demultiplexed, the generator yields tuples
140+
# (stdout, stderr)
141+
out = [None, None]
142+
for frame in frames:
143+
# It is guaranteed that for each frame, one and only one stream
144+
# is not None.
145+
assert frame != (None, None)
146+
if frame[0] is not None:
147+
if out[0] is None:
148+
out[0] = frame[0]
149+
else:
150+
out[0] += frame[0]
151+
else:
152+
if out[1] is None:
153+
out[1] = frame[1]
154+
else:
155+
out[1] += frame[1]
156+
return tuple(out)
157+
158+
159+
def demux_adaptor(stream_id, data):
160+
"""
161+
Utility to demultiplex stdout and stderr when reading frames from the
162+
socket.
163+
"""
164+
if stream_id == STDOUT:
165+
return (data, None)
166+
elif stream_id == STDERR:
167+
return (None, data)
168+
else:
169+
raise ValueError('{0} is not a valid stream'.format(stream_id))

docs/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,4 +92,5 @@ That's just a taste of what you can do with the Docker SDK for Python. For more,
9292
volumes
9393
api
9494
tls
95+
user_guides/index
9596
change-log

docs/user_guides/index.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
User guides and tutorials
2+
=========================
3+
4+
.. toctree::
5+
:maxdepth: 2
6+
7+
multiplex
8+
swarm_services

docs/user_guides/multiplex.rst

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
Handling multiplexed streams
2+
============================
3+
4+
.. note::
5+
The following instruction assume you're interested in getting output from
6+
an ``exec`` command. These instruction are similarly applicable to the
7+
output of ``attach``.
8+
9+
First create a container that runs in the background:
10+
11+
>>> client = docker.from_env()
12+
>>> container = client.containers.run(
13+
... 'bfirsh/reticulate-splines', detach=True)
14+
15+
Prepare the command we are going to use. It prints "hello stdout"
16+
in `stdout`, followed by "hello stderr" in `stderr`:
17+
18+
>>> cmd = '/bin/sh -c "echo hello stdout ; echo hello stderr >&2"'
19+
We'll run this command with all four the combinations of ``stream``
20+
and ``demux``.
21+
With ``stream=False`` and ``demux=False``, the output is a string
22+
that contains both the `stdout` and the `stderr` output:
23+
>>> res = container.exec_run(cmd, stream=False, demux=False)
24+
>>> res.output
25+
b'hello stderr\nhello stdout\n'
26+
27+
With ``stream=True``, and ``demux=False``, the output is a
28+
generator that yields strings containing the output of both
29+
`stdout` and `stderr`:
30+
31+
>>> res = container.exec_run(cmd, stream=True, demux=False)
32+
>>> next(res.output)
33+
b'hello stdout\n'
34+
>>> next(res.output)
35+
b'hello stderr\n'
36+
>>> next(res.output)
37+
Traceback (most recent call last):
38+
File "<stdin>", line 1, in <module>
39+
StopIteration
40+
41+
With ``stream=True`` and ``demux=True``, the generator now
42+
separates the streams, and yield tuples
43+
``(stdout, stderr)``:
44+
45+
>>> res = container.exec_run(cmd, stream=True, demux=True)
46+
>>> next(res.output)
47+
(b'hello stdout\n', None)
48+
>>> next(res.output)
49+
(None, b'hello stderr\n')
50+
>>> next(res.output)
51+
Traceback (most recent call last):
52+
File "<stdin>", line 1, in <module>
53+
StopIteration
54+
55+
Finally, with ``stream=False`` and ``demux=True``, the whole output
56+
is returned, but the streams are still separated:
57+
58+
>>> res = container.exec_run(cmd, stream=True, demux=True)
59+
>>> next(res.output)
60+
(b'hello stdout\n', None)
61+
>>> next(res.output)
62+
(None, b'hello stderr\n')
63+
>>> next(res.output)
64+
Traceback (most recent call last):
65+
File "<stdin>", line 1, in <module>
66+
StopIteration

docs/user_guides/swarm_services.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Swarm services
22

3+
> Warning:
4+
> This is a stale document and may contain outdated information.
5+
> Refer to the API docs for updated classes and method signatures.
6+
37
Starting with Engine version 1.12 (API 1.24), it is possible to manage services
48
using the Docker Engine API. Note that the engine needs to be part of a
59
[Swarm cluster](../swarm.rst) before you can use the service-related methods.

tests/integration/api_container_test.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
import docker
99
from docker.constants import IS_WINDOWS_PLATFORM
10-
from docker.utils.socket import next_frame_size
10+
from docker.utils.socket import next_frame_header
1111
from docker.utils.socket import read_exactly
1212

1313
import pytest
@@ -1242,7 +1242,8 @@ def test_run_container_reading_socket(self):
12421242

12431243
self.client.start(container)
12441244

1245-
next_size = next_frame_size(pty_stdout)
1245+
(stream, next_size) = next_frame_header(pty_stdout)
1246+
assert stream == 1 # correspond to stdout
12461247
assert next_size == len(line)
12471248
data = read_exactly(pty_stdout, next_size)
12481249
assert data.decode('utf-8') == line

0 commit comments

Comments
 (0)