Skip to content

Commit 8731ecf

Browse files
authored
Merge pull request #1014 from minrk/flush-buffers
2 parents 71eb805 + b36a6a7 commit 8731ecf

File tree

4 files changed

+122
-43
lines changed

4 files changed

+122
-43
lines changed

repo2docker/buildpacks/base.py

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@
99
import sys
1010
import hashlib
1111
import escapism
12-
import xml.etree.ElementTree as ET
13-
14-
from traitlets import Dict
1512

1613
# Only use syntax features supported by Docker 17.09
1714
TEMPLATE = r"""
@@ -181,6 +178,8 @@
181178
{% endif -%}
182179
183180
# Add entrypoint
181+
ENV PYTHONUNBUFFERED=1
182+
COPY /python3-login /usr/local/bin/python3-login
184183
COPY /repo2docker-entrypoint /usr/local/bin/repo2docker-entrypoint
185184
ENTRYPOINT ["/usr/local/bin/repo2docker-entrypoint"]
186185
@@ -193,9 +192,7 @@
193192
{% endif %}
194193
"""
195194

196-
ENTRYPOINT_FILE = os.path.join(
197-
os.path.dirname(os.path.abspath(__file__)), "repo2docker-entrypoint"
198-
)
195+
HERE = os.path.dirname(os.path.abspath(__file__))
199196

200197
# Also used for the group
201198
DEFAULT_NB_UID = 1000
@@ -582,7 +579,8 @@ def _filter_tar(tar):
582579
dest_path, src_path = self.generate_build_context_filename(src)
583580
tar.add(src_path, dest_path, filter=_filter_tar)
584581

585-
tar.add(ENTRYPOINT_FILE, "repo2docker-entrypoint", filter=_filter_tar)
582+
for fname in ("repo2docker-entrypoint", "python3-login"):
583+
tar.add(os.path.join(HERE, fname), fname, filter=_filter_tar)
586584

587585
tar.add(".", "src/", filter=_filter_tar)
588586

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash -l
2+
# This is an executable that launches Python in a login shell
3+
# to ensure that full profile setup occurs.
4+
# shebang on linux only allows 1 argument,
5+
# so we couldn't pick a login shell in one shebang line
6+
# for a Python script
7+
8+
# -u means unbuffered, which one ~always wants in a container
9+
# otherwise output can be mysteriously missing
10+
11+
exec python3 -u "$@"
Lines changed: 97 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,97 @@
1-
#!/bin/bash -l
2-
# lightest possible entrypoint that ensures that
3-
# we use a login shell to get a fully configured shell environment
4-
# (e.g. sourcing /etc/profile.d, ~/.bashrc, and friends)
5-
6-
# Setup a file descriptor (FD) that is connected to a tee process which
7-
# writes its input to $REPO_DIR/.jupyter-server-log.txt
8-
# We later use this FD as a place to redirect the output of the actual
9-
# command to. We can't add `tee` to the command directly as that will prevent
10-
# the container from exiting when `docker stop` is run.
11-
# See https://stackoverflow.com/a/55678435
12-
exec {log_fd}> >(exec tee $REPO_DIR/.jupyter-server-log.txt)
13-
14-
if [[ ! -z "${R2D_ENTRYPOINT:-}" ]]; then
15-
if [[ ! -x "$R2D_ENTRYPOINT" ]]; then
16-
chmod u+x "$R2D_ENTRYPOINT"
17-
fi
18-
exec "$R2D_ENTRYPOINT" "$@" 2>&1 >&"$log_fd"
19-
else
20-
exec "$@" 2>&1 >&"$log_fd"
21-
fi
22-
23-
# Close the logging output again
24-
exec {log_fd}>&-
1+
#!/usr/local/bin/python3-login
2+
# note: must run on Python >= 3.5, which mainly means no f-strings
3+
4+
# goals:
5+
# - load environment variables from a login shell (bash -l)
6+
# - preserve signal handling of subprocess (kill -TERM and friends)
7+
# - tee output to a log file
8+
9+
import fcntl
10+
import os
11+
import select
12+
import signal
13+
import subprocess
14+
import sys
15+
16+
# output chunk size to read
17+
CHUNK_SIZE = 1024
18+
19+
# signals to be forwarded to the child
20+
# everything catchable, excluding SIGCHLD
21+
SIGNALS = set(signal.Signals) - {signal.SIGKILL, signal.SIGSTOP, signal.SIGCHLD}
22+
23+
24+
def main():
25+
26+
# open log file to send output
27+
log_file = open(
28+
os.path.join(os.environ.get("REPO_DIR", "."), ".jupyter-server-log.txt"),
29+
"ab",
30+
)
31+
32+
# build the command
33+
# like `exec "$@"`
34+
command = sys.argv[1:]
35+
# load entrypoint override from env
36+
r2d_entrypoint = os.environ.get("R2D_ENTRYPOINT")
37+
if r2d_entrypoint:
38+
command.insert(0, r2d_entrypoint)
39+
40+
# launch the subprocess
41+
child = subprocess.Popen(
42+
command,
43+
bufsize=1,
44+
stdout=subprocess.PIPE,
45+
stderr=subprocess.STDOUT,
46+
)
47+
48+
# hook up ~all signals so that every signal the parent gets,
49+
# the children also get
50+
51+
def relay_signal(sig, frame):
52+
"""Relay a signal to children"""
53+
# DEBUG: show signal
54+
child.send_signal(sig)
55+
56+
for signum in SIGNALS:
57+
signal.signal(signum, relay_signal)
58+
59+
# tee output from child to both our stdout and the log file
60+
def tee(chunk):
61+
"""Tee output from child to both our stdout and the log file"""
62+
for f in [sys.stdout.buffer, log_file]:
63+
f.write(chunk)
64+
f.flush()
65+
66+
# make stdout pipe non-blocking
67+
# this means child.stdout.read(nbytes)
68+
# will always return immediately, even if there's nothing to read
69+
flags = fcntl.fcntl(child.stdout, fcntl.F_GETFL)
70+
fcntl.fcntl(child.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
71+
poller = select.poll()
72+
poller.register(child.stdout)
73+
74+
# while child is running, constantly relay output
75+
while child.poll() is None:
76+
chunk = child.stdout.read(CHUNK_SIZE)
77+
if chunk:
78+
tee(chunk)
79+
else:
80+
# empty chunk means nothing to read
81+
# wait for output on the pipe
82+
# timeout is in milliseconds
83+
poller.poll(1000)
84+
85+
# child has exited, continue relaying any remaining output
86+
# At this point, read() will return an empty string when it's done
87+
chunk = child.stdout.read()
88+
while chunk:
89+
tee(chunk)
90+
chunk = child.stdout.read()
91+
92+
# make our returncode match the child's returncode
93+
sys.exit(child.returncode)
94+
95+
96+
if __name__ == "__main__":
97+
main()

tests/unit/test_env.py

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@
33
"""
44
import os
55
import subprocess
6+
import sys
67
import tempfile
78
import time
89
from getpass import getuser
910

1011

11-
def test_env():
12+
def test_env(capfd):
1213
"""
1314
Validate that you can define environment variables
1415
@@ -42,32 +43,28 @@ def test_env():
4243
# value
4344
"--env",
4445
"SPAM_2=",
45-
# "--",
4646
tmpdir,
47+
"--",
4748
"/bin/bash",
4849
"-c",
4950
# Docker exports all passed env variables, so we can
5051
# just look at exported variables.
51-
"export; sleep 1",
52-
# "export; echo TIMDONE",
53-
# "export",
52+
"export",
5453
],
55-
universal_newlines=True,
56-
stdout=subprocess.PIPE,
57-
stderr=subprocess.PIPE,
5854
)
55+
captured = capfd.readouterr()
56+
print(captured.out, end="")
57+
print(captured.err, file=sys.stderr, end="")
58+
5959
assert result.returncode == 0
6060

6161
# all docker output is returned by repo2docker on stderr
6262
# extract just the declare for better failure message formatting
6363
# stdout should be empty
6464
assert not result.stdout
6565

66-
print(result.stderr.split("\n"))
67-
# assert False
68-
6966
# stderr should contain lines of output
70-
declares = [x for x in result.stderr.split("\n") if x.startswith("declare")]
67+
declares = [x for x in captured.err.splitlines() if x.startswith("declare")]
7168
assert 'declare -x FOO="{}"'.format(ts) in declares
7269
assert 'declare -x BAR="baz"' in declares
7370
assert 'declare -x SPAM="eggs"' in declares

0 commit comments

Comments
 (0)