Skip to content

Commit e701fae

Browse files
committed
Merge remote-tracking branch 'origin/dmw'
* origin/dmw: issue #615: ensure 4GB max_message_size is configured for task workers. issue #615: update Changelog. issue #615: route a dead message to recipients when no reply is expected issue #615: fetch_file() might be called with AnsibleUnicode. issue #615: redirect 'fetch' action to 'mitogen_fetch'. issue #615: extricate slurp brainwrong from mitogen_fetch issue #615: ansible: import Ansible fetch.py action plug-in issue #533: include object identity of Stream in repr() docs: lots more changelog issue #595: add buildah to docs and changelog. docs: a few more internals.rst additions
2 parents 93c97a9 + 6775937 commit e701fae

File tree

11 files changed

+301
-28
lines changed

11 files changed

+301
-28
lines changed

ansible_mitogen/connection.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -953,7 +953,8 @@ def fetch_file(self, in_path, out_path):
953953
self._connect()
954954
ansible_mitogen.target.transfer_file(
955955
context=self.context,
956-
in_path=in_path,
956+
# in_path may be AnsibleUnicode
957+
in_path=mitogen.utils.cast(in_path),
957958
out_path=out_path
958959
)
959960

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
# (c) 2012-2014, Michael DeHaan <[email protected]>
2+
#
3+
# This file is part of Ansible
4+
#
5+
# Ansible is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# Ansible is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
17+
from __future__ import (absolute_import, division, print_function)
18+
__metaclass__ = type
19+
20+
import os
21+
22+
from ansible.module_utils._text import to_bytes
23+
from ansible.module_utils.six import string_types
24+
from ansible.module_utils.parsing.convert_bool import boolean
25+
from ansible.plugins.action import ActionBase
26+
from ansible.utils.hashing import checksum, md5, secure_hash
27+
from ansible.utils.path import makedirs_safe
28+
29+
30+
REMOTE_CHECKSUM_ERRORS = {
31+
'0': "unable to calculate the checksum of the remote file",
32+
'1': "the remote file does not exist",
33+
'2': "no read permission on remote file",
34+
'3': "remote file is a directory, fetch cannot work on directories",
35+
'4': "python isn't present on the system. Unable to compute checksum",
36+
'5': "stdlib json was not found on the remote machine. Only the raw module can work without those installed",
37+
}
38+
39+
40+
class ActionModule(ActionBase):
41+
42+
def run(self, tmp=None, task_vars=None):
43+
''' handler for fetch operations '''
44+
if task_vars is None:
45+
task_vars = dict()
46+
47+
result = super(ActionModule, self).run(tmp, task_vars)
48+
del tmp # tmp no longer has any effect
49+
50+
try:
51+
if self._play_context.check_mode:
52+
result['skipped'] = True
53+
result['msg'] = 'check mode not (yet) supported for this module'
54+
return result
55+
56+
source = self._task.args.get('src', None)
57+
dest = self._task.args.get('dest', None)
58+
flat = boolean(self._task.args.get('flat'), strict=False)
59+
fail_on_missing = boolean(self._task.args.get('fail_on_missing', True), strict=False)
60+
validate_checksum = boolean(self._task.args.get('validate_checksum', True), strict=False)
61+
62+
# validate source and dest are strings FIXME: use basic.py and module specs
63+
if not isinstance(source, string_types):
64+
result['msg'] = "Invalid type supplied for source option, it must be a string"
65+
66+
if not isinstance(dest, string_types):
67+
result['msg'] = "Invalid type supplied for dest option, it must be a string"
68+
69+
if source is None or dest is None:
70+
result['msg'] = "src and dest are required"
71+
72+
if result.get('msg'):
73+
result['failed'] = True
74+
return result
75+
76+
source = self._connection._shell.join_path(source)
77+
source = self._remote_expand_user(source)
78+
79+
# calculate checksum for the remote file, don't bother if using
80+
# become as slurp will be used Force remote_checksum to follow
81+
# symlinks because fetch always follows symlinks
82+
remote_checksum = self._remote_checksum(source, all_vars=task_vars, follow=True)
83+
84+
# calculate the destination name
85+
if os.path.sep not in self._connection._shell.join_path('a', ''):
86+
source = self._connection._shell._unquote(source)
87+
source_local = source.replace('\\', '/')
88+
else:
89+
source_local = source
90+
91+
dest = os.path.expanduser(dest)
92+
if flat:
93+
if os.path.isdir(to_bytes(dest, errors='surrogate_or_strict')) and not dest.endswith(os.sep):
94+
result['msg'] = "dest is an existing directory, use a trailing slash if you want to fetch src into that directory"
95+
result['file'] = dest
96+
result['failed'] = True
97+
return result
98+
if dest.endswith(os.sep):
99+
# if the path ends with "/", we'll use the source filename as the
100+
# destination filename
101+
base = os.path.basename(source_local)
102+
dest = os.path.join(dest, base)
103+
if not dest.startswith("/"):
104+
# if dest does not start with "/", we'll assume a relative path
105+
dest = self._loader.path_dwim(dest)
106+
else:
107+
# files are saved in dest dir, with a subdir for each host, then the filename
108+
if 'inventory_hostname' in task_vars:
109+
target_name = task_vars['inventory_hostname']
110+
else:
111+
target_name = self._play_context.remote_addr
112+
dest = "%s/%s/%s" % (self._loader.path_dwim(dest), target_name, source_local)
113+
114+
dest = dest.replace("//", "/")
115+
116+
if remote_checksum in REMOTE_CHECKSUM_ERRORS:
117+
result['changed'] = False
118+
result['file'] = source
119+
result['msg'] = REMOTE_CHECKSUM_ERRORS[remote_checksum]
120+
# Historically, these don't fail because you may want to transfer
121+
# a log file that possibly MAY exist but keep going to fetch other
122+
# log files. Today, this is better achieved by adding
123+
# ignore_errors or failed_when to the task. Control the behaviour
124+
# via fail_when_missing
125+
if fail_on_missing:
126+
result['failed'] = True
127+
del result['changed']
128+
else:
129+
result['msg'] += ", not transferring, ignored"
130+
return result
131+
132+
# calculate checksum for the local file
133+
local_checksum = checksum(dest)
134+
135+
if remote_checksum != local_checksum:
136+
# create the containing directories, if needed
137+
makedirs_safe(os.path.dirname(dest))
138+
139+
# fetch the file and check for changes
140+
self._connection.fetch_file(source, dest)
141+
new_checksum = secure_hash(dest)
142+
# For backwards compatibility. We'll return None on FIPS enabled systems
143+
try:
144+
new_md5 = md5(dest)
145+
except ValueError:
146+
new_md5 = None
147+
148+
if validate_checksum and new_checksum != remote_checksum:
149+
result.update(dict(failed=True, md5sum=new_md5,
150+
msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None,
151+
checksum=new_checksum, remote_checksum=remote_checksum))
152+
else:
153+
result.update({'changed': True, 'md5sum': new_md5, 'dest': dest,
154+
'remote_md5sum': None, 'checksum': new_checksum,
155+
'remote_checksum': remote_checksum})
156+
else:
157+
# For backwards compatibility. We'll return None on FIPS enabled systems
158+
try:
159+
local_md5 = md5(dest)
160+
except ValueError:
161+
local_md5 = None
162+
result.update(dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum))
163+
164+
finally:
165+
self._remove_tmp_path(self._connection._shell.tmpdir)
166+
167+
return result

ansible_mitogen/process.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
u"__author__ = %r\n"
7373
)
7474

75+
MAX_MESSAGE_SIZE = 4096 * 1048576
76+
7577
worker_model_msg = (
7678
'Mitogen connection types may only be instantiated when one of the '
7779
'"mitogen_*" or "operon_*" strategies are active.'
@@ -502,6 +504,7 @@ def _reconnect(self, path):
502504
# with_items loops.
503505
raise ansible.errors.AnsibleError(shutting_down_msg % (e,))
504506

507+
self.router.max_message_size = MAX_MESSAGE_SIZE
505508
self.listener_path = path
506509

507510
def _on_process_exit(self):
@@ -692,7 +695,7 @@ def _setup_master(self):
692695
self.broker = mitogen.master.Broker(install_watcher=False)
693696
self.router = mitogen.master.Router(
694697
broker=self.broker,
695-
max_message_size=4096 * 1048576,
698+
max_message_size=MAX_MESSAGE_SIZE,
696699
)
697700
_setup_responder(self.router.responder)
698701
mitogen.core.listen(self.broker, 'shutdown', self._on_broker_shutdown)

ansible_mitogen/strategy.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ def wrap_action_loader__get(name, *args, **kwargs):
127127
action plugins outside the Ansible tree.
128128
"""
129129
get_kwargs = {'class_only': True}
130+
if name in ('fetch',):
131+
name = 'mitogen_' + name
130132
if ansible.__version__ >= '2.8':
131133
get_kwargs['collection_list'] = kwargs.pop('collection_list', None)
132134

docs/ansible_detailed.rst

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,8 @@ Noteworthy Differences
175175

176176
your_ssh_username = (ALL) NOPASSWD:/usr/bin/python -c*
177177

178-
* The `docker <https://docs.ansible.com/ansible/2.6/plugins/connection/docker.html>`_,
178+
* The `buildah <https://docs.ansible.com/ansible/latest/plugins/connection/buildah.html>`_,
179+
`docker <https://docs.ansible.com/ansible/2.6/plugins/connection/docker.html>`_,
179180
`jail <https://docs.ansible.com/ansible/2.6/plugins/connection/jail.html>`_,
180181
`kubectl <https://docs.ansible.com/ansible/2.6/plugins/connection/kubectl.html>`_,
181182
`local <https://docs.ansible.com/ansible/2.6/plugins/connection/local.html>`_,
@@ -722,6 +723,19 @@ establishment of additional reuseable interpreters as necessary to match the
722723
configuration of each task.
723724

724725

726+
.. _method-buildah:
727+
728+
Buildah
729+
~~~~~~~
730+
731+
Like `buildah
732+
<https://docs.ansible.com/ansible/2.6/plugins/connection/buildah.html>`_ except
733+
connection delegation is supported.
734+
735+
* ``ansible_host``: Name of Buildah container (default: inventory hostname).
736+
* ``ansible_user``: Name of user within the container to execute as.
737+
738+
725739
.. _doas:
726740

727741
Doas

docs/api.rst

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,20 @@ Router Class
8787
Connection Methods
8888
==================
8989

90+
.. currentmodule:: mitogen.parent
91+
.. method:: Router.buildah (container=None, buildah_path=None, username=None, \**kwargs)
92+
93+
Construct a context on the local machine over a ``buildah`` invocation.
94+
Accepts all parameters accepted by :meth:`local`, in addition to:
95+
96+
:param str container:
97+
The name of the Buildah container to connect to.
98+
:param str doas_path:
99+
Filename or complete path to the ``buildah`` binary. ``PATH`` will be
100+
searched if given as a filename. Defaults to ``buildah``.
101+
:param str username:
102+
Username to use, defaults to unset.
103+
90104
.. currentmodule:: mitogen.parent
91105
.. method:: Router.fork (on_fork=None, on_start=None, debug=False, profiling=False, via=None)
92106

docs/changelog.rst

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,25 +32,32 @@ Enhancements
3232
<https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html>`_
3333
are not yet handled.
3434

35-
* The ``MITOGEN_CPU_COUNT`` environment variable shards the connection
36-
multiplexer into per-CPU workers. This improves throughput for large runs
37-
especially involving file transfer, and is a prerequisite for future
38-
in-process SSH support. One multiplexer starts by default, to match existing
39-
behaviour.
35+
* `Operon <https://networkgenomics.com/operon/>`_ no longer requires a custom
36+
installation, both Operon and Ansible are supported by a unified release.
4037

4138
* `#419 <https://github.com/dw/mitogen/issues/419>`_,
4239
`#470 <https://github.com/dw/mitogen/issues/470>`_, file descriptor usage
4340
during large runs is halved, as it is no longer necessary to manage read and
44-
write sides distinctly in order to work around a design limitation.
41+
write sides distinctly in order to work around a design problem.
4542

4643
* `#419 <https://github.com/dw/mitogen/issues/419>`_: almost all connection
47-
setup happens on one thread, reducing GIL contention and context switching
48-
early in a run.
44+
setup happens on one thread, reducing contention and context switching early
45+
in a run.
4946

5047
* `#419 <https://github.com/dw/mitogen/issues/419>`_: Connection setup is
51-
pipelined, eliminating several network round-trips. Most infrastructure is in
52-
place to support future removal of the final round-trip between a target
53-
fully booting and receiving its first function call.
48+
better pipelined, eliminating some network round-trips. Most infrastructure
49+
is in place to support future removal of the final round-trips between a
50+
target fully booting and receiving function calls.
51+
52+
* `#595 <https://github.com/dw/mitogen/pull/595>`_: the
53+
:meth:`Router.buildah() <mitogen.parent.Router.buildah>` connection method is
54+
available to manipulate `Buildah <https://buildah.io/>`_ containers, and is
55+
exposed to Ansible as the ``buildah`` transport.
56+
57+
* The ``MITOGEN_CPU_COUNT`` environment variable shards the connection
58+
multiplexer into per-CPU workers. This may improve throughput for runs
59+
involving large file transfers, and is required for future in-process SSH
60+
support. One multiplexer starts by default, to match existing behaviour.
5461

5562
* `d6faff06 <https://github.com/dw/mitogen/commit/d6faff06>`_,
5663
`807cbef9 <https://github.com/dw/mitogen/commit/807cbef9>`_,
@@ -193,6 +200,14 @@ Core Library
193200
* `#612 <https://github.com/dw/mitogen/issues/612>`_: fix various errors
194201
introduced by stream refactoring.
195202

203+
* `#615 <https://github.com/dw/mitogen/issues/615>`_: when routing fails to
204+
deliver a message for some reason other than the sender cannot or should not
205+
reach the recipient, and no reply-to address is present on the message,
206+
instead send a dead message to the original recipient. This ensures a
207+
descriptive messages is delivered to a thread sleeping on the reply to a
208+
function call, where the reply might be dropped due to exceeding the maximum
209+
configured message size.
210+
196211
* `a5536c35 <https://github.com/dw/mitogen/commit/a5536c35>`_: avoid quadratic
197212
buffer management when logging lines received from a child's redirected
198213
standard IO.
@@ -230,6 +245,7 @@ bug reports, testing, features and fixes in this release contributed by
230245
`El Mehdi CHAOUKI <https://github.com/elmchaouki>`_,
231246
`Florent Dutheil <https://github.com/fdutheil>`_,
232247
`James Hogarth <https://github.com/hogarthj>`_,
248+
`Jordan Webb <https://github.com/jordemort>`_,
233249
`Marc Hartmayer <https://github.com/marc1006>`_,
234250
`Nigel Metheringham <https://github.com/nigelm>`_,
235251
`Orion Poplawski <https://github.com/opoplawski>`_,

docs/internals.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ Stream, Side & Protocol
6565
.. autoclass:: Stream
6666
:members:
6767

68+
.. currentmodule:: mitogen.core
69+
.. autoclass:: BufferedWriter
70+
:members:
71+
6872
.. currentmodule:: mitogen.core
6973
.. autoclass:: Side
7074
:members:
@@ -81,6 +85,10 @@ Stream, Side & Protocol
8185
.. autoclass:: DelimitedProtocol
8286
:members:
8387

88+
.. currentmodule:: mitogen.parent
89+
.. autoclass:: LogProtocol
90+
:members:
91+
8492
.. currentmodule:: mitogen.core
8593
.. autoclass:: IoLoggerProtocol
8694
:members:

0 commit comments

Comments
 (0)