Merge pull request #1264 from moreati/ansible-2.19

moreati · web-flow · commit 9dee94641fe9 · 2025-06-04T13:05:43.000+01:00
Ansible 2.19 (review/sandbox)
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -177,10 +177,16 @@ jobs:
           - name: Ans_313_11
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible11
+          - name: Ans_313_12
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible12
 
           - name: Van_313_11
             python_version: '3.13'
             tox_env: py313-mode_ansible-ansible11-strategy_linear
+          - name: Van_313_12
+            python_version: '3.13'
+            tox_env: py313-mode_ansible-ansible12-strategy_linear
 
           - name: Mito_313
             python_version: '3.13'
@@ -273,6 +279,12 @@ jobs:
           - name: Van_313_11
             tox_env: py313-mode_localhost-ansible11-strategy_linear
 
+          - name: Loc_313_12
+            tox_env: py313-mode_localhost-ansible12
+
+          - name: Van_313_12
+            tox_env: py313-mode_localhost-ansible12-strategy_linear
+
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
diff --git a/ansible_mitogen/connection.py b/ansible_mitogen/connection.py
@@ -767,7 +767,7 @@ def _stack_from_spec(self, spec, stack=(), seen_names=()):
                               C.BECOME_ALLOW_SAME_USER):
             stack += (CONNECTION_METHOD[spec.become_method()](spec),)
 
-        return stack
+        return ansible_mitogen.utils.unsafe.cast(stack)
 
     def _build_stack(self):
         """
diff --git a/ansible_mitogen/mixins.py b/ansible_mitogen/mixins.py
@@ -402,15 +402,17 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
             if not self._mitogen_rediscovered_interpreter:
                 result['ansible_facts'][self._discovered_interpreter_key] = self._discovered_interpreter
 
-        if self._discovery_warnings:
+        discovery_warnings = getattr(self, '_discovery_warnings', [])
+        if discovery_warnings:
             if result.get('warnings') is None:
                 result['warnings'] = []
-            result['warnings'].extend(self._discovery_warnings)
+            result['warnings'].extend(discovery_warnings)
 
-        if self._discovery_deprecation_warnings:
+        discovery_deprecation_warnings = getattr(self, '_discovery_deprecation_warnings', [])
+        if discovery_deprecation_warnings:
             if result.get('deprecations') is None:
                 result['deprecations'] = []
-            result['deprecations'].extend(self._discovery_deprecation_warnings)
+            result['deprecations'].extend(discovery_deprecation_warnings)
 
         return ansible.utils.unsafe_proxy.wrap_var(result)
 
@@ -429,7 +431,10 @@ def _postprocess_response(self, result):
                     "stderr": "stderr data"
                 }
         """
-        data = self._parse_returned_data(result)
+        if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+            data = self._parse_returned_data(result, profile='legacy')
+        else:
+            data = self._parse_returned_data(result)
 
         # Cutpasted from the base implementation.
         if 'stdout' in data and 'stdout_lines' not in data:
diff --git a/ansible_mitogen/planner.py b/ansible_mitogen/planner.py
@@ -170,6 +170,7 @@ def get_kwargs(self, **kwargs):
         """
         binding = self._inv.connection.get_binding()
 
+        kwargs = ansible_mitogen.utils.unsafe.cast(kwargs)
         new = dict((mitogen.core.UnicodeType(k), kwargs[k])
                    for k in kwargs)
         new.setdefault('good_temp_dir',
@@ -204,7 +205,7 @@ def get_kwargs(self, **kwargs):
             module=self._inv.module_name,
             path=self._inv.module_path,
             json_args=json.dumps(self._inv.module_args),
-            env=self._inv.env,
+            env=ansible_mitogen.utils.unsafe.cast(self._inv.env),
             **kwargs
         )
 
@@ -546,7 +547,7 @@ def _invoke_async_task(invocation, planner):
         call_recv = context.call_async(
             ansible_mitogen.target.run_module_async,
             job_id=job_id,
-            timeout_secs=invocation.timeout_secs,
+            timeout_secs=ansible_mitogen.utils.unsafe.cast(invocation.timeout_secs),
             started_sender=started_recv.to_sender(),
             kwargs=planner.get_kwargs(),
         )
diff --git a/ansible_mitogen/runner.py b/ansible_mitogen/runner.py
@@ -73,6 +73,7 @@
     from io import StringIO
 
 # Prevent accidental import of an Ansible module from hanging on stdin read.
+# FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
 
@@ -635,6 +636,7 @@ def __init__(self, args, temp_dir):
         sys.stderr = StringIO()
         encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args})
         ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded)
+        ansible.module_utils.basic._ANSIBLE_PROFILE = 'legacy'
         sys.stdin = StringIO(mitogen.core.to_text(encoded))
 
         self.original_get_path = getattr(ansible.module_utils.basic,
@@ -649,7 +651,9 @@ def revert(self):
         sys.stdout = self.original_stdout
         sys.stderr = self.original_stderr
         sys.stdin = self.original_stdin
+        # FIXME Should probably be b'{}' or None. Ansible 2.19 has bytes | None = None.
         ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
+        ansible.module_utils.basic._ANSIBLE_PROFILE = None
 
 
 class ProgramRunner(Runner):
diff --git a/ansible_mitogen/services.py b/ansible_mitogen/services.py
@@ -57,6 +57,7 @@
 import ansible_mitogen.loaders
 import ansible_mitogen.module_finder
 import ansible_mitogen.target
+import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
 
@@ -338,7 +339,12 @@ def _on_context_disconnect(self, context):
         'ansible_mitogen.target',
         'mitogen.fork',
         'mitogen.service',
-    )
+    ) + ((
+        'ansible.module_utils._internal._json._profiles._module_legacy_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+        'ansible.module_utils._internal._json._profiles._module_modern_c2m',
+        'ansible.module_utils._internal._json._profiles._module_legacy_m2c',
+    ) if ansible_mitogen.utils.ansible_version[:2] >= (2, 19) else ())
 
     def _send_module_forwards(self, context):
         if hasattr(self.router.responder, 'forward_modules'):
diff --git a/ansible_mitogen/utils/unsafe.py b/ansible_mitogen/utils/unsafe.py
@@ -16,8 +16,11 @@
 
 def _cast_to_dict(obj): return {cast(k): cast(v) for k, v in obj.items()}
 def _cast_to_list(obj): return [cast(v) for v in obj]
+def _cast_to_set(obj): return set(cast(v) for v in obj)
+def _cast_to_tuple(obj): return tuple(cast(v) for v in obj)
 def _cast_unsafe(obj): return obj._strip_unsafe()
 def _passthrough(obj): return obj
+def _untag(obj): return obj._native_copy()
 
 
 # A dispatch table to cast objects based on their exact type.
@@ -26,30 +29,64 @@ def _passthrough(obj): return obj
     bytes: bytes,
     dict: _cast_to_dict,
     list: _cast_to_list,
-    tuple: _cast_to_list,
     mitogen.core.UnicodeType: mitogen.core.UnicodeType,
 }
 _CAST_DISPATCH.update({t: _passthrough for t in mitogen.utils.PASSTHROUGH})
 
-if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+_CAST_SUBTYPES = [
+    dict,
+    list,
+]
+
+if hasattr(ansible.utils.unsafe_proxy, 'TrustedAsTemplate'):
+    import datetime
+    import ansible.module_utils._internal._datatag
     _CAST_DISPATCH.update({
+        set: _cast_to_set,
+        tuple: _cast_to_tuple,
+        ansible.module_utils._internal._datatag._AnsibleTaggedBytes: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDate: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDateTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedDict: _cast_to_dict,
+        ansible.module_utils._internal._datatag._AnsibleTaggedFloat: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedInt: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedList: _cast_to_list,
+        ansible.module_utils._internal._datatag._AnsibleTaggedSet: _cast_to_set,
+        ansible.module_utils._internal._datatag._AnsibleTaggedStr: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTime: _untag,
+        ansible.module_utils._internal._datatag._AnsibleTaggedTuple: _cast_to_tuple,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
+        datetime.date: _passthrough,
+        datetime.datetime: _passthrough,
+        datetime.time: _passthrough,
     })
+    _CAST_SUBTYPES.extend([
+        set,
+        tuple,
+    ])
 elif hasattr(ansible.utils.unsafe_proxy.AnsibleUnsafeText, '_strip_unsafe'):
     _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: _cast_unsafe,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: _cast_unsafe,
         ansible.utils.unsafe_proxy.NativeJinjaUnsafeText: _cast_unsafe,
     })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
 elif ansible_mitogen.utils.ansible_version[:2] <= (2, 16):
     _CAST_DISPATCH.update({
+        tuple: _cast_to_list,
         ansible.utils.unsafe_proxy.AnsibleUnsafeBytes: bytes,
         ansible.utils.unsafe_proxy.AnsibleUnsafeText: mitogen.core.UnicodeType,
     })
+    _CAST_SUBTYPES.extend([
+        tuple,
+    ])
 else:
     mitogen_ver = '.'.join(str(v) for v in mitogen.__version__)
-    raise ImportError("Mitogen %s can't unwrap Ansible %s AnsibleUnsafe objects"
+    raise ImportError("Mitogen %s can't cast Ansible %s objects"
                       % (mitogen_ver, ansible.__version__))
 
 
@@ -78,7 +115,9 @@ def cast(obj):
         return unwrapper(obj)
 
     # Slow path: obj is some unknown subclass
-    if isinstance(obj, dict): return _cast_to_dict(obj)
-    if isinstance(obj, (list, tuple)): return _cast_to_list(obj)
+    for typ_ in _CAST_SUBTYPES:
+        if isinstance(obj, typ_):
+            unwrapper = _CAST_DISPATCH[typ_]
+            return unwrapper(obj)
 
     return mitogen.utils.cast(obj)
diff --git a/docs/ansible_detailed.rst b/docs/ansible_detailed.rst
@@ -141,7 +141,9 @@ Noteworthy Differences
   +-----------------+ 3.10 - 3.13     |
   | 10              |                 |
   +-----------------+-----------------+
-  | 11              | 3.11 - 3.13     |
+  | 11              |                 |
+  +-----------------+ 3.11 - 3.13+    |
+  | 12              |                 |
   +-----------------+-----------------+
 
   Verify your installation is running one of these versions by checking
diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -22,6 +22,10 @@ In progress (unreleased)
 ------------------------
 
 * :gh:issue:`1258` Initial Ansible 12 (ansible-core 2.19) support
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Initial Ansible datatag support
+  (:gh:anspull:`84621`)
+* :gh:issue:`1258` :mod:`ansible_mitogen`: Ansible 12 (ansible-core 2.19) test
+  jobs
 
 
 v0.3.24 (2025-05-29)
diff --git a/docs/conf.py b/docs/conf.py
@@ -64,6 +64,15 @@ def changelog_version(path, encoding='utf-8'):
         'text': '#%s',
         'url': 'https://github.com/mitogen-hq/mitogen/pull/%s',
     },
+    'gh:ansissue': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/issues/%s',
+    },
+    'gh:anspull': {
+        'text': 'Ansible #%s',
+        'url': 'https://github.com/ansible/ansible/pull/%s',
+    },
+
     'ans:mod': {
         'text': '%s module',
         'url': 'https://docs.ansible.com/ansible/latest/modules/%s_module.html',
diff --git a/tests/ansible/integration/action/transfer_data.yml b/tests/ansible/integration/action/transfer_data.yml
@@ -1,7 +1,14 @@
 
-- name: integration/action/transfer_data.yml
+- name: integration/action/transfer_data.yml, json
   hosts: test-targets
   tasks:
+    - meta: end_play
+      when:
+        # Ansible >= 12 (ansible-core >= 2.19) only allows bytes|str through
+        # `ansible.plugins.action.ActionBase._transfer_data()`.
+        - ansible_version.full is version('2.18.999', '>', strict=True)
+        - not is_mitogen
+
     - name: Cleanup transfer data
       file:
         path: /tmp/transfer-data
@@ -15,26 +22,41 @@
           data: {
             "I am JSON": true
           }
+
     - name: Slurp JSON transfer data
       slurp:
         src: /tmp/transfer-data
       register: out
+
     - assert:
         that: |
           out.content|b64decode == '{"I am JSON": true}'
         fail_msg: |
           out={{ out }}
 
+    - name: Cleanup transfer data
+      file:
+        path: /tmp/transfer-data
+        state: absent
+  tags:
+    - transfer_data
+
+
+- name: integration/action/transfer_data.yml, text
+  hosts: test-targets
+  tasks:
     - name: Create text transfer data
       action_passthrough:
         method: _transfer_data
         kwargs:
           remote_path: /tmp/transfer-data
           data: "I am text."
+
     - name: Slurp text transfer data
       slurp:
         src: /tmp/transfer-data
       register: out
+
     - assert:
         that:
           out.content|b64decode == 'I am text.'
diff --git a/tests/ansible/integration/runner/crashy_new_style_module.yml b/tests/ansible/integration/runner/crashy_new_style_module.yml
@@ -18,10 +18,17 @@
         - not out.changed
         - out is failed
         # https://github.com/ansible/ansible/commit/62d8c8fde6a76d9c567ded381e9b34dad69afcd6
-        - out.msg is match(msg_pattern)
-        - (out.module_stdout == "" and out.module_stderr is search(tb_pattern))
-          or
-          (out.module_stdout is search(tb_pattern) and out.module_stderr is match("Shared connection to localhost closed."))
+        - |
+          out.msg is match(msg_pattern)
+          or out.msg in (
+             "Task failed: Module failed: name 'kaboom' is not defined",
+             'Module result deserialization failed: No start of json char found',
+          )
+        # - out.exception is undefined
+        #   or out.exception | default('') is match(tb_pattern)
+        #   or out.module_stderr is search(tb_pattern)
+        # - out.module_stdout == ''
+        # - out.module_stderr is search(tb_pattern)
         fail_msg: |
           out={{ out }}
   tags:
diff --git a/tests/ansible/integration/runner/custom_binary_producing_junk.yml b/tests/ansible/integration/runner/custom_binary_producing_junk.yml
@@ -30,6 +30,7 @@
         - out.failed
         - out.results[0].failed
         - out.results[0].msg.startswith('MODULE FAILURE')
+          or out.results[0].msg.startswith('Module result deserialization failed')
         - out.results[0].rc == 0
         fail_msg: |
           out={{ out }}
diff --git a/tests/ansible/integration/runner/custom_binary_single_null.yml b/tests/ansible/integration/runner/custom_binary_single_null.yml
@@ -15,7 +15,9 @@
       that:
       - "out.failed"
       - "out.results[0].failed"
-      - "out.results[0].msg.startswith('MODULE FAILURE')"
+      - |
+        out.results[0].msg.startswith('MODULE FAILURE')
+        or out.results[0].msg == 'Module result deserialization failed: No start of json char found'
       # On Ubuntu 16.04 /bin/sh is dash 0.5.8. It treats custom_binary_single_null
       # as a valid executable. There's no error message, and rc == 0.
       - |
diff --git a/tests/ansible/integration/runner/missing_module.yml b/tests/ansible/integration/runner/missing_module.yml
@@ -26,6 +26,7 @@
     - assert:
         that: |
           'The module missing_module was not found in configured module paths' in out.stdout
+          or "Cannot resolve 'missing_module' to an action or module" in out.stdout
         fail_msg: |
           out={{ out }}
   tags:
diff --git a/tests/ansible/integration/ssh/timeouts.yml b/tests/ansible/integration/ssh/timeouts.yml
@@ -43,6 +43,7 @@
           '"unreachable": true' in out.stdout
         - |
           '"msg": "Connection timed out."' in out.stdout
+          or '"msg": "Task failed: Connection timed out."' in out.stdout
         fail_msg: |
           out={{ out }}
   tags:
diff --git a/tests/ansible/integration/transport_config/password.yml b/tests/ansible/integration/transport_config/password.yml
diff --git a/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml b/tests/ansible/regression/issue_109__target_has_old_ansible_installed.yml
diff --git a/tests/ansible/regression/issue_766__get_with_context.yml b/tests/ansible/regression/issue_766__get_with_context.yml
diff --git a/tests/ansible/tests/utils_unsafe_test.py b/tests/ansible/tests/utils_unsafe_test.py
