fix(ray): runtime_env can be None leading to a job error [backport 3.16] (#14903)

Backport f11c438 from #14837 to 3.16.

When submitting a job, `runtime_env` could be none in `kwargs`, leading
to an `AttributeError`, see this
[link](https://dd.datad0g.com/llm/distributed-ai/jobs?query=%40component%3Aray%20parent_id%3A0%20job-113464b6-33d0-47&agg_m=count&agg_m_source=base&agg_t=count&colorLegendSort=time&fromUser=false&spanId=5240087393744960581&traceId=68e818d800000000e5341ed29f2823d9&start=1759437349664&end=1760042149664&paused=false).

This PR fixes this issue. It also updates two snapshots that were
missing a span. The tests were xfailing so this was not caught in CI.

Co-authored-by: Louis Tricot <[email protected]>

Author: dd-octo-sts[bot]

ddtrace/contrib/internal/ray/patch.py

@@ -232,7 +232,11 @@ def traced_submit_job(wrapped, instance, args, kwargs):
             submit_span.set_tag_str(RAY_SUBMISSION_ID_TAG, submission_id)
 
             # Inject the context of the job so that ray.job.run is its child
-            env_vars = kwargs.setdefault("runtime_env", {}).setdefault("env_vars", {})
+            runtime_env = kwargs.get("runtime_env") or {}
+            kwargs["runtime_env"] = runtime_env
+            env_vars = runtime_env.get("env_vars") or {}
+            runtime_env["env_vars"] = env_vars
+
             _TraceContext._inject(job_span.context, env_vars)
             env_vars[RAY_SUBMISSION_ID] = submission_id
             if job_name:

releasenotes/notes/fix-ray-none-runtime-068a35befe0d2533.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    ray: This fix resolves an issue where submitting Ray jobs caused an ``AttributeError`` crash in certain configurations.

tests/snapshots/tests.contrib.ray.test_ray.test_simple_put.json

@@ -10,18 +10,17 @@
     "error": 0,
     "meta": {
       "_dd.base_service": "tests.contrib.ray",
-      "_dd.hostname": "docker-desktop",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "68de3cb400000000",
+      "_dd.p.tid": "68e8d2ce00000000",
       "component": "ray",
       "language": "python",
       "ray.hostname": "docker-desktop",
       "ray.job_id": "01000000",
-      "ray.node_id": "6e08a6b14aa1db44ba918cb17ed0223d61e75c5193e0df2b881dd2e2",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
       "ray.put.value_size_bytes": "28",
       "ray.put.value_type": "int",
       "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-      "runtime-id": "5e2c80f345d34a8082da382e68b097bc",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
       "span.kind": "producer"
     },
     "metrics": {
@@ -34,8 +33,8 @@
       "_sampling_priority_v1": 2,
       "process_id": 543
     },
-    "duration": 190167,
-    "start": 1759394996599461798
+    "duration": 214833,
+    "start": 1760088782207075883
   }],
 [
   {
@@ -48,17 +47,16 @@
     "type": "ray",
     "error": 0,
     "meta": {
-      "_dd.hostname": "docker-desktop",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "68de3cb400000000",
+      "_dd.p.tid": "68e8d2ce00000000",
       "component": "ray",
       "language": "python",
       "ray.hostname": "docker-desktop",
       "ray.job_id": "01000000",
-      "ray.node_id": "6e08a6b14aa1db44ba918cb17ed0223d61e75c5193e0df2b881dd2e2",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
       "ray.task.submit_status": "success",
       "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-      "runtime-id": "5e2c80f345d34a8082da382e68b097bc",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
       "span.kind": "producer"
     },
     "metrics": {
@@ -71,8 +69,8 @@
       "_sampling_priority_v1": 2,
       "process_id": 543
     },
-    "duration": 2298667,
-    "start": 1759394996599837756
+    "duration": 2818542,
+    "start": 1760088782207505258
   },
      {
        "name": "task.execute",
@@ -87,10 +85,10 @@
          "component": "ray",
          "ray.hostname": "docker-desktop",
          "ray.job_id": "01000000",
-         "ray.node_id": "6e08a6b14aa1db44ba918cb17ed0223d61e75c5193e0df2b881dd2e2",
+         "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
          "ray.task.status": "success",
          "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-         "runtime-id": "5e2c80f345d34a8082da382e68b097bc",
+         "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
          "span.kind": "consumer"
        },
        "metrics": {
@@ -102,6 +100,43 @@
          "_sampling_priority_v1": 2,
          "process_id": 543
        },
-       "duration": 196875,
-       "start": 1759394996601777715
-     }]]
+       "duration": 259500,
+       "start": 1760088782209866550
+     }],
+[
+  {
+    "name": "ray.get",
+    "service": "unnamed.ray.job",
+    "resource": "ray.get",
+    "trace_id": 2,
+    "span_id": 1,
+    "parent_id": 0,
+    "type": "ray",
+    "error": 0,
+    "meta": {
+      "_dd.base_service": "tests.contrib.ray",
+      "_dd.p.dm": "-0",
+      "_dd.p.tid": "68e8d2ce00000000",
+      "component": "ray",
+      "language": "python",
+      "ray.get.value_size_bytes": "64",
+      "ray.hostname": "docker-desktop",
+      "ray.job_id": "01000000",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
+      "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
+      "span.kind": "producer"
+    },
+    "metrics": {
+      "_dd.ai_obs.enabled": 1,
+      "_dd.djm.enabled": 1,
+      "_dd.filter.kept": 1,
+      "_dd.measured": 1,
+      "_dd.top_level": 1,
+      "_dd.tracer_kr": 1.0,
+      "_sampling_priority_v1": 2,
+      "process_id": 543
+    },
+    "duration": 347000,
+    "start": 1760088782210520300
+  }]]

tests/snapshots/tests.contrib.ray.test_ray.test_simple_wait.json

@@ -9,17 +9,16 @@
     "type": "ray",
     "error": 0,
     "meta": {
-      "_dd.hostname": "docker-desktop",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "68dd410c00000000",
+      "_dd.p.tid": "68e8d2ce00000000",
       "component": "ray",
       "language": "python",
       "ray.hostname": "docker-desktop",
       "ray.job_id": "01000000",
-      "ray.node_id": "53c13c58eb47ac2803bc79a5dc776959895f9f822320fdf1ab6a4f41",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
       "ray.task.submit_status": "success",
       "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-      "runtime-id": "7fb280a5458e4eecaeae186719e61896",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
       "span.kind": "producer"
     },
     "metrics": {
@@ -32,8 +31,8 @@
       "_sampling_priority_v1": 2,
       "process_id": 543
     },
-    "duration": 3002916,
-    "start": 1759330572760058212
+    "duration": 3052292,
+    "start": 1760088782235622758
   },
      {
        "name": "task.execute",
@@ -48,10 +47,10 @@
          "component": "ray",
          "ray.hostname": "docker-desktop",
          "ray.job_id": "01000000",
-         "ray.node_id": "53c13c58eb47ac2803bc79a5dc776959895f9f822320fdf1ab6a4f41",
+         "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
          "ray.task.status": "success",
          "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-         "runtime-id": "7fb280a5458e4eecaeae186719e61896",
+         "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
          "span.kind": "consumer"
        },
        "metrics": {
@@ -63,8 +62,8 @@
          "_sampling_priority_v1": 2,
          "process_id": 543
        },
-       "duration": 291083,
-       "start": 1759330572762500212
+       "duration": 248958,
+       "start": 1760088782238245050
      }],
 [
   {
@@ -77,18 +76,17 @@
     "type": "ray",
     "error": 0,
     "meta": {
-      "_dd.hostname": "docker-desktop",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "68dd410c00000000",
+      "_dd.p.tid": "68e8d2ce00000000",
       "component": "ray",
       "language": "python",
       "ray.hostname": "docker-desktop",
       "ray.job_id": "01000000",
-      "ray.node_id": "53c13c58eb47ac2803bc79a5dc776959895f9f822320fdf1ab6a4f41",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
       "ray.wait.num_returns": "1",
       "ray.wait.timeout_s": "60",
       "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
-      "runtime-id": "7fb280a5458e4eecaeae186719e61896",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
       "span.kind": "producer"
     },
     "metrics": {
@@ -101,6 +99,43 @@
       "_sampling_priority_v1": 2,
       "process_id": 543
     },
-    "duration": 414292,
-    "start": 1759330572763224920
+    "duration": 235042,
+    "start": 1760088782238835341
+  }],
+[
+  {
+    "name": "ray.get",
+    "service": "unnamed.ray.job",
+    "resource": "ray.get",
+    "trace_id": 2,
+    "span_id": 1,
+    "parent_id": 0,
+    "type": "ray",
+    "error": 0,
+    "meta": {
+      "_dd.base_service": "tests.contrib.ray",
+      "_dd.p.dm": "-0",
+      "_dd.p.tid": "68e8d2ce00000000",
+      "component": "ray",
+      "language": "python",
+      "ray.get.value_size_bytes": "88",
+      "ray.hostname": "docker-desktop",
+      "ray.job_id": "01000000",
+      "ray.node_id": "84adfe319dc863f16fa08d39416a29717ea886f5c78ec87297358957",
+      "ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
+      "runtime-id": "5e417ffae56949ccbc063a6f4c923f97",
+      "span.kind": "producer"
+    },
+    "metrics": {
+      "_dd.ai_obs.enabled": 1,
+      "_dd.djm.enabled": 1,
+      "_dd.filter.kept": 1,
+      "_dd.measured": 1,
+      "_dd.top_level": 1,
+      "_dd.tracer_kr": 1.0,
+      "_sampling_priority_v1": 2,
+      "process_id": 543
+    },
+    "duration": 327208,
+    "start": 1760088782239202258
   }]]