dyninst/*: report why @duration is missing more clearly (#44370)

### What does this PR do?

This commit does plumbing to make it more explicit why we don't have a value for @duration.

### Motivation

It was confusing when before it just said it wasn't available.

### Describe how you validated your changes

There's testing

### Additional Notes

Fixes [DEBUG-4866](https://datadoghq.atlassian.net/browse/DEBUG-4866)

[DEBUG-4866]: https://datadoghq.atlassian.net/browse/DEBUG-4866?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ

Co-authored-by: andrew.werner <andrew.werner@datadoghq.com>

Author: ajwerner

pkg/dyninst/compiler/functions.go

@@ -50,6 +50,7 @@ type ProcessEvent struct {
 	StringSizeLimit     uint32
 	Frameless           bool
 	HasAssociatedReturn bool
+	NoReturnReason      ir.NoReturnReason
 	EventKind           ir.EventKind
 	TopPCOffset         int8
 	EventRootType       *ir.EventRootType

pkg/dyninst/compiler/generate.go

@@ -169,6 +169,7 @@ func (g *generator) addEventHandler(
 		StringSizeLimit:     captureConfig.GetMaxLength(),
 		Frameless:           injectionPoint.Frameless,
 		HasAssociatedReturn: injectionPoint.HasAssociatedReturn,
+		NoReturnReason:      injectionPoint.NoReturnReason,
 		TopPCOffset:         injectionPoint.TopPCOffset,
 		ProbeID:             probeID,
 		EventKind:           eventKind,

pkg/dyninst/decode/decoder.go

@@ -300,6 +300,7 @@ func (s *message) init(
 		s.Debugger.Snapshot.captures.Lines = &decoder.line
 	}
 	var returnHeader *output.EventHeader
+	var durationMissingReason *string
 	if event.Return != nil {
 		if err := decoder._return.init(
 			event.Return, decoder.program.Types, &s.Debugger.EvaluationErrors,
@@ -331,20 +332,27 @@ func (s *message) init(
 			reason = "call map capacity exceeded"
 		case output.EventPairingExpectationCallCountExceeded:
 			reason = "maximum call count exceeded"
+		case output.EventPairingExpectationNoneInlined:
+			reason = "function was inlined"
+		case output.EventPairingExpectationNoneNoBody:
+			reason = "function has no body"
 		}
+		log.Tracef("no return reason: %v pairing expectation: %v", reason, pairingExpectation)
 		// The choice to use @duration here is somewhat arbitrary; we want to
 		// choose something that definitely can't collide with a real variable
 		// and is evocative of the thing that is missing. Indeed we know in this
 		// situation we will never @duration, so it seems like a good choice.
 		const missingReturnReasonExpression = "@duration"
 		if reason != "" {
+			message := "not available: " + reason
 			s.Debugger.EvaluationErrors = append(
 				s.Debugger.EvaluationErrors,
 				evaluationError{
 					Expression: missingReturnReasonExpression,
-					Message:    "no return value available: " + reason,
+					Message:    message,
 				},
 			)
+			durationMissingReason = &message
 		}
 	}
 
@@ -432,6 +440,8 @@ func (s *message) init(
 		}
 		if s.Duration != 0 {
 			s.Message.duration = &s.Duration
+		} else if durationMissingReason != nil {
+			s.Message.durationMissingReason = durationMissingReason
 		}
 	}
 

pkg/dyninst/decode/decoder_test.go

@@ -1448,28 +1448,42 @@ func TestDecoderMissingReturnEventEvaluationError(t *testing.T) {
 			name:                    "return pairing expected",
 			pairingExpectation:      output.EventPairingExpectationReturnPairingExpected,
 			expectedErrorExpression: "@duration",
-			expectedErrorMessage:    "no return value available: return event not received",
+			expectedErrorMessage:    "not available: return event not received",
 			shouldHaveError:         true,
 		},
 		{
 			name:                    "buffer full",
 			pairingExpectation:      output.EventPairingExpectationBufferFull,
 			expectedErrorExpression: "@duration",
-			expectedErrorMessage:    "no return value available: userspace buffer capacity exceeded",
+			expectedErrorMessage:    "not available: userspace buffer capacity exceeded",
 			shouldHaveError:         true,
 		},
 		{
 			name:                    "call map full",
 			pairingExpectation:      output.EventPairingExpectationCallMapFull,
 			expectedErrorExpression: "@duration",
-			expectedErrorMessage:    "no return value available: call map capacity exceeded",
+			expectedErrorMessage:    "not available: call map capacity exceeded",
 			shouldHaveError:         true,
 		},
 		{
 			name:                    "call count exceeded",
 			pairingExpectation:      output.EventPairingExpectationCallCountExceeded,
 			expectedErrorExpression: "@duration",
-			expectedErrorMessage:    "no return value available: maximum call count exceeded",
+			expectedErrorMessage:    "not available: maximum call count exceeded",
+			shouldHaveError:         true,
+		},
+		{
+			name:                    "inlined",
+			pairingExpectation:      output.EventPairingExpectationNoneInlined,
+			expectedErrorExpression: "@duration",
+			expectedErrorMessage:    "not available: function was inlined",
+			shouldHaveError:         true,
+		},
+		{
+			name:                    "no body",
+			pairingExpectation:      output.EventPairingExpectationNoneNoBody,
+			expectedErrorExpression: "@duration",
+			expectedErrorMessage:    "not available: function has no body",
 			shouldHaveError:         true,
 		},
 		{

pkg/dyninst/decode/marshal.go

@@ -38,10 +38,11 @@ type debuggerData struct {
 }
 
 type messageData struct {
-	duration    *uint64
-	entryOrLine *captureEvent
-	_return     *captureEvent
-	template    *ir.Template
+	duration              *uint64
+	durationMissingReason *string
+	entryOrLine           *captureEvent
+	_return               *captureEvent
+	template              *ir.Template
 }
 
 func (m *messageData) MarshalJSONTo(enc *jsontext.Encoder) error {
@@ -83,7 +84,11 @@ func (m *messageData) MarshalJSONTo(enc *jsontext.Encoder) error {
 			writeBoundedError(&result, limits, "error", seg.Error)
 		case *ir.DurationSegment:
 			if m.duration == nil {
-				writeBoundedError(&result, limits, "error", "@duration is not available")
+				if m.durationMissingReason != nil {
+					writeBoundedError(&result, limits, "error", *m.durationMissingReason)
+				} else {
+					writeBoundedError(&result, limits, "error", "@duration is not available")
+				}
 			} else {
 				n, _ := fmt.Fprintf(&result, "%f", time.Duration(*m.duration).Seconds()*1000)
 				limits.consume(n)

pkg/dyninst/ebpf/event.c

@@ -182,7 +182,19 @@ probe_run(uint64_t start_ns, const probe_params_t* params, struct pt_regs* regs)
       }
     }
   } else {
-    header->event_pairing_expectation = EVENT_PAIRING_EXPECTATION_NONE;
+    switch (params->no_return_reason) {
+    case NO_RETURN_REASON_INLINED:
+      LOG(4, "no return reason: inlined for goid %lld stack byte depth %d probe id %d", header->goid, header->stack_byte_depth, params->probe_id);
+      header->event_pairing_expectation = EVENT_PAIRING_EXPECTATION_NONE_INLINED;
+      break;
+    case NO_RETURN_REASON_NO_BODY:
+      LOG(4, "no return body for goid %lld stack byte depth %d probe id %d", header->goid, header->stack_byte_depth, params->probe_id);
+      header->event_pairing_expectation = EVENT_PAIRING_EXPECTATION_NONE_NO_BODY;
+      break;
+    default:
+      header->event_pairing_expectation = EVENT_PAIRING_EXPECTATION_NONE;
+      break;
+    }
   }
   __maybe_unused int process_steps = 0;
   __maybe_unused int chase_steps = 0;

pkg/dyninst/ebpf/framing.h

@@ -16,7 +16,10 @@ typedef enum event_pairing_expectation {
   EVENT_PAIRING_RETURN_PAIRING_EXPECTED = 2,
   EVENT_PAIRING_EXPECTATION_CALL_COUNT_EXCEEDED = 3,
   EVENT_PAIRING_EXPECTATION_CALL_MAP_FULL = 4,
-} return_call_ommitted_reason_t;
+  EVENT_PAIRING_EXPECTATION_BUFFER_FULL = 5, // only used in userspace
+  EVENT_PAIRING_EXPECTATION_NONE_INLINED = 6,
+  EVENT_PAIRING_EXPECTATION_NONE_NO_BODY = 7,
+} event_pairing_expectation_t;
 
 // The message header used for the event program.
 typedef struct di_event_header {

pkg/dyninst/ebpf/types.h

@@ -14,7 +14,8 @@ typedef struct probe_params {
   bool has_associated_return;
   char kind; // actually an event_kind_t
   char top_pc_offset;
-  char __padding[4];
+  char no_return_reason;
+  char __padding[3];
 } probe_params_t;
 
 typedef struct throttler_params {
@@ -49,6 +50,16 @@ typedef enum event_kind {
   EVENT_KIND_RETURN = 2,
 } event_kind_t;
 
+// To be kept in sync with the ir.NoReturnReason enum in the ir/program.go file.
+typedef enum no_return_reason {
+  NO_RETURN_REASON_NONE = 0,
+  NO_RETURN_REASON_RETURNS_DISABLED = 1,
+  NO_RETURN_REASON_LINE_PROBE = 2,
+  NO_RETURN_REASON_INLINED = 3,
+  NO_RETURN_REASON_NO_BODY = 4,
+  NO_RETURN_REASON_IS_RETURN = 5,
+} no_return_reason_t;
+
 typedef enum sm_opcode {
   SM_OP_INVALID = 0,
   // Execution flow ops.

pkg/dyninst/ir/program.go

@@ -224,6 +224,22 @@ type InjectionPoint struct {
 	// HasAssociatedReturn is true if there is going to be a return associated
 	// with this call.
 	HasAssociatedReturn bool `json:"-"`
+	// NoReturnReason is the reason why there is no return associated with this
+	// call. Only set if HasAssociatedReturn is false.
+	NoReturnReason NoReturnReason `json:"-"`
 	// TopPCOffset is the offset of the top PC from the entry PC.
 	TopPCOffset int8 `json:"-"`
 }
+
+// This must be kept in sync with the no_return_reason enum in the ebpf/types.h
+// file.
+type NoReturnReason uint8
+
+const (
+	NoReturnReasonNone            NoReturnReason = 0
+	NoReturnReasonReturnsDisabled NoReturnReason = 1
+	NoReturnReasonLineProbe       NoReturnReason = 2
+	NoReturnReasonInlined         NoReturnReason = 3
+	NoReturnReasonNoBody          NoReturnReason = 4
+	NoReturnReasonIsReturn        NoReturnReason = 5
+)

pkg/dyninst/irgen/irgen.go

@@ -3559,6 +3559,7 @@ func pickInjectionPoint(
 				PC:                  ranges[0][0],
 				Frameless:           frameless,
 				HasAssociatedReturn: false,
+				NoReturnReason:      ir.NoReturnReasonInlined,
 			})
 		} else {
 			call, err := pickCallInjectionPoint(arch, addr, frameless, lines)
@@ -3580,21 +3581,30 @@ func pickInjectionPoint(
 				return buf, nil, issue, nil
 			}
 
-			// Add a workaround for the fact that single-instruction functions
-			// would have the same entry and exit probes, but the ordering between
-			// them would not be well-defined, so in this extremely uncommon case
-			// the user doesn't get to see the return probe. It's okay because
-			// there literally cannot be a return value.
-			hasAssociatedReturn := !skipReturnEvents
-			if len(returnLocations) == 1 && returnLocations[0].PC == call.pc {
+			var hasAssociatedReturn bool
+			var noReturnReason ir.NoReturnReason
+			if skipReturnEvents {
 				hasAssociatedReturn = false
+				noReturnReason = ir.NoReturnReasonReturnsDisabled
+			} else if len(returnLocations) == 1 && returnLocations[0].PC == call.pc {
+				// Add a workaround for the fact that single-instruction
+				// functions would have the same entry and exit probes, but the
+				// ordering between them would not be well-defined, so in this
+				// extremely uncommon case the user doesn't get to see the
+				// return probe. It's okay because there literally cannot be a
+				// return value.
+				hasAssociatedReturn = false
+				noReturnReason = ir.NoReturnReasonNoBody
 				returnLocations = returnLocations[:0]
+			} else {
+				hasAssociatedReturn = true
 			}
 
 			buf = append(buf, ir.InjectionPoint{
 				PC:                  call.pc,
 				Frameless:           call.frameless,
 				HasAssociatedReturn: hasAssociatedReturn,
+				NoReturnReason:      noReturnReason,
 				TopPCOffset:         call.topPCOffset,
 			})
 			if hasAssociatedReturn {
@@ -3635,6 +3645,7 @@ func pickInjectionPoint(
 			PC:                  injectionPC,
 			Frameless:           frameless,
 			HasAssociatedReturn: false,
+			NoReturnReason:      ir.NoReturnReasonLineProbe,
 		})
 	}
 	return buf, returnEvent, ir.Issue{}, nil
@@ -3826,6 +3837,7 @@ func disassembleAmd64Function(
 					PC:                  epilogueStart,
 					Frameless:           frameless,
 					HasAssociatedReturn: false,
+					NoReturnReason:      ir.NoReturnReasonIsReturn,
 				})
 			}
 		}
@@ -3834,6 +3846,7 @@ func disassembleAmd64Function(
 				PC:                  addr + uint64(offset),
 				Frameless:           frameless,
 				HasAssociatedReturn: false,
+				NoReturnReason:      ir.NoReturnReasonIsReturn,
 			})
 		}
 		offset += instruction.Len