[receiver/github] add job queue spans (#39511)

#### Description

The current implementation of the GitHub receiver doesn't capture
directly the time a job is stuck in queue. The queue duration is a solid
indicator of issues in the build system and necessary for good
diagnosis. This pull request adds queue spans for the workflows jobs
(workflow runs have to be handled separately) and a new
`cicd.pipeline.run.queue.duration` attribute associated with each queue
span. It predates finalization of the semantic conventions, but is in
line with current WIP.

#### Link to tracking issue
Relates to #39081

#### Testing

Updated golden tests. Screenshot of actual traces below.


![image](https://github.com/user-attachments/assets/e1d8dc12-4b9e-4171-ab66-606592bb9f4e)

#### Documentation

n/a

Author: adrielp

.chloggen/gh-queue-spans.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: githubreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add dedicated job queue spans with cicd.pipeline.run.queue.duration attribute
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [39081]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/githubreceiver/model.go

@@ -94,12 +94,13 @@ const (
 	AttributeCICDPipelineTaskRunSenderLogin     = "cicd.pipeline.task.run.sender.login" // GitHub's Task Sender Login
 	AttributeCICDPipelineFilePath               = "cicd.pipeline.file.path"             // GitHub's Path in workflow_run
 	AttributeCICDPipelinePreviousAttemptURLFull = "cicd.pipeline.run.previous_attempt.url.full"
-	AttributeCICDPipelineWorkerID               = "cicd.pipeline.worker.id"         // GitHub's Runner ID
-	AttributeCICDPipelineWorkerGroupID          = "cicd.pipeline.worker.group.id"   // GitHub's Runner Group ID
-	AttributeCICDPipelineWorkerName             = "cicd.pipeline.worker.name"       // GitHub's Runner Name
-	AttributeCICDPipelineWorkerGroupName        = "cicd.pipeline.worker.group.name" // GitHub's Runner Group Name
-	AttributeCICDPipelineWorkerNodeID           = "cicd.pipeline.worker.node.id"    // GitHub's Runner Node ID
-	AttributeCICDPipelineWorkerLabels           = "cicd.pipeline.worker.labels"     // GitHub's Runner Labels
+	AttributeCICDPipelineWorkerID               = "cicd.pipeline.worker.id"          // GitHub's Runner ID
+	AttributeCICDPipelineWorkerGroupID          = "cicd.pipeline.worker.group.id"    // GitHub's Runner Group ID
+	AttributeCICDPipelineWorkerName             = "cicd.pipeline.worker.name"        // GitHub's Runner Name
+	AttributeCICDPipelineWorkerGroupName        = "cicd.pipeline.worker.group.name"  // GitHub's Runner Group Name
+	AttributeCICDPipelineWorkerNodeID           = "cicd.pipeline.worker.node.id"     // GitHub's Runner Node ID
+	AttributeCICDPipelineWorkerLabels           = "cicd.pipeline.worker.labels"      // GitHub's Runner Labels
+	AttributeCICDPipelineRunQueueDuration       = "cicd.pipeline.run.queue.duration" // GitHub's Queue Duration
 
 	// The following attributes are exclusive to GitHub but not listed under
 	// Vendor Extensions within Semantic Conventions yet.

receiver/githubreceiver/testdata/workflow-job-expected.yaml

@@ -57,16 +57,30 @@ resourceSpans:
     scopeSpans:
       - scope: {}
         spans:
-          - endTimeUnixNano: "1744837823000000000"
+          - endTimeUnixNano: "1744837825000000000"
             kind: 2
             name: test (1.23)
             parentSpanId: aba151af7cfbcf0f
             spanId: e68a36b3514c8634
-            startTimeUnixNano: "1744837742000000000"
+            startTimeUnixNano: "1744837738000000000"
             status:
               code: 1
               message: success
             traceId: 731ec8a47fd7450f753a812a4a8aa5a0
+      - scope: {}
+        spans:
+          - attributes:
+              - key: cicd.pipeline.run.queue.duration
+                value:
+                  doubleValue: 5e+09
+            endTimeUnixNano: "1744837743000000000"
+            kind: 2
+            name: queue-test (1.23)
+            parentSpanId: e68a36b3514c8634
+            spanId: d328504edfc4f5dc
+            startTimeUnixNano: "1744837738000000000"
+            status: {}
+            traceId: 731ec8a47fd7450f753a812a4a8aa5a0
       - scope: {}
         spans:
           - attributes:
@@ -79,7 +93,7 @@ resourceSpans:
             endTimeUnixNano: "1744837744000000000"
             kind: 2
             name: Set up job
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: 103b52e85d34ae1c
             startTimeUnixNano: "1744837742000000000"
             status:
@@ -98,7 +112,7 @@ resourceSpans:
             endTimeUnixNano: "1744837745000000000"
             kind: 2
             name: Run actions/checkout@v4
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: b0a2cdd89199b9a5
             startTimeUnixNano: "1744837744000000000"
             status:
@@ -117,7 +131,7 @@ resourceSpans:
             endTimeUnixNano: "1744837745000000000"
             kind: 2
             name: Set up Go
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: c04460ffd6bf3917
             startTimeUnixNano: "1744837745000000000"
             status:
@@ -136,7 +150,7 @@ resourceSpans:
             endTimeUnixNano: "1744837820000000000"
             kind: 2
             name: Make test-all
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: 1684627ec86a096c
             startTimeUnixNano: "1744837746000000000"
             status:
@@ -155,7 +169,7 @@ resourceSpans:
             endTimeUnixNano: "1744837823000000000"
             kind: 2
             name: Upload coverage to Codecov
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: 25aeeca484f9f7e5
             startTimeUnixNano: "1744837820000000000"
             status:
@@ -174,7 +188,7 @@ resourceSpans:
             endTimeUnixNano: "1744837823000000000"
             kind: 2
             name: Post Set up Go
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: 0715a13bb338e46e
             startTimeUnixNano: "1744837823000000000"
             status:
@@ -193,7 +207,7 @@ resourceSpans:
             endTimeUnixNano: "1744837825000000000"
             kind: 2
             name: Post Run actions/checkout@v4
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: e231dce10420723b
             startTimeUnixNano: "1744837825000000000"
             status:
@@ -212,7 +226,7 @@ resourceSpans:
             endTimeUnixNano: "1744837823000000000"
             kind: 2
             name: Complete job
-            parentSpanId: e68a36b3514c8634
+            parentSpanId: d328504edfc4f5dc
             spanId: bb94c07ba3717632
             startTimeUnixNano: "1744837823000000000"
             status:

receiver/githubreceiver/trace_event_handling.go

@@ -67,7 +67,13 @@ func (gtr *githubTracesReceiver) handleWorkflowJob(e *github.WorkflowJobEvent) (
 		return ptrace.Traces{}, errors.New("failed to create parent span")
 	}
 
-	err = gtr.createStepSpans(r, e, traceID, parentID)
+	queueSpanID, err := gtr.createJobQueueSpan(r, e, traceID, parentID)
+	if err != nil {
+		gtr.logger.Sugar().Error("failed to create job queue span", zap.Error(err))
+		return ptrace.Traces{}, errors.New("failed to create job queue span")
+	}
+
+	err = gtr.createStepSpans(r, e, traceID, queueSpanID)
 	if err != nil {
 		gtr.logger.Sugar().Error("failed to create step spans", zap.Error(err))
 		return ptrace.Traces{}, errors.New("failed to create step spans")
@@ -190,18 +196,8 @@ func (gtr *githubTracesReceiver) createParentSpan(
 	span.SetName(event.GetWorkflowJob().GetName())
 	span.SetKind(ptrace.SpanKindServer)
 
-	// Workflow Job event start times provided by GitHub do not always match the
-	// start time of the actual job. Generally they are reported a second after
-	// the actual step start time. Thus, we use the first step start time as the
-	// span start time while using the normal completedAt time for the end time.
-	steps := event.GetWorkflowJob().Steps
-	if len(steps) > 0 {
-		span.SetStartTimestamp(pcommon.NewTimestampFromTime(steps[0].GetStartedAt().Time))
-		span.SetEndTimestamp(pcommon.NewTimestampFromTime(steps[len(steps)-1].GetCompletedAt().Time))
-	} else {
-		span.SetStartTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetStartedAt().Time))
-		span.SetEndTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetCompletedAt().Time))
-	}
+	span.SetStartTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetCreatedAt().Time))
+	span.SetEndTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetCompletedAt().Time))
 
 	switch strings.ToLower(event.WorkflowJob.GetConclusion()) {
 	case "success":
@@ -361,3 +357,41 @@ func newStepSpanID(runID int64, runAttempt int, jobName string, stepName string,
 
 	return spanID, nil
 }
+
+// createJobQueueSpan creates a span for the job queue based on the provided
+// event by using the delta between the job created and completed times.
+func (gtr *githubTracesReceiver) createJobQueueSpan(
+	resourceSpans ptrace.ResourceSpans,
+	event *github.WorkflowJobEvent,
+	traceID pcommon.TraceID,
+	parentSpanID pcommon.SpanID,
+) (pcommon.SpanID, error) {
+	scopeSpans := resourceSpans.ScopeSpans().AppendEmpty()
+	span := scopeSpans.Spans().AppendEmpty()
+	jobName := event.GetWorkflowJob().GetName()
+	spanName := fmt.Sprintf("queue-%s", jobName)
+
+	span.SetName(spanName)
+	span.SetKind(ptrace.SpanKindServer)
+	span.SetTraceID(traceID)
+	span.SetParentSpanID(parentSpanID)
+
+	runID := event.GetWorkflowJob().GetRunID()
+	runAttempt := int(event.GetWorkflowJob().GetRunAttempt())
+	spanID, err := newStepSpanID(runID, runAttempt, jobName, spanName, 1)
+	if err != nil {
+		return pcommon.SpanID{}, fmt.Errorf("failed to generate step span ID: %w", err)
+	}
+
+	span.SetSpanID(spanID)
+
+	time := event.GetWorkflowJob().GetStartedAt().Sub(event.GetWorkflowJob().GetCreatedAt().Time)
+
+	attrs := span.Attributes()
+	attrs.PutDouble(AttributeCICDPipelineRunQueueDuration, float64(time.Nanoseconds()))
+
+	span.SetStartTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetCreatedAt().Time))
+	span.SetEndTimestamp(pcommon.NewTimestampFromTime(event.GetWorkflowJob().GetStartedAt().Time))
+
+	return spanID, nil
+}