Handle edge case for ecs observer job relabel (#352)

* Handle edge case for ecs observer job relabel

* Update readme for ecsobserver job_label_name

* Fix linting

---------

Co-authored-by: Akansha Agarwal <[email protected]>

Author: agarakan

extension/observer/ecsobserver/README.md

@@ -33,6 +33,7 @@ extensions:
     cluster_name: 'Cluster-1' # cluster name need manual config
     cluster_region: 'us-west-2' # region can be configured directly or use AWS_REGION env var
     result_file: '/etc/ecs_sd_targets.yaml' # the directory for file must already exists
+    job_label_name: 'prometheus_job' # optional: override for job label name
     services:
       - name_pattern: '^retail-.*$'
     docker_labels:
@@ -86,6 +87,7 @@ service:
 | cluster_region   | Mandatory | target ECS cluster's AWS region name                                                                                |
 | refresh_interval | Optional  | how often to look for changes in endpoints (default: 10s)                                                           |
 | result_file      | Mandatory | path of YAML file to write scrape target results. NOTE: the observer always returns empty in initial implementation |
+| job_label_name   | Optional  | override for prometheus job label name. If empty or set to "job", no behavior change.                               |
 | services         | Optional  | list of service name patterns [detail](#ecs-service-name-based-filter-configuration)                                |
 | task_definitions | Optional  | list of task definition arn patterns [detail](#ecs-task-definition-based-filter-configuration)                      |
 | docker_labels    | Optional  | list of docker labels [detail](#docker-label-based-filter-configuration)                                            |
@@ -303,6 +305,15 @@ Required for prometheus to scrape the target.
 | ` __metrics_path__` | ECS TaskDefinition or Config | string | Default is `/metrics`, changes based on config/label                      |
 | `job`               | ECS TaskDefinition or Config | string | Name for scrape job                                                       |
 
+### Job Label Behavior
+
+The `job_label_name` configuration controls how job labels are handled in the targets printed to the results file:
+
+- **Empty or unset or set to "job"**: Preserves the original `job` label from Docker labels or task definitions
+- **Set to custom name**: Renames the `job` label to the specified name (e.g., `prometheus_job`)
+
+This renaming works around Prometheus receiver limitations where the `job` label conflicts with internal processing.
+
 ### Additional Labels
 
 Additional information from ECS and EC2.

extension/observer/ecsobserver/sd_test.go

@@ -173,6 +173,41 @@ func TestNewDiscovery(t *testing.T) {
 		assert.Equal(t, string(expectedContent), string(mustReadFile(t, outputFile)))
 	})
 
+	t.Run("job_label_name_empty_string", func(t *testing.T) {
+		cfg2 := cfg
+		cfg2.JobLabelName = ""
+
+		sd, err := newDiscovery(cfg2, opts)
+		require.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), cfg2.RefreshInterval*2)
+		defer cancel()
+		err = sd.runAndWriteFile(ctx)
+		require.NoError(t, err)
+
+		assert.FileExists(t, outputFile)
+		expectedFile := "testdata/ut_targets_expected_no_job_relabel.yaml"
+		expectedContent := bytes.ReplaceAll(mustReadFile(t, expectedFile), []byte("\r\n"), []byte("\n"))
+		assert.Equal(t, string(expectedContent), string(mustReadFile(t, outputFile)))
+	})
+
+	t.Run("job_label_name_is_job", func(t *testing.T) {
+		cfg2 := cfg
+		cfg2.JobLabelName = "job"
+		sd, err := newDiscovery(cfg2, opts)
+		require.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), cfg2.RefreshInterval*2)
+		defer cancel()
+		err = sd.runAndWriteFile(ctx)
+		require.NoError(t, err)
+
+		assert.FileExists(t, outputFile)
+		expectedFile := "testdata/ut_targets_expected_no_job_relabel.yaml"
+		expectedContent := bytes.ReplaceAll(mustReadFile(t, expectedFile), []byte("\r\n"), []byte("\n"))
+		assert.Equal(t, string(expectedContent), string(mustReadFile(t, outputFile)))
+	})
+
 	t.Run("fail to write file", func(t *testing.T) {
 		cfg2 := cfg
 		cfg2.ResultFile = "testdata/folder/does/not/exists/ut_targets.yaml"

extension/observer/ecsobserver/target.go

@@ -158,7 +158,7 @@ func targetsToFileSDTargets(targets []prometheusECSTarget, jobLabelName string)
 		// We can't relabel it using prometheus's relabel config as it would cause the same problem on receiver.
 		// We 'relabel' it to job outside prometheus receiver using other processors in collector's pipeline.
 		job := labels[labelJob]
-		if job != "" && jobLabelName != labelJob {
+		if job != "" && jobLabelName != "" && jobLabelName != labelJob {
 			delete(labels, labelJob)
 			labels[jobLabelName] = job
 		}

extension/observer/ecsobserver/target_test.go

@@ -22,3 +22,58 @@ func TestTargetToLabels(t *testing.T) {
 		assert.Equal(t, "same", m["__meta_ecs_task_tags_ab"])
 	})
 }
+
+func TestTargetsToFileSDTargets(t *testing.T) {
+	targets := []prometheusECSTarget{
+		{
+			Address: "192.168.1.1:9090",
+			Job:     "test-job",
+		},
+	}
+
+	t.Run("job label renamed when custom job_label_name provided", func(t *testing.T) {
+		result, err := targetsToFileSDTargets(targets, "prometheus_job")
+		assert.NoError(t, err)
+		assert.Len(t, result, 1)
+
+		labels := result[0].Labels
+		assert.Equal(t, "test-job", labels["prometheus_job"])
+		assert.NotContains(t, labels, "job")
+	})
+
+	t.Run("job label kept when job_label_name is 'job'", func(t *testing.T) {
+		result, err := targetsToFileSDTargets(targets, "job")
+		assert.NoError(t, err)
+		assert.Len(t, result, 1)
+
+		labels := result[0].Labels
+		assert.Equal(t, "test-job", labels["job"])
+		assert.NotContains(t, labels, "prometheus_job")
+	})
+
+	t.Run("job label kept when job_label_name is empty string", func(t *testing.T) {
+		result, err := targetsToFileSDTargets(targets, "")
+		assert.NoError(t, err)
+		assert.Len(t, result, 1)
+
+		labels := result[0].Labels
+		assert.Equal(t, "test-job", labels["job"])
+		assert.NotContains(t, labels, "")
+	})
+
+	t.Run("no job relabeling when job is empty", func(t *testing.T) {
+		emptyJobTargets := []prometheusECSTarget{
+			{
+				Address: "192.168.1.1:9090",
+				Job:     "",
+			},
+		}
+		result, err := targetsToFileSDTargets(emptyJobTargets, "prometheus_job")
+		assert.NoError(t, err)
+		assert.Len(t, result, 1)
+
+		labels := result[0].Labels
+		assert.NotContains(t, labels, "job")
+		assert.NotContains(t, labels, "prometheus_job")
+	})
+}