Remove job ID from bytes_scanned and chunks_scanned metrics (#4206)

We have several metrics that have captured job ID as a dimension. This is, in a strict sense, "wrong" because the cardinality is unbounded, but our job counts have been low enough that we've been able to get away with it. However, recent exploration of a new distributed job technique has caused job counts to substantially increase, and this extra, "incorrect" dimension is now harder to justify.

We have been recently seeing some trouble with some of our Prometheus scrapes, and while I haven't drawn a direct connection between that trouble and the increased cardinality, I do know that we don't use the job ID dimension anywhere. We always sum it away! So this commit removes it. We're shouldn't be doing it, we're not using it, and it might be causing problems we're seeing.

Author: rosecodym

pkg/engine/engine.go

@@ -820,13 +820,11 @@ func (e *Engine) scannerWorker(ctx context.Context) {
 
 		scanBytesPerChunk.Observe(dataSize)
 		jobBytesScanned.WithLabelValues(
-			strconv.Itoa(int(chunk.JobID)),
 			chunk.SourceType.String(),
 			chunk.SourceName,
 		).Add(dataSize)
 		chunksScannedLatency.Observe(float64(time.Since(startTime).Microseconds()))
 		jobChunksScanned.WithLabelValues(
-			strconv.Itoa(int(chunk.JobID)),
 			chunk.SourceType.String(),
 			chunk.SourceName,
 		).Inc()

pkg/engine/metrics.go

@@ -52,7 +52,7 @@ var (
 		Name:      "job_bytes_scanned",
 		Help:      "Total number of bytes scanned for a job.",
 	},
-		[]string{"job_id", "source_type", "source_name"},
+		[]string{"source_type", "source_name"},
 	)
 
 	scanBytesPerChunk = promauto.NewHistogram(prometheus.HistogramOpts{
@@ -69,7 +69,7 @@ var (
 		Name:      "job_chunks_scanned",
 		Help:      "Total number of chunks scanned for a job.",
 	},
-		[]string{"job_id", "source_type", "source_name"},
+		[]string{"source_type", "source_name"},
 	)
 
 	detectBytesPerMatch = promauto.NewHistogram(prometheus.HistogramOpts{