[receiver/prometheusreceiver] fix stalness tracking (open-telemetry#43925)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

In the dependency upgrade open-telemetry#43890 there was the PR
prometheus/prometheus#16429 which changed the
provisions for staleness tracking. Now the code only does the tracking
if the series was successfully appended in storage. This is indicated by
a non zero storage reference returned by the appender.

Since we used to return 0 in all cases, the staleness tracking is now
broken.

Solution is to return a fake reference of 1, just to indicate success.

Alternative solution would be to make a unique reference for each series
label set, but that incurs an overhead which we can avoid.

Performance impact: probably small as the cache should be stable over
time - unless targets churn a lot between receiver instances.

Resource utilization: +1 hashmap with series identifier (name+labels)
pointing to cache entries. Size related to number of series. `addRef`
here:
https://github.com/prometheus/prometheus/blob/c8f1de18a7870e5dc99ed50119c194691c41e41c/scrape/scrape.go#L1089

<!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes open-telemetry#43893

<!--Describe what testing was performed and which tests were added.-->
#### Testing

Un-skip related tests.

---------

Signed-off-by: György Krajcsovits <[email protected]>

Author: krajorama

.chloggen/prometheus-receiver-fix-stalenss.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/filelog)
+component: receiver/prometheus
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fix missing staleness tracking leading to missing no recorded value data points.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [43893]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/prometheusexporter/end_to_end_test.go

@@ -138,7 +138,7 @@ func TestEndToEndSummarySupport(t *testing.T) {
 		`test_scrape_samples_scraped.instance="127.0.0.1:.*",job="otel-collector",otel_scope_name=\"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver\",otel_scope_schema_url=\"\",otel_scope_version=\"latest\". 13 .*`,
 		`. HELP test_scrape_series_added The approximate number of new series in this scrape`,
 		`. TYPE test_scrape_series_added gauge`,
-		`test_scrape_series_added.instance="127.0.0.1:.*",job="otel-collector",otel_scope_name=\"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver\",otel_scope_schema_url=\"\",otel_scope_version=\"latest\". 13 .*`,
+		`test_scrape_series_added.instance="127.0.0.1:.*",job="otel-collector",otel_scope_name=\"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver\",otel_scope_schema_url=\"\",otel_scope_version=\"latest\". (0|13) .*`,
 		`. HELP test_up The scraping was successful`,
 		`. TYPE test_up gauge`,
 		`test_up.instance="127.0.0.1:.*",job="otel-collector",otel_scope_name=\"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver\",otel_scope_schema_url=\"\",otel_scope_version=\"latest\". 1 .*`,

receiver/prometheusreceiver/internal/staleness_end_to_end_test.go

@@ -43,7 +43,6 @@ import (
 // Prometheus remotewrite exporter that staleness markers are emitted per timeseries.
 // See https://github.com/open-telemetry/opentelemetry-collector/issues/3413
 func TestStalenessMarkersEndToEnd(t *testing.T) {
-	t.Skip("Skipping test until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/43893 is resolved")
 	if testing.Short() {
 		t.Skip("This test can take a long time")
 	}

receiver/prometheusreceiver/internal/transaction.go

@@ -187,9 +187,14 @@ func (t *transaction) Append(_ storage.SeriesRef, ls labels.Labels, atMs int64,
 	err = curMF.addSeries(seriesRef, metricName, ls, atMs, val)
 	if err != nil {
 		t.logger.Warn("failed to add datapoint", zap.Error(err), zap.String("metric_name", metricName), zap.Any("labels", ls))
+		// never return errors, as that fails the while scrape
+		// return ref==0 indicating that the series was not added
+		return 0, nil
 	}
 
-	return 0, nil // never return errors, as that fails the whole scrape
+	// never return errors, as that fails the whole scrape
+	// return ref==1 indicating that the series was added and needs staleness tracking
+	return 1, nil
 }
 
 // detectAndStoreNativeHistogramStaleness returns true if it detects
@@ -350,9 +355,14 @@ func (t *transaction) AppendHistogram(_ storage.SeriesRef, ls labels.Labels, atM
 	}
 	if err != nil {
 		t.logger.Warn("failed to add histogram datapoint", zap.Error(err), zap.String("metric_name", metricName), zap.Any("labels", ls))
+		// never return errors, as that fails the while scrape
+		// return ref==0 indicating that the series was not added
+		return 0, nil
 	}
 
-	return 0, nil // never return errors, as that fails the whole scrape
+	// never return errors, as that fails the whole scrape
+	// return ref==1 indicating that the series was added and needs staleness tracking
+	return 1, nil
 }
 
 func (t *transaction) AppendCTZeroSample(_ storage.SeriesRef, ls labels.Labels, atMs, ctMs int64) (storage.SeriesRef, error) {

receiver/prometheusreceiver/metrics_receiver_non_numerical_test.go

@@ -46,7 +46,6 @@ var totalScrapes = 10
 
 // TestStaleNaNs validates that staleness marker gets generated when the timeseries is no longer present
 func TestStaleNaNs(t *testing.T) {
-	t.Skip("Skipping test until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/43893 is resolved")
 	var mockResponses []mockPrometheusResponse
 	for i := range totalScrapes {
 		if i%2 == 0 {

receiver/prometheusreceiver/metrics_receiver_protobuf_test.go

@@ -529,7 +529,6 @@ func TestNativeVsClassicHistogramScrapeViaProtobuf(t *testing.T) {
 }
 
 func TestStaleExponentialHistogram(t *testing.T) {
-	t.Skip("Skipping test until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/43893 is resolved")
 	mf := &dto.MetricFamily{
 		Name: "test_counter",
 		Type: dto.MetricType_COUNTER,