Skip to content

Commit 1cdc956

Browse files
krajoramakrajorama
committed
test(rw2): add nhcb testcases to remote write 2.0
Ref: prometheus#15021 Also modified spansToSpansProto to not allocate empty bucket spans array when converting internal model to remote write model. Otherwise the test TestDecodeWriteV2Request fails since empty array is marshaled/unmarshaled as nil so we don't get back the exact same thing. Signed-off-by: György Krajcsovits <gyorgy.krajcsovits@grafana.com>
1 parent 10b4e1b commit 1cdc956

File tree

3 files changed

+79
-18
lines changed

3 files changed

+79
-18
lines changed

‎prompb/io/prometheus/write/v2/codec.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,9 @@ func FromFloatHistogram(timestamp int64, fh *histogram.FloatHistogram) Histogram
196196
}
197197

198198
func spansToSpansProto(s []histogram.Span) []BucketSpan {
199+
if len(s) == 0 {
200+
return nil
201+
}
199202
spans := make([]BucketSpan, len(s))
200203
for i := 0; i < len(s); i++ {
201204
spans[i] = BucketSpan{Offset: s[i].Offset, Length: s[i].Length}

‎storage/remote/codec_test.go‎

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,12 @@ var (
4343
Schema: 2,
4444
ZeroThreshold: 1e-128,
4545
ZeroCount: 0,
46-
Count: 0,
46+
Count: 3,
4747
Sum: 20,
4848
PositiveSpans: []histogram.Span{{Offset: 0, Length: 1}},
4949
PositiveBuckets: []int64{1},
5050
NegativeSpans: []histogram.Span{{Offset: 0, Length: 1}},
51-
NegativeBuckets: []int64{-1},
51+
NegativeBuckets: []int64{2},
5252
}
5353

5454
writeRequestFixture = &prompb.WriteRequest{
@@ -90,6 +90,15 @@ var (
9090
Help: "Test counter for test purposes",
9191
}
9292

93+
testHistogramCustomBuckets = histogram.Histogram{
94+
Schema: histogram.CustomBucketsSchema,
95+
Count: 16,
96+
Sum: 20,
97+
PositiveSpans: []histogram.Span{{Offset: 1, Length: 2}},
98+
PositiveBuckets: []int64{10, -4}, // Means 10 observations for upper bound 1.0 and 6 for upper bound +Inf.
99+
CustomValues: []float64{0.1, 1.0}, // +Inf is implied.
100+
}
101+
93102
// writeV2RequestFixture represents the same request as writeRequestFixture,
94103
// but using the v2 representation, plus includes writeV2RequestSeries1Metadata and writeV2RequestSeries2Metadata.
95104
// NOTE: Use TestWriteV2RequestFixture and copy the diff to regenerate if needed.
@@ -104,9 +113,14 @@ var (
104113
HelpRef: 15, // Symbolized writeV2RequestSeries1Metadata.Help.
105114
UnitRef: 16, // Symbolized writeV2RequestSeries1Metadata.Unit.
106115
},
107-
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
108-
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}},
109-
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))},
116+
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
117+
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{11, 12}, Value: 1, Timestamp: 10}},
118+
Histograms: []writev2.Histogram{
119+
writev2.FromIntHistogram(10, &testHistogram),
120+
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
121+
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
122+
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
123+
},
110124
CreatedTimestamp: 1, // CT needs to be lower than the sample's timestamp.
111125
},
112126
{
@@ -117,14 +131,40 @@ var (
117131
HelpRef: 17, // Symbolized writeV2RequestSeries2Metadata.Help.
118132
// No unit.
119133
},
120-
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
121-
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}},
122-
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))},
134+
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
135+
Exemplars: []writev2.Exemplar{{LabelsRefs: []uint32{13, 14}, Value: 2, Timestamp: 20}},
136+
Histograms: []writev2.Histogram{
137+
writev2.FromIntHistogram(50, &testHistogram),
138+
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
139+
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
140+
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
141+
},
123142
},
124143
},
125144
}
126145
)
127146

147+
func TestHistogramFixtureValid(t *testing.T) {
148+
for _, ts := range writeRequestFixture.Timeseries {
149+
for _, h := range ts.Histograms {
150+
if h.IsFloatHistogram() {
151+
require.NoError(t, h.ToFloatHistogram().Validate())
152+
} else {
153+
require.NoError(t, h.ToIntHistogram().Validate())
154+
}
155+
}
156+
}
157+
for _, ts := range writeV2RequestFixture.Timeseries {
158+
for _, h := range ts.Histograms {
159+
if h.IsFloatHistogram() {
160+
require.NoError(t, h.ToFloatHistogram().Validate())
161+
} else {
162+
require.NoError(t, h.ToIntHistogram().Validate())
163+
}
164+
}
165+
}
166+
}
167+
128168
func TestWriteV2RequestFixture(t *testing.T) {
129169
// Generate dynamically writeV2RequestFixture, reusing v1 fixture elements.
130170
st := writev2.NewSymbolTable()
@@ -141,9 +181,14 @@ func TestWriteV2RequestFixture(t *testing.T) {
141181
HelpRef: st.Symbolize(writeV2RequestSeries1Metadata.Help),
142182
UnitRef: st.Symbolize(writeV2RequestSeries1Metadata.Unit),
143183
},
144-
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
145-
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}},
146-
Histograms: []writev2.Histogram{writev2.FromIntHistogram(10, &testHistogram), writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil))},
184+
Samples: []writev2.Sample{{Value: 1, Timestamp: 10}},
185+
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar1LabelRefs, Value: 1, Timestamp: 10}},
186+
Histograms: []writev2.Histogram{
187+
writev2.FromIntHistogram(10, &testHistogram),
188+
writev2.FromFloatHistogram(20, testHistogram.ToFloat(nil)),
189+
writev2.FromIntHistogram(30, &testHistogramCustomBuckets),
190+
writev2.FromFloatHistogram(40, testHistogramCustomBuckets.ToFloat(nil)),
191+
},
147192
CreatedTimestamp: 1,
148193
},
149194
{
@@ -153,9 +198,14 @@ func TestWriteV2RequestFixture(t *testing.T) {
153198
HelpRef: st.Symbolize(writeV2RequestSeries2Metadata.Help),
154199
// No unit.
155200
},
156-
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
157-
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}},
158-
Histograms: []writev2.Histogram{writev2.FromIntHistogram(30, &testHistogram), writev2.FromFloatHistogram(40, testHistogram.ToFloat(nil))},
201+
Samples: []writev2.Sample{{Value: 2, Timestamp: 20}},
202+
Exemplars: []writev2.Exemplar{{LabelsRefs: exemplar2LabelRefs, Value: 2, Timestamp: 20}},
203+
Histograms: []writev2.Histogram{
204+
writev2.FromIntHistogram(50, &testHistogram),
205+
writev2.FromFloatHistogram(60, testHistogram.ToFloat(nil)),
206+
writev2.FromIntHistogram(70, &testHistogramCustomBuckets),
207+
writev2.FromFloatHistogram(80, testHistogramCustomBuckets.ToFloat(nil)),
208+
},
159209
},
160210
},
161211
Symbols: st.Symbols(),

‎storage/remote/write_handler_test.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
391391
desc: "Partial write; first series with one dup histogram sample",
392392
input: func() []writev2.TimeSeries {
393393
f := proto.Clone(writeV2RequestFixture).(*writev2.Request)
394-
f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[1])
394+
f.Timeseries[0].Histograms = append(f.Timeseries[0].Histograms, f.Timeseries[0].Histograms[len(f.Timeseries[0].Histograms)-1])
395395
return f.Timeseries
396396
}(),
397397
expectedCode: http.StatusBadRequest,
@@ -483,7 +483,7 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
483483
// Double check mandatory 2.0 stats.
484484
// writeV2RequestFixture has 2 series with 1 sample, 2 histograms, 1 exemplar each.
485485
expectHeaderValue(t, 2, resp.Header.Get(rw20WrittenSamplesHeader))
486-
expectHeaderValue(t, 4, resp.Header.Get(rw20WrittenHistogramsHeader))
486+
expectHeaderValue(t, 8, resp.Header.Get(rw20WrittenHistogramsHeader))
487487
if tc.appendExemplarErr != nil {
488488
expectHeaderValue(t, 0, resp.Header.Get(rw20WrittenExemplarsHeader))
489489
} else {
@@ -496,6 +496,8 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
496496
i, j, k, m int
497497
)
498498
for _, ts := range writeV2RequestFixture.Timeseries {
499+
zeroHistogramIngested := false
500+
zeroFloatHistogramIngested := false
499501
ls := ts.ToLabels(&b, writeV2RequestFixture.Symbols)
500502

501503
for _, s := range ts.Samples {
@@ -509,21 +511,27 @@ func TestRemoteWriteHandler_V2Message(t *testing.T) {
509511
for _, hp := range ts.Histograms {
510512
if hp.IsFloatHistogram() {
511513
fh := hp.ToFloatHistogram()
512-
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
514+
if !zeroFloatHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
513515
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, nil, &histogram.FloatHistogram{}}, appendable.histograms[k])
514516
k++
517+
zeroFloatHistogramIngested = true
515518
}
516519
requireEqual(t, mockHistogram{ls, hp.Timestamp, nil, fh}, appendable.histograms[k])
517520
} else {
518521
h := hp.ToIntHistogram()
519-
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
522+
if !zeroHistogramIngested && ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
520523
requireEqual(t, mockHistogram{ls, ts.CreatedTimestamp, &histogram.Histogram{}, nil}, appendable.histograms[k])
521524
k++
525+
zeroHistogramIngested = true
522526
}
523527
requireEqual(t, mockHistogram{ls, hp.Timestamp, h, nil}, appendable.histograms[k])
524528
}
525529
k++
526530
}
531+
if ts.CreatedTimestamp != 0 && tc.ingestCTZeroSample {
532+
require.True(t, zeroHistogramIngested)
533+
require.True(t, zeroFloatHistogramIngested)
534+
}
527535
if tc.appendExemplarErr == nil {
528536
for _, e := range ts.Exemplars {
529537
exemplarLabels := e.ToExemplar(&b, writeV2RequestFixture.Symbols).Labels

0 commit comments

Comments
 (0)