Skip to content

Commit f231dd8

Browse files
yeya24yeya24
committed
add new histogram field for full native histogram representation
Signed-off-by: Ben Ye <[email protected]>
1 parent bc69e73 commit f231dd8

File tree

6 files changed

+309
-125
lines changed

6 files changed

+309
-125
lines changed

‎pkg/querier/tripperware/merge.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,18 @@ func mergeSampleStreams(output map[string]SampleStream, sampleStreams []SampleSt
364364
stream.Histograms = sliceHistograms(stream.Histograms, existingEndTs)
365365
}
366366
}
367+
// Same for above.
368+
if len(existing.RawHistograms) > 0 && len(stream.RawHistograms) > 0 {
369+
existingEndTs := existing.RawHistograms[len(existing.RawHistograms)-1].GetTimestampMs()
370+
if existingEndTs == stream.RawHistograms[0].GetTimestampMs() {
371+
stream.RawHistograms = stream.RawHistograms[1:]
372+
} else if existingEndTs > stream.RawHistograms[0].GetTimestampMs() {
373+
stream.RawHistograms = sliceRawHistograms(stream.RawHistograms, existingEndTs)
374+
}
375+
}
367376
existing.Samples = append(existing.Samples, stream.Samples...)
368377
existing.Histograms = append(existing.Histograms, stream.Histograms...)
378+
existing.RawHistograms = append(existing.RawHistograms, stream.RawHistograms...)
369379

370380
output[metric] = existing
371381
}
@@ -396,7 +406,27 @@ func sliceSamples(samples []cortexpb.Sample, minTs int64) []cortexpb.Sample {
396406
// return a sub slice whose first element's is the smallest timestamp that is strictly
397407
// bigger than the given minTs. Empty slice is returned if minTs is bigger than all the
398408
// timestamps in histogram.
399-
func sliceHistograms(histograms []SampleHistogramPair, minTs int64) []SampleHistogramPair {
409+
func sliceHistograms(histograms []*SampleHistogramPair, minTs int64) []*SampleHistogramPair {
410+
if len(histograms) <= 0 || minTs < histograms[0].GetTimestampMs() {
411+
return histograms
412+
}
413+
414+
if len(histograms) > 0 && minTs > histograms[len(histograms)-1].GetTimestampMs() {
415+
return histograms[len(histograms):]
416+
}
417+
418+
searchResult := sort.Search(len(histograms), func(i int) bool {
419+
return histograms[i].GetTimestampMs() > minTs
420+
})
421+
422+
return histograms[searchResult:]
423+
}
424+
425+
// sliceRawHistograms assumes given histogram are sorted by timestamp in ascending order and
426+
// return a sub slice whose first element's is the smallest timestamp that is strictly
427+
// bigger than the given minTs. Empty slice is returned if minTs is bigger than all the
428+
// timestamps in histogram.
429+
func sliceRawHistograms(histograms []*cortexpb.Histogram, minTs int64) []*cortexpb.Histogram {
400430
if len(histograms) <= 0 || minTs < histograms[0].GetTimestampMs() {
401431
return histograms
402432
}

‎pkg/querier/tripperware/merge_test.go‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -99,15 +99,15 @@ func TestMergeSampleStreams(t *testing.T) {
9999
sampleStreams: []SampleStream{
100100
{
101101
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
102-
Histograms: []SampleHistogramPair{
102+
Histograms: []*SampleHistogramPair{
103103
{Histogram: testHistogram1, TimestampMs: 0},
104104
},
105105
},
106106
},
107107
expectedOutput: map[string]SampleStream{
108108
ingester_client.LabelsToKeyString(lbls): {
109109
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
110-
Histograms: []SampleHistogramPair{
110+
Histograms: []*SampleHistogramPair{
111111
{Histogram: testHistogram1, TimestampMs: 0},
112112
},
113113
},
@@ -121,7 +121,7 @@ func TestMergeSampleStreams(t *testing.T) {
121121
Samples: []cortexpb.Sample{
122122
{Value: 0, TimestampMs: 0},
123123
},
124-
Histograms: []SampleHistogramPair{
124+
Histograms: []*SampleHistogramPair{
125125
{Histogram: testHistogram1, TimestampMs: 0},
126126
},
127127
},
@@ -132,7 +132,7 @@ func TestMergeSampleStreams(t *testing.T) {
132132
Samples: []cortexpb.Sample{
133133
{Value: 0, TimestampMs: 0},
134134
},
135-
Histograms: []SampleHistogramPair{
135+
Histograms: []*SampleHistogramPair{
136136
{Histogram: testHistogram1, TimestampMs: 0},
137137
},
138138
},
@@ -175,15 +175,15 @@ func TestMergeSampleStreams(t *testing.T) {
175175
sampleStreams: []SampleStream{
176176
{
177177
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
178-
Histograms: []SampleHistogramPair{
178+
Histograms: []*SampleHistogramPair{
179179
{Histogram: testHistogram1, TimestampMs: 0},
180180
{Histogram: testHistogram1, TimestampMs: 2},
181181
{Histogram: testHistogram1, TimestampMs: 3},
182182
},
183183
},
184184
{
185185
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
186-
Histograms: []SampleHistogramPair{
186+
Histograms: []*SampleHistogramPair{
187187
{Histogram: testHistogram1, TimestampMs: 0},
188188
{Histogram: testHistogram1, TimestampMs: 1},
189189
{Histogram: testHistogram1, TimestampMs: 4},
@@ -193,7 +193,7 @@ func TestMergeSampleStreams(t *testing.T) {
193193
expectedOutput: map[string]SampleStream{
194194
ingester_client.LabelsToKeyString(lbls): {
195195
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
196-
Histograms: []SampleHistogramPair{
196+
Histograms: []*SampleHistogramPair{
197197
{Histogram: testHistogram1, TimestampMs: 0},
198198
{Histogram: testHistogram1, TimestampMs: 2},
199199
{Histogram: testHistogram1, TimestampMs: 3},
@@ -212,7 +212,7 @@ func TestMergeSampleStreams(t *testing.T) {
212212
{Value: 2, TimestampMs: 2},
213213
{Value: 3, TimestampMs: 3},
214214
},
215-
Histograms: []SampleHistogramPair{
215+
Histograms: []*SampleHistogramPair{
216216
{Histogram: testHistogram1, TimestampMs: 0},
217217
{Histogram: testHistogram1, TimestampMs: 2},
218218
{Histogram: testHistogram1, TimestampMs: 3},
@@ -225,7 +225,7 @@ func TestMergeSampleStreams(t *testing.T) {
225225
{Value: 1, TimestampMs: 1},
226226
{Value: 4, TimestampMs: 4},
227227
},
228-
Histograms: []SampleHistogramPair{
228+
Histograms: []*SampleHistogramPair{
229229
{Histogram: testHistogram1, TimestampMs: 0},
230230
{Histogram: testHistogram1, TimestampMs: 1},
231231
{Histogram: testHistogram1, TimestampMs: 4},
@@ -241,7 +241,7 @@ func TestMergeSampleStreams(t *testing.T) {
241241
{Value: 3, TimestampMs: 3},
242242
{Value: 4, TimestampMs: 4},
243243
},
244-
Histograms: []SampleHistogramPair{
244+
Histograms: []*SampleHistogramPair{
245245
{Histogram: testHistogram1, TimestampMs: 0},
246246
{Histogram: testHistogram1, TimestampMs: 2},
247247
{Histogram: testHistogram1, TimestampMs: 3},
@@ -309,30 +309,30 @@ func TestMergeSampleStreams(t *testing.T) {
309309
sampleStreams: []SampleStream{
310310
{
311311
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
312-
Histograms: []SampleHistogramPair{
312+
Histograms: []*SampleHistogramPair{
313313
{Histogram: testHistogram1, TimestampMs: 0},
314314
{Histogram: testHistogram1, TimestampMs: 2},
315315
{Histogram: testHistogram1, TimestampMs: 3},
316316
},
317317
},
318318
{
319319
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
320-
Histograms: []SampleHistogramPair{
320+
Histograms: []*SampleHistogramPair{
321321
{Histogram: testHistogram2, TimestampMs: 1},
322322
{Histogram: testHistogram2, TimestampMs: 4},
323323
},
324324
},
325325
{
326326
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
327-
Histograms: []SampleHistogramPair{
327+
Histograms: []*SampleHistogramPair{
328328
{Histogram: testHistogram1, TimestampMs: 0},
329329
{Histogram: testHistogram1, TimestampMs: 1},
330330
{Histogram: testHistogram1, TimestampMs: 4},
331331
},
332332
},
333333
{
334334
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
335-
Histograms: []SampleHistogramPair{
335+
Histograms: []*SampleHistogramPair{
336336
{Histogram: testHistogram2, TimestampMs: 2},
337337
{Histogram: testHistogram2, TimestampMs: 3},
338338
},
@@ -341,7 +341,7 @@ func TestMergeSampleStreams(t *testing.T) {
341341
expectedOutput: map[string]SampleStream{
342342
ingester_client.LabelsToKeyString(lbls): {
343343
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
344-
Histograms: []SampleHistogramPair{
344+
Histograms: []*SampleHistogramPair{
345345
{Histogram: testHistogram1, TimestampMs: 0},
346346
{Histogram: testHistogram1, TimestampMs: 2},
347347
{Histogram: testHistogram1, TimestampMs: 3},
@@ -350,7 +350,7 @@ func TestMergeSampleStreams(t *testing.T) {
350350
},
351351
ingester_client.LabelsToKeyString(lbls1): {
352352
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
353-
Histograms: []SampleHistogramPair{
353+
Histograms: []*SampleHistogramPair{
354354
{Histogram: testHistogram1, TimestampMs: 0},
355355
{Histogram: testHistogram1, TimestampMs: 1},
356356
{Histogram: testHistogram1, TimestampMs: 4},
@@ -452,21 +452,21 @@ func TestSliceHistograms(t *testing.T) {
452452
t.Parallel()
453453
for _, tc := range []struct {
454454
name string
455-
histograms []SampleHistogramPair
455+
histograms []*SampleHistogramPair
456456
minTs int64
457-
expectedHistograms []SampleHistogramPair
457+
expectedHistograms []*SampleHistogramPair
458458
}{
459459
{name: "empty histograms"},
460460
{
461461
name: "minTs smaller than first histogram's timestamp",
462-
histograms: []SampleHistogramPair{
462+
histograms: []*SampleHistogramPair{
463463
{
464464
TimestampMs: 1,
465465
Histogram: testHistogram1,
466466
},
467467
},
468468
minTs: 0,
469-
expectedHistograms: []SampleHistogramPair{
469+
expectedHistograms: []*SampleHistogramPair{
470470
{
471471
TimestampMs: 1,
472472
Histogram: testHistogram1,
@@ -475,7 +475,7 @@ func TestSliceHistograms(t *testing.T) {
475475
},
476476
{
477477
name: "input histograms are not sorted, return all histograms",
478-
histograms: []SampleHistogramPair{
478+
histograms: []*SampleHistogramPair{
479479
{
480480
TimestampMs: 3,
481481
Histogram: testHistogram1,
@@ -486,7 +486,7 @@ func TestSliceHistograms(t *testing.T) {
486486
},
487487
},
488488
minTs: 2,
489-
expectedHistograms: []SampleHistogramPair{
489+
expectedHistograms: []*SampleHistogramPair{
490490
{
491491
TimestampMs: 3,
492492
Histogram: testHistogram1,
@@ -499,7 +499,7 @@ func TestSliceHistograms(t *testing.T) {
499499
},
500500
{
501501
name: "minTs greater than the last histogram's timestamp",
502-
histograms: []SampleHistogramPair{
502+
histograms: []*SampleHistogramPair{
503503
{
504504
TimestampMs: 1,
505505
Histogram: testHistogram1,
@@ -510,11 +510,11 @@ func TestSliceHistograms(t *testing.T) {
510510
},
511511
},
512512
minTs: 3,
513-
expectedHistograms: []SampleHistogramPair{},
513+
expectedHistograms: []*SampleHistogramPair{},
514514
},
515515
{
516516
name: "input histograms not sorted, minTs greater than the last histogram's timestamp",
517-
histograms: []SampleHistogramPair{
517+
histograms: []*SampleHistogramPair{
518518
{
519519
TimestampMs: 0,
520520
Histogram: testHistogram1,
@@ -529,11 +529,11 @@ func TestSliceHistograms(t *testing.T) {
529529
},
530530
},
531531
minTs: 2,
532-
expectedHistograms: []SampleHistogramPair{},
532+
expectedHistograms: []*SampleHistogramPair{},
533533
},
534534
{
535535
name: "input histograms are sorted",
536-
histograms: []SampleHistogramPair{
536+
histograms: []*SampleHistogramPair{
537537
{
538538
TimestampMs: 2,
539539
Histogram: testHistogram1,
@@ -548,7 +548,7 @@ func TestSliceHistograms(t *testing.T) {
548548
},
549549
},
550550
minTs: 1,
551-
expectedHistograms: []SampleHistogramPair{
551+
expectedHistograms: []*SampleHistogramPair{
552552
{
553553
TimestampMs: 2,
554554
Histogram: testHistogram1,
@@ -565,7 +565,7 @@ func TestSliceHistograms(t *testing.T) {
565565
},
566566
{
567567
name: "input histograms are sorted, get sliced histograms",
568-
histograms: []SampleHistogramPair{
568+
histograms: []*SampleHistogramPair{
569569
{
570570
TimestampMs: 1,
571571
Histogram: testHistogram1,
@@ -580,7 +580,7 @@ func TestSliceHistograms(t *testing.T) {
580580
},
581581
},
582582
minTs: 2,
583-
expectedHistograms: []SampleHistogramPair{
583+
expectedHistograms: []*SampleHistogramPair{
584584
{
585585
TimestampMs: 3,
586586
Histogram: testHistogram1,

‎pkg/querier/tripperware/query.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ func decodeSampleStream(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
109109
for iter.ReadArray() {
110110
h := SampleHistogramPair{}
111111
UnmarshalSampleHistogramPairJSON(unsafe.Pointer(&h), iter)
112-
ss.Histograms = append(ss.Histograms, h)
112+
ss.Histograms = append(ss.Histograms, &h)
113113
}
114114
default:
115115
iter.ReportError("unmarshal SampleStream", fmt.Sprint("unexpected key:", field))
@@ -309,7 +309,7 @@ func encodeSampleStream(ptr unsafe.Pointer, stream *jsoniter.Stream) {
309309
if i > 0 {
310310
stream.WriteMore()
311311
}
312-
MarshalSampleHistogramPairJSON(unsafe.Pointer(&h), stream)
312+
MarshalSampleHistogramPairJSON(unsafe.Pointer(h), stream)
313313
}
314314
stream.WriteArrayEnd()
315315
}

0 commit comments

Comments
 (0)