Skip to content

Commit a153624

Browse files
yeya24yeya24
committed
add new histogram field for full native histogram representation
Signed-off-by: Ben Ye <[email protected]>
1 parent bc1b5be commit a153624

File tree

6 files changed

+309
-126
lines changed

6 files changed

+309
-126
lines changed

‎pkg/querier/tripperware/merge.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,18 @@ func mergeSampleStreams(output map[string]SampleStream, sampleStreams []SampleSt
372372
stream.Histograms = sliceHistograms(stream.Histograms, existingEndTs)
373373
}
374374
}
375+
// Same for above.
376+
if len(existing.RawHistograms) > 0 && len(stream.RawHistograms) > 0 {
377+
existingEndTs := existing.RawHistograms[len(existing.RawHistograms)-1].GetTimestampMs()
378+
if existingEndTs == stream.RawHistograms[0].GetTimestampMs() {
379+
stream.RawHistograms = stream.RawHistograms[1:]
380+
} else if existingEndTs > stream.RawHistograms[0].GetTimestampMs() {
381+
stream.RawHistograms = sliceRawHistograms(stream.RawHistograms, existingEndTs)
382+
}
383+
}
375384
existing.Samples = append(existing.Samples, stream.Samples...)
376385
existing.Histograms = append(existing.Histograms, stream.Histograms...)
386+
existing.RawHistograms = append(existing.RawHistograms, stream.RawHistograms...)
377387

378388
output[metric] = existing
379389
}
@@ -404,7 +414,27 @@ func sliceSamples(samples []cortexpb.Sample, minTs int64) []cortexpb.Sample {
404414
// return a sub slice whose first element's is the smallest timestamp that is strictly
405415
// bigger than the given minTs. Empty slice is returned if minTs is bigger than all the
406416
// timestamps in histogram.
407-
func sliceHistograms(histograms []SampleHistogramPair, minTs int64) []SampleHistogramPair {
417+
func sliceHistograms(histograms []*SampleHistogramPair, minTs int64) []*SampleHistogramPair {
418+
if len(histograms) <= 0 || minTs < histograms[0].GetTimestampMs() {
419+
return histograms
420+
}
421+
422+
if len(histograms) > 0 && minTs > histograms[len(histograms)-1].GetTimestampMs() {
423+
return histograms[len(histograms):]
424+
}
425+
426+
searchResult := sort.Search(len(histograms), func(i int) bool {
427+
return histograms[i].GetTimestampMs() > minTs
428+
})
429+
430+
return histograms[searchResult:]
431+
}
432+
433+
// sliceRawHistograms assumes given histogram are sorted by timestamp in ascending order and
434+
// return a sub slice whose first element's is the smallest timestamp that is strictly
435+
// bigger than the given minTs. Empty slice is returned if minTs is bigger than all the
436+
// timestamps in histogram.
437+
func sliceRawHistograms(histograms []*cortexpb.Histogram, minTs int64) []*cortexpb.Histogram {
408438
if len(histograms) <= 0 || minTs < histograms[0].GetTimestampMs() {
409439
return histograms
410440
}

‎pkg/querier/tripperware/merge_test.go‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -100,15 +100,15 @@ func TestMergeSampleStreams(t *testing.T) {
100100
sampleStreams: []SampleStream{
101101
{
102102
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
103-
Histograms: []SampleHistogramPair{
103+
Histograms: []*SampleHistogramPair{
104104
{Histogram: testHistogram1, TimestampMs: 0},
105105
},
106106
},
107107
},
108108
expectedOutput: map[string]SampleStream{
109109
ingester_client.LabelsToKeyString(lbls): {
110110
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
111-
Histograms: []SampleHistogramPair{
111+
Histograms: []*SampleHistogramPair{
112112
{Histogram: testHistogram1, TimestampMs: 0},
113113
},
114114
},
@@ -122,7 +122,7 @@ func TestMergeSampleStreams(t *testing.T) {
122122
Samples: []cortexpb.Sample{
123123
{Value: 0, TimestampMs: 0},
124124
},
125-
Histograms: []SampleHistogramPair{
125+
Histograms: []*SampleHistogramPair{
126126
{Histogram: testHistogram1, TimestampMs: 0},
127127
},
128128
},
@@ -133,7 +133,7 @@ func TestMergeSampleStreams(t *testing.T) {
133133
Samples: []cortexpb.Sample{
134134
{Value: 0, TimestampMs: 0},
135135
},
136-
Histograms: []SampleHistogramPair{
136+
Histograms: []*SampleHistogramPair{
137137
{Histogram: testHistogram1, TimestampMs: 0},
138138
},
139139
},
@@ -176,15 +176,15 @@ func TestMergeSampleStreams(t *testing.T) {
176176
sampleStreams: []SampleStream{
177177
{
178178
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
179-
Histograms: []SampleHistogramPair{
179+
Histograms: []*SampleHistogramPair{
180180
{Histogram: testHistogram1, TimestampMs: 0},
181181
{Histogram: testHistogram1, TimestampMs: 2},
182182
{Histogram: testHistogram1, TimestampMs: 3},
183183
},
184184
},
185185
{
186186
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
187-
Histograms: []SampleHistogramPair{
187+
Histograms: []*SampleHistogramPair{
188188
{Histogram: testHistogram1, TimestampMs: 0},
189189
{Histogram: testHistogram1, TimestampMs: 1},
190190
{Histogram: testHistogram1, TimestampMs: 4},
@@ -194,7 +194,7 @@ func TestMergeSampleStreams(t *testing.T) {
194194
expectedOutput: map[string]SampleStream{
195195
ingester_client.LabelsToKeyString(lbls): {
196196
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
197-
Histograms: []SampleHistogramPair{
197+
Histograms: []*SampleHistogramPair{
198198
{Histogram: testHistogram1, TimestampMs: 0},
199199
{Histogram: testHistogram1, TimestampMs: 2},
200200
{Histogram: testHistogram1, TimestampMs: 3},
@@ -213,7 +213,7 @@ func TestMergeSampleStreams(t *testing.T) {
213213
{Value: 2, TimestampMs: 2},
214214
{Value: 3, TimestampMs: 3},
215215
},
216-
Histograms: []SampleHistogramPair{
216+
Histograms: []*SampleHistogramPair{
217217
{Histogram: testHistogram1, TimestampMs: 0},
218218
{Histogram: testHistogram1, TimestampMs: 2},
219219
{Histogram: testHistogram1, TimestampMs: 3},
@@ -226,7 +226,7 @@ func TestMergeSampleStreams(t *testing.T) {
226226
{Value: 1, TimestampMs: 1},
227227
{Value: 4, TimestampMs: 4},
228228
},
229-
Histograms: []SampleHistogramPair{
229+
Histograms: []*SampleHistogramPair{
230230
{Histogram: testHistogram1, TimestampMs: 0},
231231
{Histogram: testHistogram1, TimestampMs: 1},
232232
{Histogram: testHistogram1, TimestampMs: 4},
@@ -242,7 +242,7 @@ func TestMergeSampleStreams(t *testing.T) {
242242
{Value: 3, TimestampMs: 3},
243243
{Value: 4, TimestampMs: 4},
244244
},
245-
Histograms: []SampleHistogramPair{
245+
Histograms: []*SampleHistogramPair{
246246
{Histogram: testHistogram1, TimestampMs: 0},
247247
{Histogram: testHistogram1, TimestampMs: 2},
248248
{Histogram: testHistogram1, TimestampMs: 3},
@@ -310,30 +310,30 @@ func TestMergeSampleStreams(t *testing.T) {
310310
sampleStreams: []SampleStream{
311311
{
312312
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
313-
Histograms: []SampleHistogramPair{
313+
Histograms: []*SampleHistogramPair{
314314
{Histogram: testHistogram1, TimestampMs: 0},
315315
{Histogram: testHistogram1, TimestampMs: 2},
316316
{Histogram: testHistogram1, TimestampMs: 3},
317317
},
318318
},
319319
{
320320
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
321-
Histograms: []SampleHistogramPair{
321+
Histograms: []*SampleHistogramPair{
322322
{Histogram: testHistogram2, TimestampMs: 1},
323323
{Histogram: testHistogram2, TimestampMs: 4},
324324
},
325325
},
326326
{
327327
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
328-
Histograms: []SampleHistogramPair{
328+
Histograms: []*SampleHistogramPair{
329329
{Histogram: testHistogram1, TimestampMs: 0},
330330
{Histogram: testHistogram1, TimestampMs: 1},
331331
{Histogram: testHistogram1, TimestampMs: 4},
332332
},
333333
},
334334
{
335335
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
336-
Histograms: []SampleHistogramPair{
336+
Histograms: []*SampleHistogramPair{
337337
{Histogram: testHistogram2, TimestampMs: 2},
338338
{Histogram: testHistogram2, TimestampMs: 3},
339339
},
@@ -342,7 +342,7 @@ func TestMergeSampleStreams(t *testing.T) {
342342
expectedOutput: map[string]SampleStream{
343343
ingester_client.LabelsToKeyString(lbls): {
344344
Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
345-
Histograms: []SampleHistogramPair{
345+
Histograms: []*SampleHistogramPair{
346346
{Histogram: testHistogram1, TimestampMs: 0},
347347
{Histogram: testHistogram1, TimestampMs: 2},
348348
{Histogram: testHistogram1, TimestampMs: 3},
@@ -351,7 +351,7 @@ func TestMergeSampleStreams(t *testing.T) {
351351
},
352352
ingester_client.LabelsToKeyString(lbls1): {
353353
Labels: cortexpb.FromLabelsToLabelAdapters(lbls1),
354-
Histograms: []SampleHistogramPair{
354+
Histograms: []*SampleHistogramPair{
355355
{Histogram: testHistogram1, TimestampMs: 0},
356356
{Histogram: testHistogram1, TimestampMs: 1},
357357
{Histogram: testHistogram1, TimestampMs: 4},
@@ -453,21 +453,21 @@ func TestSliceHistograms(t *testing.T) {
453453
t.Parallel()
454454
for _, tc := range []struct {
455455
name string
456-
histograms []SampleHistogramPair
456+
histograms []*SampleHistogramPair
457457
minTs int64
458-
expectedHistograms []SampleHistogramPair
458+
expectedHistograms []*SampleHistogramPair
459459
}{
460460
{name: "empty histograms"},
461461
{
462462
name: "minTs smaller than first histogram's timestamp",
463-
histograms: []SampleHistogramPair{
463+
histograms: []*SampleHistogramPair{
464464
{
465465
TimestampMs: 1,
466466
Histogram: testHistogram1,
467467
},
468468
},
469469
minTs: 0,
470-
expectedHistograms: []SampleHistogramPair{
470+
expectedHistograms: []*SampleHistogramPair{
471471
{
472472
TimestampMs: 1,
473473
Histogram: testHistogram1,
@@ -476,7 +476,7 @@ func TestSliceHistograms(t *testing.T) {
476476
},
477477
{
478478
name: "input histograms are not sorted, return all histograms",
479-
histograms: []SampleHistogramPair{
479+
histograms: []*SampleHistogramPair{
480480
{
481481
TimestampMs: 3,
482482
Histogram: testHistogram1,
@@ -487,7 +487,7 @@ func TestSliceHistograms(t *testing.T) {
487487
},
488488
},
489489
minTs: 2,
490-
expectedHistograms: []SampleHistogramPair{
490+
expectedHistograms: []*SampleHistogramPair{
491491
{
492492
TimestampMs: 3,
493493
Histogram: testHistogram1,
@@ -500,7 +500,7 @@ func TestSliceHistograms(t *testing.T) {
500500
},
501501
{
502502
name: "minTs greater than the last histogram's timestamp",
503-
histograms: []SampleHistogramPair{
503+
histograms: []*SampleHistogramPair{
504504
{
505505
TimestampMs: 1,
506506
Histogram: testHistogram1,
@@ -511,11 +511,11 @@ func TestSliceHistograms(t *testing.T) {
511511
},
512512
},
513513
minTs: 3,
514-
expectedHistograms: []SampleHistogramPair{},
514+
expectedHistograms: []*SampleHistogramPair{},
515515
},
516516
{
517517
name: "input histograms not sorted, minTs greater than the last histogram's timestamp",
518-
histograms: []SampleHistogramPair{
518+
histograms: []*SampleHistogramPair{
519519
{
520520
TimestampMs: 0,
521521
Histogram: testHistogram1,
@@ -530,11 +530,11 @@ func TestSliceHistograms(t *testing.T) {
530530
},
531531
},
532532
minTs: 2,
533-
expectedHistograms: []SampleHistogramPair{},
533+
expectedHistograms: []*SampleHistogramPair{},
534534
},
535535
{
536536
name: "input histograms are sorted",
537-
histograms: []SampleHistogramPair{
537+
histograms: []*SampleHistogramPair{
538538
{
539539
TimestampMs: 2,
540540
Histogram: testHistogram1,
@@ -549,7 +549,7 @@ func TestSliceHistograms(t *testing.T) {
549549
},
550550
},
551551
minTs: 1,
552-
expectedHistograms: []SampleHistogramPair{
552+
expectedHistograms: []*SampleHistogramPair{
553553
{
554554
TimestampMs: 2,
555555
Histogram: testHistogram1,
@@ -566,7 +566,7 @@ func TestSliceHistograms(t *testing.T) {
566566
},
567567
{
568568
name: "input histograms are sorted, get sliced histograms",
569-
histograms: []SampleHistogramPair{
569+
histograms: []*SampleHistogramPair{
570570
{
571571
TimestampMs: 1,
572572
Histogram: testHistogram1,
@@ -581,7 +581,7 @@ func TestSliceHistograms(t *testing.T) {
581581
},
582582
},
583583
minTs: 2,
584-
expectedHistograms: []SampleHistogramPair{
584+
expectedHistograms: []*SampleHistogramPair{
585585
{
586586
TimestampMs: 3,
587587
Histogram: testHistogram1,

‎pkg/querier/tripperware/query.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ func decodeSampleStream(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
123123
for iter.ReadArray() {
124124
h := SampleHistogramPair{}
125125
UnmarshalSampleHistogramPairJSON(unsafe.Pointer(&h), iter)
126-
ss.Histograms = append(ss.Histograms, h)
126+
ss.Histograms = append(ss.Histograms, &h)
127127
}
128128
default:
129129
iter.ReportError("unmarshal SampleStream", fmt.Sprint("unexpected key:", field))
@@ -323,7 +323,7 @@ func encodeSampleStream(ptr unsafe.Pointer, stream *jsoniter.Stream) {
323323
if i > 0 {
324324
stream.WriteMore()
325325
}
326-
MarshalSampleHistogramPairJSON(unsafe.Pointer(&h), stream)
326+
MarshalSampleHistogramPairJSON(unsafe.Pointer(h), stream)
327327
}
328328
stream.WriteArrayEnd()
329329
}

0 commit comments

Comments
 (0)