Add trace IDs to cached extents to help track down query gaps. (#1541)

Signed-off-by: Tom Wilkie <[email protected]>

Author: tomwilkie

pkg/querier/frontend/frontend.pb.go

@@ -63,4 +63,5 @@ message Extent  {
 	int64 start = 1 [(gogoproto.jsontag) = "start"];
 	int64 end = 2 [(gogoproto.jsontag) = "end"];
 	APIResponse response = 3 [(gogoproto.jsontag) = "response"];
+	string trace_id = 4 [(gogoproto.jsontag) = "-"];
 }

pkg/querier/frontend/frontend.proto

@@ -7,15 +7,18 @@ import (
 	"sort"
 	"time"
 
-	"github.com/cortexproject/cortex/pkg/chunk/cache"
-	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/cortexproject/cortex/pkg/util/validation"
 	"github.com/go-kit/kit/log/level"
 	"github.com/gogo/protobuf/proto"
 	opentracing "github.com/opentracing/opentracing-go"
 	otlog "github.com/opentracing/opentracing-go/log"
 	"github.com/prometheus/common/model"
+	"github.com/uber/jaeger-client-go"
 	"github.com/weaveworks/common/user"
+
+	"github.com/cortexproject/cortex/pkg/chunk/cache"
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/spanlogger"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 // ResultsCacheConfig is the config for the results cache.
@@ -97,6 +100,7 @@ func (s resultsCache) handleMiss(ctx context.Context, r *QueryRangeRequest) (*AP
 			Start:    r.Start,
 			End:      r.End,
 			Response: response,
+			TraceId:  jaegerTraceID(ctx),
 		},
 	}
 	return response, extents, nil
@@ -107,6 +111,8 @@ func (s resultsCache) handleHit(ctx context.Context, r *QueryRangeRequest, exten
 		reqResps []requestResponse
 		err      error
 	)
+	log, ctx := spanlogger.New(ctx, "handleHit")
+	defer log.Finish()
 
 	requests, responses := partition(r, extents)
 	if len(requests) == 0 {
@@ -126,6 +132,7 @@ func (s resultsCache) handleHit(ctx context.Context, r *QueryRangeRequest, exten
 			Start:    reqResp.req.Start,
 			End:      reqResp.req.End,
 			Response: reqResp.resp,
+			TraceId:  jaegerTraceID(ctx),
 		})
 	}
 	sort.Slice(extents, func(i, j int) bool {
@@ -141,6 +148,9 @@ func (s resultsCache) handleHit(ctx context.Context, r *QueryRangeRequest, exten
 			continue
 		}
 
+		log.Log("msg", "merging extent", "start", accumulator.Start, "old_end", accumulator.End, "new_end", extents[i].End, "from_trace", accumulator.TraceId, "with_trace", accumulator.TraceId)
+
+		accumulator.TraceId = jaegerTraceID(ctx)
 		accumulator.End = extents[i].End
 		accumulator.Response, err = mergeAPIResponses([]*APIResponse{accumulator.Response, extents[i].Response})
 		if err != nil {
@@ -238,3 +248,13 @@ func (s resultsCache) put(ctx context.Context, key string, extents []Extent) {
 
 	s.cache.Store(ctx, []string{cache.HashKey(key)}, [][]byte{buf})
 }
+
+func jaegerTraceID(ctx context.Context) string {
+	span := opentracing.SpanFromContext(ctx)
+	if span == nil {
+		return ""
+	}
+
+	spanContext := span.Context().(jaeger.SpanContext)
+	return spanContext.TraceID().String()
+}

pkg/querier/frontend/results_cache.go

@@ -386,12 +386,8 @@ github.com/tinylib/msgp/msgp
 # github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5
 github.com/tmc/grpc-websocket-proxy/wsproxy
 # github.com/uber/jaeger-client-go v2.16.0+incompatible
-github.com/uber/jaeger-client-go/config
 github.com/uber/jaeger-client-go
-github.com/uber/jaeger-client-go/internal/baggage/remote
-github.com/uber/jaeger-client-go/internal/throttler/remote
-github.com/uber/jaeger-client-go/rpcmetrics
-github.com/uber/jaeger-client-go/transport
+github.com/uber/jaeger-client-go/config
 github.com/uber/jaeger-client-go/internal/baggage
 github.com/uber/jaeger-client-go/internal/spanlog
 github.com/uber/jaeger-client-go/internal/throttler
@@ -401,8 +397,12 @@ github.com/uber/jaeger-client-go/thrift-gen/jaeger
 github.com/uber/jaeger-client-go/thrift-gen/sampling
 github.com/uber/jaeger-client-go/thrift-gen/zipkincore
 github.com/uber/jaeger-client-go/utils
-github.com/uber/jaeger-client-go/thrift-gen/baggage
+github.com/uber/jaeger-client-go/internal/baggage/remote
+github.com/uber/jaeger-client-go/internal/throttler/remote
+github.com/uber/jaeger-client-go/rpcmetrics
+github.com/uber/jaeger-client-go/transport
 github.com/uber/jaeger-client-go/thrift-gen/agent
+github.com/uber/jaeger-client-go/thrift-gen/baggage
 # github.com/uber/jaeger-lib v2.0.0+incompatible
 github.com/uber/jaeger-lib/metrics/prometheus
 github.com/uber/jaeger-lib/metrics