Fix group-replica strategy (#49)

Author: hczhu-db

cmd/thanos/query.go

@@ -575,7 +575,7 @@ func runQuery(
 		GroupReplicaPartialResponseStrategy: groupReplicaPartialResponseStrategy,
 		EnableDedupMerge:                    enableDedupMerge,
 	}
-	level.Info(logger).Log("msg", "databricks querier features", "opts", opts)
+	level.Info(logger).Log("msg", "databricks querier features", "opts", fmt.Sprintf("%+v", opts))
 	queryableCreator = query.NewQueryableCreatorWithOptions(
 		logger,
 		extprom.WrapRegistererWithPrefix("thanos_query_", reg),

pkg/store/proxy.go

@@ -389,15 +389,15 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb.
 			level.Error(reqLogger).Log(
 				"msg", "Multipel replicas have failures for the same group",
 				"group", st.GroupKey(),
-				"replicas", failedStores[st.GroupKey()],
+				"replicas", fmt.Sprintf("%+v", failedStores[st.GroupKey()]),
 			)
 			return err
 		}
 		if len(groupReplicaStores[st.GroupKey()]) == 1 && failedStores[st.GroupKey()][st.ReplicaKey()] > 1 {
 			level.Error(reqLogger).Log(
 				"msg", "A single replica group has multiple failures",
 				"group", st.GroupKey(),
-				"replicas", failedStores[st.GroupKey()],
+				"replicas", fmt.Sprintf("%+v", failedStores[st.GroupKey()]),
 			)
 			return err
 		}
@@ -422,6 +422,7 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb.
 
 		respSet, err := newAsyncRespSet(ctx, st, r, s.responseTimeout, s.retrievalStrategy, &s.buffers, r.ShardInfo, reqLogger, s.metrics.emptyStreamResponses)
 		if err != nil {
+			// NB: respSet is nil in case of error.
 			level.Error(reqLogger).Log("err", err)
 			level.Warn(s.logger).Log("msg", "Store failure", "group", st.GroupKey(), "replica", st.ReplicaKey())
 			bumpCounter(st.GroupKey(), st.ReplicaKey(), failedStores)
@@ -430,6 +431,7 @@ func (s *ProxyStore) Series(originalRequest *storepb.SeriesRequest, srv storepb.
 				if checkGroupReplicaErrors(st, err) != nil {
 					return err
 				}
+				continue
 			} else if !r.PartialResponseDisabled || r.PartialResponseStrategy == storepb.PartialResponseStrategy_WARN {
 				if err := srv.Send(storepb.NewWarnSeriesResponse(err)); err != nil {
 					return err