diff --git a/pkg/handler/resources.go b/pkg/handler/resources.go index 2e70f2e73..0272d3ecf 100644 --- a/pkg/handler/resources.go +++ b/pkg/handler/resources.go @@ -34,7 +34,7 @@ func (h *Handlers) GetClusters(ctx context.Context) func(w http.ResponseWriter, // Fetch and merge values for K8S_ClusterName values, code, err := h.getLabelValues(ctx, clients, fields.Cluster) if err != nil { - writeError(w, code, "Error while fetching label cluster values from Loki: "+err.Error()) + writeError(w, code, err.Error()) return } @@ -65,14 +65,14 @@ func (h *Handlers) GetZones(ctx context.Context) func(w http.ResponseWriter, r * // Fetch and merge values for SrcK8S_Zone and DstK8S_Zone values1, code, err := h.getLabelValues(ctx, clients, fields.SrcZone) if err != nil { - writeError(w, code, "Error while fetching label source zone values from Loki: "+err.Error()) + writeError(w, code, err.Error()) return } values = append(values, values1...) values2, code, err := h.getLabelValues(ctx, clients, fields.DstZone) if err != nil { - writeError(w, code, "Error while fetching label destination zone values from Loki: "+err.Error()) + writeError(w, code, err.Error()) return } values = append(values, values2...) @@ -104,14 +104,14 @@ func (h *Handlers) GetNamespaces(ctx context.Context) func(w http.ResponseWriter // Fetch and merge values for SrcK8S_Namespace and DstK8S_Namespace values1, code, err := h.getLabelValues(ctx, clients, fields.SrcNamespace) if err != nil { - writeError(w, code, "Error while fetching label source namespace values from Loki: "+err.Error()) + writeError(w, code, err.Error()) return } values = append(values, values1...) values2, code, err := h.getLabelValues(ctx, clients, fields.DstNamespace) if err != nil { - writeError(w, code, "Error while fetching label destination namespace values from Loki: "+err.Error()) + writeError(w, code, err.Error()) return } values = append(values, values2...) @@ -123,10 +123,27 @@ func (h *Handlers) GetNamespaces(ctx context.Context) func(w http.ResponseWriter func (h *Handlers) getLabelValues(ctx context.Context, cl clients, label string) ([]string, int, error) { if h.PromInventory != nil && h.PromInventory.LabelExists(label) { - return prometheus.GetLabelValues(ctx, cl.promAdmin, label, nil) + resp, code, err := prometheus.GetLabelValues(ctx, cl.promAdmin, label, nil) + if err != nil { + if code == http.StatusUnauthorized || code == http.StatusForbidden { + // In case this was a prometheus 401 / 403 error, the query is repeated with Loki + // This is because multi-tenancy is currently not managed for prom datasource, hence such queries have to go with Loki + // Unfortunately we don't know a safe and generic way to pre-flight check if the user will be authorized + hlog.Info("Retrying with Loki...") + // continuing with loki below + } else { + return nil, code, fmt.Errorf("error while fetching label %s values from Prometheus: %w", label, err) + } + } else { + return resp, code, nil + } } if h.Cfg.IsLokiEnabled() { - return getLokiLabelValues(h.Cfg.Loki.URL, cl.loki, label) + resp, code, err := getLokiLabelValues(h.Cfg.Loki.URL, cl.loki, label) + if err != nil { + return nil, code, fmt.Errorf("error while fetching label %s values from Loki: %w", label, err) + } + return resp, code, nil } // Loki disabled AND label not managed in metrics => send an error return nil, http.StatusBadRequest, fmt.Errorf("label %s not found in Prometheus metrics", label) @@ -186,7 +203,7 @@ func (h *Handlers) getNamesForPrefix(ctx context.Context, cl clients, prefix, ki searchField = prefix + fields.Name } - if h.Cfg.IsPromEnabled() { + if h.Cfg.IsPromEnabled() && h.PromInventory.LabelExists(searchField) { // Label match query (any metric) q := prometheus.QueryFilters("", filts) return prometheus.GetLabelValues(ctx, cl.promAdmin, searchField, []string{q}) diff --git a/pkg/prometheus/client.go b/pkg/prometheus/client.go index 787430743..843590ad2 100644 --- a/pkg/prometheus/client.go +++ b/pkg/prometheus/client.go @@ -79,15 +79,7 @@ func executeQueryRange(ctx context.Context, cl api.Client, q *Query) (pmod.Value } if err != nil { log.Tracef("Error:\n%v", err) - code = http.StatusServiceUnavailable - var promError *v1.Error - if errors.As(err, &promError) { - if promError.Type == v1.ErrClient && strings.Contains(promError.Msg, "401") { - code = http.StatusUnauthorized - } else if promError.Type == v1.ErrClient && strings.Contains(promError.Msg, "403") { - code = http.StatusForbidden - } - } + code = translateErrorCode(err) return nil, code, fmt.Errorf("error from Prometheus query: %w", err) } @@ -125,12 +117,13 @@ func GetLabelValues(ctx context.Context, cl api.Client, label string, match []st log.Debugf("GetLabelValues: %s", label) v1api := v1.NewAPI(cl) result, warnings, err := v1api.LabelValues(ctx, label, match, time.Now().Add(-3*time.Hour), time.Now()) - if err != nil { - return nil, http.StatusServiceUnavailable, err - } if len(warnings) > 0 { log.Infof("GetLabelValues warnings: %v", warnings) } + if err != nil { + code := translateErrorCode(err) + return nil, code, fmt.Errorf("could not get label values: %w", err) + } log.Tracef("Result:\n%v", result) var asStrings []string for _, s := range result { @@ -138,3 +131,15 @@ func GetLabelValues(ctx context.Context, cl api.Client, label string, match []st } return asStrings, http.StatusOK, nil } + +func translateErrorCode(err error) int { + var promError *v1.Error + if errors.As(err, &promError) { + if promError.Type == v1.ErrClient && strings.Contains(promError.Msg, "401") { + return http.StatusUnauthorized + } else if promError.Type == v1.ErrClient && strings.Contains(promError.Msg, "403") { + return http.StatusForbidden + } + } + return http.StatusServiceUnavailable +} diff --git a/web/src/components/toolbar/filters/autocomplete-filter.tsx b/web/src/components/toolbar/filters/autocomplete-filter.tsx index adb578159..f4ce29151 100644 --- a/web/src/components/toolbar/filters/autocomplete-filter.tsx +++ b/web/src/components/toolbar/filters/autocomplete-filter.tsx @@ -86,9 +86,7 @@ export const AutocompleteFilter: React.FC = ({ setCurrentValue(newValue); filterDefinition .getOptions(newValue) - .then(opts => { - setOptions(opts); - }) + .then(setOptions) .catch(err => { const errorMessage = getHTTPErrorDetails(err); setMessageWithDelay(errorMessage); diff --git a/web/src/model/filters.ts b/web/src/model/filters.ts index f652b4c35..a3a20639c 100644 --- a/web/src/model/filters.ts +++ b/web/src/model/filters.ts @@ -92,10 +92,16 @@ export interface FilterOption { } export const createFilterValue = (def: FilterDefinition, value: string): Promise => { - return def.getOptions(value).then(opts => { - const option = opts.find(opt => opt.name === value || opt.value === value); - return option ? { v: option.value, display: option.name } : { v: value }; - }); + return def + .getOptions(value) + .then(opts => { + const option = opts.find(opt => opt.name === value || opt.value === value); + return option ? { v: option.value, display: option.name } : { v: value }; + }) + .catch(_ => { + // In case of error, still create the minimal possible FilterValue + return { v: value }; + }); }; export const hasEnabledFilterValues = (filter: Filter) => {