From 9bf06a66af53046dea761b8324e446814f1ed042 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Tue, 13 May 2025 15:24:09 +0200 Subject: [PATCH 1/5] NETOBSERV-2227: UDN adjustments, and auto-detect filters - Auto-detect available filters per feature (so we don't need to implement specific code to detect when NetworkName filters (for instance) are available) - Add tests for auto-detection - Do not display "None" as a UDN label --- web/locales/en/plugin__netobserv-plugin.json | 2 +- web/src/components/__tests-data__/columns.ts | 28 +++++- web/src/components/__tests-data__/filters.ts | 12 +++ .../components/drawer/record/record-field.tsx | 5 +- web/src/components/netflow-traffic.tsx | 46 ++------- .../__tests__/filter-definitions.spec.ts | 93 +++++++++++++++---- web/src/utils/filter-definitions.ts | 31 +++++-- 7 files changed, 150 insertions(+), 67 deletions(-) diff --git a/web/locales/en/plugin__netobserv-plugin.json b/web/locales/en/plugin__netobserv-plugin.json index f11b0197f..e47ebb839 100644 --- a/web/locales/en/plugin__netobserv-plugin.json +++ b/web/locales/en/plugin__netobserv-plugin.json @@ -44,7 +44,6 @@ "The flow contains packets with various flags: ": "The flow contains packets with various flags: ", "ICMP type provided but protocol is {{proto}}": "ICMP type provided but protocol is {{proto}}", "ICMP code provided but protocol is {{proto}}": "ICMP code provided but protocol is {{proto}}", - "None": "None", "Invalid data provided. Check JSON for details.": "Invalid data provided. Check JSON for details.", "dropped": "dropped", "dropped by": "dropped by", @@ -161,6 +160,7 @@ "M": "M", "S": "S", "XS": "XS", + "None": "None", "Step {{index}}/{{count}}": "Step {{index}}/{{count}}", "Step {{index}}/{{count}}_plural": "Step {{index}}/{{count}}", "Previous tip": "Previous tip", diff --git a/web/src/components/__tests-data__/columns.ts b/web/src/components/__tests-data__/columns.ts index d1513fc77..f67d6d970 100644 --- a/web/src/components/__tests-data__/columns.ts +++ b/web/src/components/__tests-data__/columns.ts @@ -1,6 +1,6 @@ /* eslint-disable max-len */ import * as _ from 'lodash'; -import { Column, ColumnsId, getDefaultColumns } from '../../utils/columns'; +import { Column, ColumnConfigDef, ColumnsId, getDefaultColumns } from '../../utils/columns'; import { FieldConfig } from '../../utils/fields'; export const ColumnConfigSampleDefs = [ @@ -170,6 +170,16 @@ export const ColumnConfigSampleDefs = [ default: false, width: 15 }, + { + id: 'SrcZone', + group: 'Source', + name: 'Zone', + field: 'SrcK8S_Zone', + filter: 'src_zone', + default: false, + width: 15, + feature: 'zones' + }, { id: 'DstK8S_Name', group: 'Destination', @@ -303,6 +313,16 @@ export const ColumnConfigSampleDefs = [ default: false, width: 15 }, + { + id: 'DstZone', + group: 'Destination', + name: 'Zone', + field: 'DstK8S_Zone', + filter: 'dst_zone', + default: false, + width: 15, + feature: 'zones' + }, { id: 'K8S_Name', name: 'Names', @@ -476,6 +496,7 @@ export const ColumnConfigSampleDefs = [ tooltip: 'DNS request identifier.', field: 'DnsId', filter: 'dns_id', + feature: 'dnsTracking', default: false, width: 5 }, @@ -485,6 +506,8 @@ export const ColumnConfigSampleDefs = [ name: 'DNS Latency', tooltip: 'Time elapsed between DNS request and response.', field: 'DnsLatencyMs', + filter: 'dns_latency', + feature: 'dnsTracking', default: false, width: 5 }, @@ -495,6 +518,7 @@ export const ColumnConfigSampleDefs = [ tooltip: 'DNS RCODE name from response header.', field: 'DnsFlagsResponseCode', filter: 'dns_flag_response_code', + feature: 'dnsTracking', default: false, width: 5 }, @@ -518,7 +542,7 @@ export const ColumnConfigSampleDefs = [ default: false, width: 10 } -]; +] as ColumnConfigDef[]; export const FieldConfigSample = [ { diff --git a/web/src/components/__tests-data__/filters.ts b/web/src/components/__tests-data__/filters.ts index bc1970d3b..5a8d0e408 100644 --- a/web/src/components/__tests-data__/filters.ts +++ b/web/src/components/__tests-data__/filters.ts @@ -182,6 +182,18 @@ export const FilterConfigSampleDefs = [ examples: 'Specify a single kubernetes name following these rules:\n - Containing any alphanumeric, hyphen, underscrore or dot character\n - Partial text like cluster, cluster-image, image-registry\n - Exact match using quotes like "cluster-image-registry"\n - Case sensitive match using quotes like "Deployment"\n - Starting text like cluster, "cluster-*"\n - Ending text like "*-registry"\n - Pattern like "cluster-*-registry", "c*-*-r*y", -i*e-' }, + { + id: 'src_zone', + name: 'Zone', + component: 'autocomplete', + category: 'source' + }, + { + id: 'dst_zone', + name: 'Zone', + component: 'autocomplete', + category: 'destination' + }, { id: 'protocol', name: 'Protocol', diff --git a/web/src/components/drawer/record/record-field.tsx b/web/src/components/drawer/record/record-field.tsx index 60aa9c61f..3eb1293ed 100644 --- a/web/src/components/drawer/record/record-field.tsx +++ b/web/src/components/drawer/record/record-field.tsx @@ -435,7 +435,10 @@ export const RecordField: React.FC = ({ case ColumnsId.udns: { if (Array.isArray(value)) { return nthContainer( - value.map(iName => simpleTextWithTooltip(iName !== '' ? String(iName) : t('None'))), + value + .map(iName => String(iName)) + .filter(iName => iName !== '') + .map(iName => simpleTextWithTooltip(iName)), true, false ); diff --git a/web/src/components/netflow-traffic.tsx b/web/src/components/netflow-traffic.tsx index a1e40c22e..7184b5ec7 100644 --- a/web/src/components/netflow-traffic.tsx +++ b/web/src/components/netflow-traffic.tsx @@ -124,22 +124,6 @@ export const NetflowTraffic: React.FC = ({ return model.config.features.includes('pktDrop'); }, [model.config.features]); - const isUdn = React.useCallback(() => { - return model.config.features.includes('udnMapping'); - }, [model.config.features]); - - const isPktXlat = React.useCallback(() => { - return model.config.features.includes('packetTranslation'); - }, [model.config.features]); - - const isNetEvents = React.useCallback(() => { - return model.config.features.includes('networkEvents'); - }, [model.config.features]); - - const isIPSec = React.useCallback(() => { - return model.config.features.includes('ipsec'); - }, [model.config.features]); - const isPromOnly = React.useCallback(() => { return !allowLoki() || model.dataSource === 'prom'; }, [allowLoki, model.dataSource]); @@ -159,14 +143,6 @@ export const NetflowTraffic: React.FC = ({ [model.config.promLabels, isPromOnly] ); - const isMultiCluster = React.useCallback(() => { - return isPromOnly() ? dataSourceHasLabels(['K8S_ClusterName']) : model.config.features.includes('multiCluster'); - }, [model.config.features, dataSourceHasLabels, isPromOnly]); - - const isZones = React.useCallback(() => { - return isPromOnly() ? dataSourceHasLabels(['SrcK8S_Zone', 'DstK8S_Zone']) : model.config.features.includes('zones'); - }, [model.config.features, dataSourceHasLabels, isPromOnly]); - const getAvailableScopes = React.useCallback(() => { return model.config.scopes.filter(sc => { if (sc.feature) { @@ -219,22 +195,14 @@ export const NetflowTraffic: React.FC = ({ }, [getAvailableColumns]); const getFilterDefs = React.useCallback(() => { - return getFilterDefinitions(model.config.filters, model.config.columns, t).filter( - fd => - (isMultiCluster() || fd.id !== 'cluster_name') && - (isZones() || !fd.id.endsWith('_zone')) && - (isConnectionTracking() || fd.id !== 'id') && - (isDNSTracking() || !fd.id.startsWith('dns_')) && - (isPktDrop() || !fd.id.startsWith('pkt_drop_')) && - (isFlowRTT() || fd.id !== 'time_flow_rtt') && - (isUdn() || fd.id !== 'udns') && - (isPktXlat() || !fd.id.startsWith('xlat_')) && - (isNetEvents() || fd.id !== 'network_events') && - (!isPromOnly() || checkFilterAvailable(fd, model.config.promLabels)) && - (isIPSec() || !fd.id.startsWith('ipsec_')) - ); + return getFilterDefinitions(model.config.filters, model.config.columns, t).filter(fd => { + if (fd.id === 'id') { + return isConnectionTracking(); + } + return checkFilterAvailable(fd, model.config, model.dataSource); + }); // eslint-disable-next-line react-hooks/exhaustive-deps - }, [model.config.columns, model.config.filters, model.config.promLabels, isPromOnly]); + }, [model.config, model.dataSource]); const getQuickFilters = React.useCallback( (c: Config = model.config) => { diff --git a/web/src/utils/__tests__/filter-definitions.spec.ts b/web/src/utils/__tests__/filter-definitions.spec.ts index e6d72ae25..2c4b70a9a 100644 --- a/web/src/utils/__tests__/filter-definitions.spec.ts +++ b/web/src/utils/__tests__/filter-definitions.spec.ts @@ -1,4 +1,6 @@ +import { ColumnConfigSampleDefs } from '../../components/__tests-data__/columns'; import { FilterDefinitionSample } from '../../components/__tests-data__/filters'; +import { Config, Feature } from '../../model/config'; import { checkFilterAvailable, findFilter } from '../filter-definitions'; describe('Resource validation', () => { @@ -99,35 +101,92 @@ describe('Resource checkCompletion', () => { }); }); -describe('Check availability', () => { +describe('Check availability for prometheus only', () => { const simpleFilter = findFilter(FilterDefinitionSample, 'src_name')!; const k8sFilter = findFilter(FilterDefinitionSample, 'src_resource')!; + const getConfig = (promLabels: string[]): Config => { + return { promLabels, dataSources: ['prom'] } as Config; + }; it('should be available', () => { - let available = checkFilterAvailable(simpleFilter, ['SrcK8S_Name', 'DstK8S_Name']); + let available = checkFilterAvailable(simpleFilter, getConfig(['SrcK8S_Name', 'DstK8S_Name']), 'prom'); expect(available).toBe(true); - available = checkFilterAvailable(k8sFilter, [ - 'SrcK8S_OwnerName', - 'SrcK8S_OwnerType', - 'SrcK8S_Namespace', - 'DstK8S_OwnerName', - 'DstK8S_OwnerType', - 'DstK8S_Namespace' - ]); + available = checkFilterAvailable( + k8sFilter, + getConfig([ + 'SrcK8S_OwnerName', + 'SrcK8S_OwnerType', + 'SrcK8S_Namespace', + 'DstK8S_OwnerName', + 'DstK8S_OwnerType', + 'DstK8S_Namespace' + ]), + 'prom' + ); expect(available).toBe(true); }); it('should not be available', () => { - let available = checkFilterAvailable(simpleFilter, ['SrcK8S_OwnerName', 'DstK8S_OwnerName']); + let available = checkFilterAvailable(simpleFilter, getConfig(['SrcK8S_OwnerName', 'DstK8S_OwnerName']), 'prom'); expect(available).toBe(false); - available = checkFilterAvailable(k8sFilter, [ - 'SrcK8S_OwnerName', - 'SrcK8S_Namespace', - 'DstK8S_OwnerName', - 'DstK8S_Namespace' - ]); + available = checkFilterAvailable( + k8sFilter, + getConfig(['SrcK8S_OwnerName', 'SrcK8S_Namespace', 'DstK8S_OwnerName', 'DstK8S_Namespace']), + 'prom' + ); expect(available).toBe(false); }); }); + +describe('Check availability against features', () => { + const getConfig = (feats: Feature[]): Config => { + return { features: feats, dataSources: ['loki'], columns: ColumnConfigSampleDefs } as Config; + }; + + it('with standard filters', () => { + const simpleFilter = findFilter(FilterDefinitionSample, 'src_name')!; + const k8sFilter = findFilter(FilterDefinitionSample, 'src_resource')!; + + let available = checkFilterAvailable(simpleFilter, getConfig([]), 'auto'); + expect(available).toBe(true); + + available = checkFilterAvailable(k8sFilter, getConfig([]), 'auto'); + expect(available).toBe(true); + + available = checkFilterAvailable(simpleFilter, getConfig(['dnsTracking']), 'auto'); + expect(available).toBe(true); + + available = checkFilterAvailable(k8sFilter, getConfig(['dnsTracking']), 'auto'); + expect(available).toBe(true); + }); + + it('with AZ filters', () => { + const azFilter = findFilter(FilterDefinitionSample, 'src_zone')!; + + let available = checkFilterAvailable(azFilter, getConfig([]), 'auto'); + expect(available).toBe(false); + + available = checkFilterAvailable(azFilter, getConfig(['dnsTracking']), 'auto'); + expect(available).toBe(false); + + available = checkFilterAvailable(azFilter, getConfig(['zones']), 'auto'); + expect(available).toBe(true); + }); + + it('with DNS filters', () => { + const dnsIdFilter = findFilter(FilterDefinitionSample, 'dns_id')!; + const dnsLatilter = findFilter(FilterDefinitionSample, 'dns_latency')!; + + let available = checkFilterAvailable(dnsIdFilter, getConfig([]), 'auto'); + expect(available).toBe(false); + available = checkFilterAvailable(dnsLatilter, getConfig([]), 'auto'); + expect(available).toBe(false); + + available = checkFilterAvailable(dnsIdFilter, getConfig(['dnsTracking']), 'auto'); + expect(available).toBe(true); + available = checkFilterAvailable(dnsLatilter, getConfig(['dnsTracking']), 'auto'); + expect(available).toBe(true); + }); +}); diff --git a/web/src/utils/filter-definitions.ts b/web/src/utils/filter-definitions.ts index 90e8af681..f26ef6074 100644 --- a/web/src/utils/filter-definitions.ts +++ b/web/src/utils/filter-definitions.ts @@ -1,6 +1,7 @@ import { TFunction } from 'i18next'; import * as _ from 'lodash'; import { Field } from '../api/ipfix'; +import { Config } from '../model/config'; import { FilterCategory, FilterComponent, @@ -11,6 +12,7 @@ import { FiltersEncoder, FilterValue } from '../model/filters'; +import { DataSource } from '../model/flow-query'; import { joinResource, SplitResource, splitResource, SplitStage } from '../model/resource'; import { getPort } from '../utils/port'; import { ColumnConfigDef } from './columns'; @@ -336,14 +338,29 @@ export const findFilter = (filterDefinitions: FilterDefinition[], id: FilterId) return filterDefinitions.find(def => def.id === id); }; -export const checkFilterAvailable = (fd: FilterDefinition, labels: string[]) => { - const q = fd.encoder([{ v: 'any' }], false, false, false); - const parts = q.split('&'); - for (let i = 0; i < parts.length; i++) { - const kv = parts[i].split('='); - if (kv.length === 0 || !labels.includes(kv[0])) { - return false; +export const checkFilterAvailable = (fd: FilterDefinition, config: Config, dataSource: DataSource) => { + const allowLoki = config.dataSources.some(ds => ds === 'loki'); + const isPromOnly = !allowLoki || dataSource === 'prom'; + + if (isPromOnly) { + // "encode" a dummy query to check related labels, and make sure they're all part of available prom labels + const q = fd.encoder([{ v: 'any' }], false, false, false); + const parts = q.split('&'); + for (let i = 0; i < parts.length; i++) { + const kv = parts[i].split('='); + if (kv.length === 0 || !config.promLabels.includes(kv[0])) { + return false; + } } + return true; } + + // Check against enabled features + const colConfig = config.columns.find(c => c.filter === fd.id); + if (colConfig?.feature) { + return config.features.includes(colConfig?.feature); + } + + // Allow by default return true; }; From f72d9a381356fabc276c464b650c309922484a55 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Wed, 14 May 2025 09:37:33 +0200 Subject: [PATCH 2/5] Remove unnecessary Filter on Field --- pkg/config/config.go | 2 -- web/src/utils/fields.ts | 1 - 2 files changed, 3 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 0708afeb9..1cd2c844e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -116,8 +116,6 @@ type FieldConfig struct { Type string `yaml:"type" json:"type"` Format string `yaml:"format,omitempty" json:"format,omitempty"` Description string `yaml:"description" json:"description"` - // lokiLabel flag is for documentation only. Use loki.labels instead - Filter string `yaml:"filter,omitempty" json:"filter,omitempty"` } type Frontend struct { diff --git a/web/src/utils/fields.ts b/web/src/utils/fields.ts index 4385ce3c5..fe6abd071 100644 --- a/web/src/utils/fields.ts +++ b/web/src/utils/fields.ts @@ -7,5 +7,4 @@ export interface FieldConfig { type: FieldType; format?: FieldFormat; description: string; - filter?: string; } From f346474638cbbbc13cd42bcb4a25cf1d0b5d4167 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Wed, 14 May 2025 13:32:27 +0200 Subject: [PATCH 3/5] write n/a for no udn --- .../components/drawer/record/record-field.tsx | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/web/src/components/drawer/record/record-field.tsx b/web/src/components/drawer/record/record-field.tsx index 3eb1293ed..d116ac240 100644 --- a/web/src/components/drawer/record/record-field.tsx +++ b/web/src/components/drawer/record/record-field.tsx @@ -218,14 +218,16 @@ export const RecordField: React.FC = ({ const nthContainer = (children: (JSX.Element | undefined)[], asChild = true, childIcon = true, forcedSize?: Size) => { return ( - {children.map((c, i) => { - const child = c ? c : emptyText(); - if (i > 0 && asChild && childIcon) { - const arrow = {'↪'}; - return sideBySideContainer(arrow, child, 'flexNone', 'flex_1', 'nowrap'); - } - return child; - })} + {children.length > 0 + ? children.map((c, i) => { + const child = c ? c : emptyText(); + if (i > 0 && asChild && childIcon) { + const arrow = {'↪'}; + return sideBySideContainer(arrow, child, 'flexNone', 'flex_1', 'nowrap'); + } + return child; + }) + : emptyText()} ); }; From 78adca22acedf32a1196cc2adf6bd201db1a3162 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Fri, 16 May 2025 10:53:43 +0200 Subject: [PATCH 4/5] Fix css alignment for n/a text --- web/src/components/drawer/record/record-field.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/components/drawer/record/record-field.tsx b/web/src/components/drawer/record/record-field.tsx index d116ac240..40ed10e71 100644 --- a/web/src/components/drawer/record/record-field.tsx +++ b/web/src/components/drawer/record/record-field.tsx @@ -227,7 +227,7 @@ export const RecordField: React.FC = ({ } return child; }) - : emptyText()} + : {t('n/a')}} ); }; From 00c0312dee82124d66c49c21656ed61471f3e3b9 Mon Sep 17 00:00:00 2001 From: Joel Takvorian Date: Fri, 16 May 2025 11:15:17 +0200 Subject: [PATCH 5/5] Fix empty udn name --- pkg/handler/k8s.go | 2 +- .../components/drawer/record/record-field.tsx | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/handler/k8s.go b/pkg/handler/k8s.go index a40ec9496..370ee1d01 100644 --- a/pkg/handler/k8s.go +++ b/pkg/handler/k8s.go @@ -56,7 +56,7 @@ func (h *Handlers) GetUDNIdss(ctx context.Context) func(w http.ResponseWriter, r } for _, udn := range udns { md := udn.Object["metadata"].(map[string]interface{}) - values = append(values, fmt.Sprintf("%s.%s", md["namespace"], md["name"])) + values = append(values, fmt.Sprintf("%s/%s", md["namespace"], md["name"])) } writeJSON(w, http.StatusOK, utils.NonEmpty(utils.Dedup(values))) } diff --git a/web/src/components/drawer/record/record-field.tsx b/web/src/components/drawer/record/record-field.tsx index 40ed10e71..980a32568 100644 --- a/web/src/components/drawer/record/record-field.tsx +++ b/web/src/components/drawer/record/record-field.tsx @@ -218,16 +218,18 @@ export const RecordField: React.FC = ({ const nthContainer = (children: (JSX.Element | undefined)[], asChild = true, childIcon = true, forcedSize?: Size) => { return ( - {children.length > 0 - ? children.map((c, i) => { - const child = c ? c : emptyText(); - if (i > 0 && asChild && childIcon) { - const arrow = {'↪'}; - return sideBySideContainer(arrow, child, 'flexNone', 'flex_1', 'nowrap'); - } - return child; - }) - : {t('n/a')}} + {children.length > 0 ? ( + children.map((c, i) => { + const child = c ? c : emptyText(); + if (i > 0 && asChild && childIcon) { + const arrow = {'↪'}; + return sideBySideContainer(arrow, child, 'flexNone', 'flex_1', 'nowrap'); + } + return child; + }) + ) : ( + {t('n/a')} + )} ); };