diff --git a/README.md b/README.md index faef5e66c..1ac9d6c6a 100644 --- a/README.md +++ b/README.md @@ -178,3 +178,19 @@ make cypress ``` 4. Click on "Run N integration specs" + +## Updating schemas + +This console plugin comes with several panels allowing GUI-based configuration for `FlowCollector` and other managed resources, including: + +- Comprehensive forms for each of the resources (includes most supported settings that are pretty common to configure). +- Simplified wizards for faster configuration. + +When you update the operator CRDs, you may have to update some schemas here as well, which contain some rules that drive how forms are displayed. Especially: + +- [uiSchema.ts](./web/src/components/forms/config/uiSchema.ts) contains a display-oriented description of every CRD field, including whether or not they should be hidden, or if they have relationships with other fields. +- [< CRD name >-wizard.tsx](./web/src/components/forms/flowCollector-wizard.tsx) contains specific fields to be displayed in wizards. + +When a CRD field is added, consider whether you need to update these files. + +Additionally, [schema.ts](./web/moduleMapper/schemas.ts) contains the full CRD schema as JSON, used in tests and in standalone mode, and should also be kept up to date. diff --git a/web/moduleMapper/dummy.tsx b/web/moduleMapper/dummy.tsx index 7c6c851df..9d691d3f0 100644 --- a/web/moduleMapper/dummy.tsx +++ b/web/moduleMapper/dummy.tsx @@ -1,3 +1,5 @@ +// File only used in tests or dev console + /* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable @typescript-eslint/no-explicit-any */ import { @@ -18,8 +20,8 @@ import { useK8sModelsWithColors } from '../src/utils/k8s-models-hook'; import { useTheme } from '../src/utils/theme-hook'; import { safeJSToYAML } from '../src/utils/yaml'; import { k8sModels } from './k8s-models'; -import { FlowCollectorSchema, FlowMetricSchema } from './schemas'; -import { GetFlowCollectorJS, GetFlowMetricJS } from './templates'; +import { flowCollectorSchema, flowMetricSchema } from './schemas'; +import { getFlowCollectorJS, getFlowMetricJS } from './templates'; // This dummy file is used to resolve @Console imports from @openshift-console for JEST / Standalone // You can add any exports needed here @@ -130,7 +132,7 @@ export function useK8sWatchResource(req: any) { served: true, storage: true, schema: { - openAPIV3Schema: FlowCollectorSchema, + openAPIV3Schema: flowCollectorSchema, } }] } @@ -154,7 +156,7 @@ export function useK8sWatchResource(req: any) { served: true, storage: true, schema: { - openAPIV3Schema: FlowMetricSchema + openAPIV3Schema: flowMetricSchema } }] } @@ -162,7 +164,7 @@ export function useK8sWatchResource(req: any) { } break; case 'FlowCollector': - const fc = _.cloneDeep(GetFlowCollectorJS()); + const fc = _.cloneDeep(getFlowCollectorJS()); fc.spec!.loki.enable = false; fc.spec!.exporters = [{ type: "Kafka" }, { type: "OpenTelemetry" }] fc.status = { @@ -236,7 +238,7 @@ export function useK8sWatchResource(req: any) { break; case 'FlowMetric': if (req.name === 'flowmetric-sample') { - const fm = _.cloneDeep(GetFlowMetricJS()); + const fm = _.cloneDeep(getFlowMetricJS()); fm.spec!.metricName = 'test_metric'; setResource(fm); } diff --git a/web/moduleMapper/k8s-models.ts b/web/moduleMapper/k8s-models.ts index 5790f7eb1..0a1aea812 100644 --- a/web/moduleMapper/k8s-models.ts +++ b/web/moduleMapper/k8s-models.ts @@ -1,4 +1,6 @@ -//these values comes from original useK8sModels using debug +// File only used in tests or dev console + +// These values come from original useK8sModels using debug export const k8sModels = { "ImageStreamImport": { "label": "ImageStreamImport", diff --git a/web/moduleMapper/schemas.ts b/web/moduleMapper/schemas.ts index f8d0f3e56..ed62c32ca 100644 --- a/web/moduleMapper/schemas.ts +++ b/web/moduleMapper/schemas.ts @@ -1,8 +1,11 @@ +// File only used in tests or dev console + /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable max-len */ import { RJSFSchema } from '@rjsf/utils'; -export const FlowCollectorSchema: RJSFSchema | any = { +// flowCollectorSchema is only used in tests or dev console +export const flowCollectorSchema: RJSFSchema | any = { title: 'FlowCollector', description: 'The schema for the network flows collection API, which pilots and configures the underlying deployments.', @@ -51,10 +54,10 @@ export const FlowCollectorSchema: RJSFSchema | any = { }, deploymentModel: { description: - '`deploymentModel` defines the desired type of deployment for flow processing. Possible values are:\n- `Direct` (default) to make the flow processor listen directly from the agents.\n- `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.\nKafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).', + '`deploymentModel` defines the desired type of deployment for flow processing. Possible values are:\n - `Direct` (default) to make the flow processor listen directly from the agents using the host network, backed by a DaemonSet.\n - `Service` to make the flow processor listen as a Kubernetes Service, backed by a scalable Deployment.\n - `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.\n Kafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).
`Direct` is not recommended on large clusters as it is less memory efficient.', type: 'string', default: 'Direct', - enum: ['Direct', 'Kafka'] + enum: ['Direct', 'Service', 'Kafka'] }, kafka: { description: @@ -3329,9 +3332,21 @@ export const FlowCollectorSchema: RJSFSchema | any = { default: 'Flows', enum: ['Flows', 'Conversations', 'EndedConversations', 'All'] }, + unmanagedReplicas: { + description: + 'If `unmanagedReplicas` is `true`, the operator will not reconcile `consumerReplicas`. This is useful when using a pod autoscaler.', + type: 'boolean' + }, + consumerReplicas: { + description: + '`consumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline`, default is 3. This setting is ignored when `spec.deploymentModel` is `Direct` or when `spec.processor.unmanagedReplicas` is `true`.', + type: 'integer', + format: 'int32', + minimum: 0 + }, kafkaConsumerReplicas: { description: - '`kafkaConsumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.\nThis setting is ignored when Kafka is disabled.', + '`kafkaConsumerAutoscaler` [deprecated (*)] is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages. This setting is ignored when Kafka is disabled. Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.processor.unmanagedReplicas` to `true`.', type: 'integer', format: 'int32', default: 3, @@ -5127,7 +5142,7 @@ export const FlowCollectorSchema: RJSFSchema | any = { enum: ['IfNotPresent', 'Always', 'Never'] }, autoscaler: { - description: '`autoscaler` spec of a horizontal pod autoscaler to set up for the plugin Deployment.', + description: '`autoscaler` [deprecated (*)] spec of a horizontal pod autoscaler to set up for the plugin Deployment. Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.consolePlugin.unmanagedReplicas` to `true`.', type: 'object', properties: { maxReplicas: { @@ -5547,6 +5562,10 @@ export const FlowCollectorSchema: RJSFSchema | any = { } } }, + unmanagedReplicas: { + description: 'If `unmanagedReplicas` is `true`, the operator will not reconcile `replicas`. This is useful when using a pod autoscaler.', + type: 'boolean', + }, replicas: { description: '`replicas` defines the number of replicas (pods) to start.', type: 'integer', @@ -5936,7 +5955,8 @@ export const FlowCollectorSchema: RJSFSchema | any = { } }; -export const FlowMetricSchema: RJSFSchema | any = { +// flowMetricSchema is only used in tests or dev console +export const flowMetricSchema: RJSFSchema | any = { title: 'FlowMetric', description: 'The API allowing to create custom metrics from the collected flow logs.', type: 'object', @@ -6127,4 +6147,4 @@ export const FlowMetricSchema: RJSFSchema | any = { } } } -}; \ No newline at end of file +}; diff --git a/web/moduleMapper/templates.ts b/web/moduleMapper/templates.ts index 4d355ddfd..e8aef586c 100644 --- a/web/moduleMapper/templates.ts +++ b/web/moduleMapper/templates.ts @@ -1,7 +1,9 @@ +// File only used in tests or dev console + import { K8sResourceKind } from '@openshift-console/dynamic-plugin-sdk'; import { safeYAMLToJS } from '../src/utils/yaml'; -export const FlowCollector = ` +const flowCollector = ` apiVersion: flows.netobserv.io/v1beta2 kind: FlowCollector metadata: @@ -123,60 +125,15 @@ spec: `; let flowCollectorJS: K8sResourceKind | null = null; -export const GetFlowCollectorJS = (): K8sResourceKind => { +export const getFlowCollectorJS = (): K8sResourceKind => { if (flowCollectorJS === null) { - flowCollectorJS = safeYAMLToJS(FlowCollector); + flowCollectorJS = safeYAMLToJS(flowCollector); } return flowCollectorJS!; }; -export const FlowMetric = ` -apiVersion: flows.netobserv.io/v1alpha1 -kind: FlowMetric -metadata: - labels: - app.kubernetes.io/name: flowmetric - app.kubernetes.io/instance: flowmetric-sample - app.kubernetes.io/part-of: netobserv-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: netobserv-operator - name: flowmetric-sample - namespace: netobserv -spec: - metricName: cluster_external_ingress_bytes_total - type: Counter - valueField: Bytes - direction: Ingress - labels: - - DstK8S_HostName - - DstK8S_Namespace - - DstK8S_OwnerName - - DstK8S_OwnerType - filters: - - field: SrcSubnetLabel - matchType: Absence - charts: - - dashboardName: Main - title: External ingress traffic - unit: Bps - type: SingleStat - queries: - - promQL: 'sum(rate($METRIC[2m]))' - legend: '' - - dashboardName: Main - sectionName: External - title: Top external ingress traffic per workload - unit: Bps - type: StackArea - queries: - - promQL: >- - sum(rate($METRIC{DstK8S_Namespace!=""}[2m])) by (DstK8S_Namespace, - DstK8S_OwnerName) - legend: '{{DstK8S_Namespace}} / {{DstK8S_OwnerName}}' -`; - // use an alternative sample for forms to avoid forcing the user to remove the filters / queries -export const FlowMetricDefaultForm = ` +export const flowMetricDefaultForm = ` apiVersion: flows.netobserv.io/v1alpha1 kind: FlowMetric metadata: @@ -189,9 +146,9 @@ spec: `; let flowMetricJS: K8sResourceKind | null = null; -export const GetFlowMetricJS = (): K8sResourceKind => { +export const getFlowMetricJS = (): K8sResourceKind => { if (flowMetricJS === null) { - flowMetricJS = safeYAMLToJS(FlowMetricDefaultForm); + flowMetricJS = safeYAMLToJS(flowMetricDefaultForm); } return flowMetricJS!; }; diff --git a/web/src/components/forms/config/uiSchema.ts b/web/src/components/forms/config/uiSchema.ts index d0ffc10f0..0f27be471 100644 --- a/web/src/components/forms/config/uiSchema.ts +++ b/web/src/components/forms/config/uiSchema.ts @@ -460,16 +460,7 @@ export const FlowCollectorUISchema: UiSchema = { processor: { 'ui:title': 'Processor configuration', filters: { - 'ui:title': 'Filters', - 'ui:widget': 'hidden', - items: { - 'ui:order': ['allOf', 'outputTarget', 'sampling', '*'], - allOf: { - items: { - 'ui:order': ['field', 'matchType', 'value', '*'] - } - } - } + 'ui:widget': 'hidden' }, multiClusterDeployment: { 'ui:title': 'Multi-cluster deployment' @@ -518,6 +509,12 @@ export const FlowCollectorUISchema: UiSchema = { }, 'ui:order': ['mode', 'sampling', '*'] }, + unmanagedReplicas: { + 'ui:title': 'Unmanaged replicas' + }, + consumerReplicas: { + 'ui:title': 'Consumer replicas' + }, kafkaConsumerQueueCapacity: { 'ui:title': 'Kafka consumer queue capacity', 'ui:dependency': { @@ -527,92 +524,10 @@ export const FlowCollectorUISchema: UiSchema = { } }, kafkaConsumerAutoscaler: { - 'ui:title': 'kafka consumer autoscaler', - 'ui:dependency': { - controlFieldPath: ['deploymentModel'], - controlFieldValue: 'Kafka', - controlFieldName: 'deploymentModel' - }, - 'ui:order': ['maxReplicas', 'metrics', 'minReplicas', 'status', '*'], - metrics: { - items: { - 'ui:order': ['type', 'containerResource', 'external', 'object', 'pods', 'resource', '*'], - containerResource: { - 'ui:order': ['container', 'name', 'target', '*'], - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - external: { - 'ui:order': ['metric', 'target', '*'], - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - object: { - 'ui:order': ['describedObject', 'metric', 'target', '*'], - describedObject: { - 'ui:order': ['kind', 'name', 'apiVersion', '*'] - }, - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - pods: { - 'ui:order': ['metric', 'target', '*'], - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - resource: { - 'ui:order': ['name', 'target', '*'], - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - } - } - } + 'ui:widget': 'hidden' }, kafkaConsumerReplicas: { - 'ui:title': 'Kafka consumer replicas', - 'ui:dependency': { - controlFieldPath: ['deploymentModel'], - controlFieldValue: 'Kafka', - controlFieldName: 'deploymentModel' - } + 'ui:widget': 'hidden' }, kafkaConsumerBatchSize: { 'ui:title': 'Kafka consumer batch size', @@ -907,37 +822,24 @@ export const FlowCollectorUISchema: UiSchema = { conversationHeartbeatInterval: { 'ui:widget': 'hidden' }, - 'ui:order': [ - 'port', - 'conversationTerminatingTimeout', - 'conversationEndTimeout', - 'profilePort', - 'env', - 'enableKubeProbes', - 'scheduling', - 'secondaryNetworks', - 'healthPort', - 'dropUnusedFields', - 'conversationHeartbeatInterval' - ] + 'ui:order': ['secondaryNetworks', '*'] }, 'ui:order': [ + 'addZone', 'filters', + 'metrics', 'multiClusterDeployment', 'clusterName', - 'addZone', 'subnetLabels', - 'logTypes', - 'logLevel', - 'imagePullPolicy', 'deduper', - 'kafkaConsumerReplicas', - 'kafkaConsumerAutoscaler', + 'unmanagedReplicas', + 'consumerReplicas', 'kafkaConsumerQueueCapacity', 'kafkaConsumerBatchSize', - 'metrics', - 'resources', - 'advanced' + 'logLevel', + 'imagePullPolicy', + 'advanced', + '*' ] }, prometheus: { @@ -1236,81 +1138,11 @@ export const FlowCollectorUISchema: UiSchema = { replicas: { 'ui:title': 'Replicas' }, + unmanagedReplicas: { + 'ui:title': 'Unmanaged replicas' + }, autoscaler: { - 'ui:title': 'Horizontal pod autoscaler', - 'ui:widget': 'hidden', - 'ui:order': ['maxReplicas', 'metrics', 'minReplicas', 'status', '*'], - metrics: { - items: { - 'ui:order': ['type', 'containerResource', 'external', 'object', 'pods', 'resource', '*'], - containerResource: { - 'ui:order': ['container', 'name', 'target', '*'], - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - external: { - 'ui:order': ['metric', 'target', '*'], - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - object: { - 'ui:order': ['describedObject', 'metric', 'target', '*'], - describedObject: { - 'ui:order': ['kind', 'name', 'apiVersion', '*'] - }, - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - pods: { - 'ui:order': ['metric', 'target', '*'], - metric: { - 'ui:order': ['name', 'selector', '*'], - selector: { - 'ui:order': ['matchExpressions', 'matchLabels', '*'], - matchExpressions: { - items: { - 'ui:order': ['key', 'operator', 'values', '*'] - } - } - } - }, - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - }, - resource: { - 'ui:order': ['name', 'target', '*'], - target: { - 'ui:order': ['type', 'averageUtilization', 'averageValue', 'value', '*'] - } - } - } - } + 'ui:widget': 'hidden' }, advanced: { 'ui:title': 'Advanced configuration', @@ -1506,10 +1338,11 @@ export const FlowCollectorUISchema: UiSchema = { 'imagePullPolicy', 'portNaming', 'quickFilters', + 'unmanagedReplicas', 'replicas', - 'autoscaler', 'resources', - 'advanced' + 'advanced', + '*' ] }, networkPolicy: { diff --git a/web/src/components/forms/flowCollector-wizard.tsx b/web/src/components/forms/flowCollector-wizard.tsx index 1180ec46e..fb515416a 100644 --- a/web/src/components/forms/flowCollector-wizard.tsx +++ b/web/src/components/forms/flowCollector-wizard.tsx @@ -73,7 +73,8 @@ export const FlowCollectorWizard: FC = props => { 'spec.agent.ebpf.privileged', 'spec.agent.ebpf.features', 'spec.processor.clusterName', - 'spec.processor.addZone' + 'spec.processor.addZone', + 'spec.processor.consumerReplicas' ]); break; case 'loki':