NETOBSERV-2429: add UI schema for consumer replicas

- Add unmanagedReplicas / consumerReplicas
- Hide autoscalers & deprecated settings
- Some reordering in processor schema
- Add a note in README describing what should be done when updating CRDs

Author: jotak

README.md

@@ -178,3 +178,19 @@ make cypress
 ```
 
 4. Click on "Run N integration specs"
+
+## Updating schemas
+
+This console plugin comes with several panels allowing GUI-based configuration for `FlowCollector` and other managed resources, including:
+
+- Comprehensive forms for each of the resources (includes most supported settings that are pretty common to configure).
+- Simplified wizards for faster configuration.
+
+When you update the operator CRDs, you may have to update some schemas here as well, which contain some rules that drive how forms are displayed. Especially:
+
+- [uiSchema.ts](./web/src/components/forms/config/uiSchema.ts) contains a display-oriented description of every CRD field, including whether or not they should be hidden, or if they have relationships with other fields.
+- [< CRD name >-wizard.tsx](./web/src/components/forms/flowCollector-wizard.tsx) contains specific fields to be displayed in wizards.
+
+When a CRD field is added, consider whether you need to update these files.
+
+Additionally, [schema.ts](./web/moduleMapper/schemas.ts) contains the full CRD schema as JSON, used in tests and in standalone mode, and should also be kept up to date.

web/moduleMapper/dummy.tsx

@@ -1,3 +1,5 @@
+// File only used in tests or dev console
+
 /* eslint-disable @typescript-eslint/no-unused-vars */
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import {
@@ -18,8 +20,8 @@ import { useK8sModelsWithColors } from '../src/utils/k8s-models-hook';
 import { useTheme } from '../src/utils/theme-hook';
 import { safeJSToYAML } from '../src/utils/yaml';
 import { k8sModels } from './k8s-models';
-import { FlowCollectorSchema, FlowMetricSchema } from './schemas';
-import { GetFlowCollectorJS, GetFlowMetricJS } from './templates';
+import { flowCollectorSchema, flowMetricSchema } from './schemas';
+import { getFlowCollectorJS, getFlowMetricJS } from './templates';
 
 // This dummy file is used to resolve @Console imports from @openshift-console for JEST / Standalone
 // You can add any exports needed here
@@ -130,7 +132,7 @@ export function useK8sWatchResource(req: any) {
                     served: true,
                     storage: true,
                     schema: {
-                      openAPIV3Schema: FlowCollectorSchema,
+                      openAPIV3Schema: flowCollectorSchema,
                     }
                   }]
                 }
@@ -154,15 +156,15 @@ export function useK8sWatchResource(req: any) {
                     served: true,
                     storage: true,
                     schema: {
-                      openAPIV3Schema: FlowMetricSchema
+                      openAPIV3Schema: flowMetricSchema
                     }
                   }]
                 }
               });
             }
             break;
           case 'FlowCollector':
-            const fc = _.cloneDeep(GetFlowCollectorJS());
+            const fc = _.cloneDeep(getFlowCollectorJS());
             fc.spec!.loki.enable = false;
             fc.spec!.exporters = [{ type: "Kafka" }, { type: "OpenTelemetry" }]
             fc.status = {
@@ -236,7 +238,7 @@ export function useK8sWatchResource(req: any) {
             break;
           case 'FlowMetric':
             if (req.name === 'flowmetric-sample') {
-              const fm = _.cloneDeep(GetFlowMetricJS());
+              const fm = _.cloneDeep(getFlowMetricJS());
               fm.spec!.metricName = 'test_metric';
               setResource(fm);
             }

web/moduleMapper/k8s-models.ts

@@ -1,4 +1,6 @@
-//these values comes from original useK8sModels using debug
+// File only used in tests or dev console
+
+// These values come from original useK8sModels using debug
 export const k8sModels = {
   "ImageStreamImport": {
     "label": "ImageStreamImport",

web/moduleMapper/schemas.ts

@@ -1,8 +1,11 @@
+// File only used in tests or dev console
+
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable max-len */
 import { RJSFSchema } from '@rjsf/utils';
 
-export const FlowCollectorSchema: RJSFSchema | any = {
+// flowCollectorSchema is only used in tests or dev console
+export const flowCollectorSchema: RJSFSchema | any = {
   title: 'FlowCollector',
   description:
     'The schema for the network flows collection API, which pilots and configures the underlying deployments.',
@@ -51,10 +54,10 @@ export const FlowCollectorSchema: RJSFSchema | any = {
         },
         deploymentModel: {
           description:
-            '`deploymentModel` defines the desired type of deployment for flow processing. Possible values are:\n- `Direct` (default) to make the flow processor listen directly from the agents.\n- `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.\nKafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).',
+            '`deploymentModel` defines the desired type of deployment for flow processing. Possible values are:\n - `Direct` (default) to make the flow processor listen directly from the agents using the host network, backed by a DaemonSet.\n - `Service` to make the flow processor listen as a Kubernetes Service, backed by a scalable Deployment.\n - `Kafka` to make flows sent to a Kafka pipeline before consumption by the processor.\n Kafka can provide better scalability, resiliency, and high availability (for more details, see https://www.redhat.com/en/topics/integration/what-is-apache-kafka).<br> `Direct` is not recommended on large clusters as it is less memory efficient.',
           type: 'string',
           default: 'Direct',
-          enum: ['Direct', 'Kafka']
+          enum: ['Direct', 'Service', 'Kafka']
         },
         kafka: {
           description:
@@ -3329,9 +3332,21 @@ export const FlowCollectorSchema: RJSFSchema | any = {
               default: 'Flows',
               enum: ['Flows', 'Conversations', 'EndedConversations', 'All']
             },
+            unmanagedReplicas: {
+              description:
+                'If `unmanagedReplicas` is `true`, the operator will not reconcile `consumerReplicas`. This is useful when using a pod autoscaler.',
+              type: 'boolean'
+            },
+            consumerReplicas: {
+              description:
+                '`consumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline`, default is 3. This setting is ignored when `spec.deploymentModel` is `Direct` or when `spec.processor.unmanagedReplicas` is `true`.',
+              type: 'integer',
+              format: 'int32',
+              minimum: 0
+            },
             kafkaConsumerReplicas: {
               description:
-                '`kafkaConsumerReplicas` defines the number of replicas (pods) to start for `flowlogs-pipeline-transformer`, which consumes Kafka messages.\nThis setting is ignored when Kafka is disabled.',
+                '`kafkaConsumerAutoscaler` [deprecated (*)] is the spec of a horizontal pod autoscaler to set up for `flowlogs-pipeline-transformer`, which consumes Kafka messages. This setting is ignored when Kafka is disabled. Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.processor.unmanagedReplicas` to `true`.',
               type: 'integer',
               format: 'int32',
               default: 3,
@@ -5127,7 +5142,7 @@ export const FlowCollectorSchema: RJSFSchema | any = {
               enum: ['IfNotPresent', 'Always', 'Never']
             },
             autoscaler: {
-              description: '`autoscaler` spec of a horizontal pod autoscaler to set up for the plugin Deployment.',
+              description: '`autoscaler` [deprecated (*)] spec of a horizontal pod autoscaler to set up for the plugin Deployment. Deprecation notice: managed autoscaler will be removed in a future version. You may configure instead an autoscaler of your choice, and set `spec.consolePlugin.unmanagedReplicas` to `true`.',
               type: 'object',
               properties: {
                 maxReplicas: {
@@ -5547,6 +5562,10 @@ export const FlowCollectorSchema: RJSFSchema | any = {
                 }
               }
             },
+            unmanagedReplicas: {
+              description: 'If `unmanagedReplicas` is `true`, the operator will not reconcile `replicas`. This is useful when using a pod autoscaler.',
+              type: 'boolean',
+            },
             replicas: {
               description: '`replicas` defines the number of replicas (pods) to start.',
               type: 'integer',
@@ -5936,7 +5955,8 @@ export const FlowCollectorSchema: RJSFSchema | any = {
   }
 };
 
-export const FlowMetricSchema: RJSFSchema | any = {
+// flowMetricSchema is only used in tests or dev console
+export const flowMetricSchema: RJSFSchema | any = {
   title: 'FlowMetric',
   description: 'The API allowing to create custom metrics from the collected flow logs.',
   type: 'object',
@@ -6127,4 +6147,4 @@ export const FlowMetricSchema: RJSFSchema | any = {
       }
     }
   }
-};
+};

web/moduleMapper/templates.ts

@@ -1,7 +1,9 @@
+// File only used in tests or dev console
+
 import { K8sResourceKind } from '@openshift-console/dynamic-plugin-sdk';
 import { safeYAMLToJS } from '../src/utils/yaml';
 
-export const FlowCollector = `
+const flowCollector = `
 apiVersion: flows.netobserv.io/v1beta2
 kind: FlowCollector
 metadata:
@@ -123,60 +125,15 @@ spec:
 `;
 
 let flowCollectorJS: K8sResourceKind | null = null;
-export const GetFlowCollectorJS = (): K8sResourceKind => {
+export const getFlowCollectorJS = (): K8sResourceKind => {
   if (flowCollectorJS === null) {
-    flowCollectorJS = safeYAMLToJS(FlowCollector);
+    flowCollectorJS = safeYAMLToJS(flowCollector);
   }
   return flowCollectorJS!;
 };
 
-export const FlowMetric = `
-apiVersion: flows.netobserv.io/v1alpha1
-kind: FlowMetric
-metadata:
-  labels:
-    app.kubernetes.io/name: flowmetric
-    app.kubernetes.io/instance: flowmetric-sample
-    app.kubernetes.io/part-of: netobserv-operator
-    app.kubernetes.io/managed-by: kustomize
-    app.kubernetes.io/created-by: netobserv-operator
-  name: flowmetric-sample
-  namespace: netobserv
-spec:
-  metricName: cluster_external_ingress_bytes_total
-  type: Counter
-  valueField: Bytes
-  direction: Ingress
-  labels:
-    - DstK8S_HostName
-    - DstK8S_Namespace
-    - DstK8S_OwnerName
-    - DstK8S_OwnerType
-  filters:
-    - field: SrcSubnetLabel
-      matchType: Absence
-  charts:
-    - dashboardName: Main
-      title: External ingress traffic
-      unit: Bps
-      type: SingleStat
-      queries:
-        - promQL: 'sum(rate($METRIC[2m]))'
-          legend: ''
-    - dashboardName: Main
-      sectionName: External
-      title: Top external ingress traffic per workload
-      unit: Bps
-      type: StackArea
-      queries:
-        - promQL: >-
-            sum(rate($METRIC{DstK8S_Namespace!=""}[2m])) by (DstK8S_Namespace,
-            DstK8S_OwnerName)
-          legend: '{{DstK8S_Namespace}} / {{DstK8S_OwnerName}}'
-`;
-
 // use an alternative sample for forms to avoid forcing the user to remove the filters / queries
-export const FlowMetricDefaultForm = `
+export const flowMetricDefaultForm = `
 apiVersion: flows.netobserv.io/v1alpha1
 kind: FlowMetric
 metadata:
@@ -189,9 +146,9 @@ spec:
 `;
 
 let flowMetricJS: K8sResourceKind | null = null;
-export const GetFlowMetricJS = (): K8sResourceKind => {
+export const getFlowMetricJS = (): K8sResourceKind => {
   if (flowMetricJS === null) {
-    flowMetricJS = safeYAMLToJS(FlowMetricDefaultForm);
+    flowMetricJS = safeYAMLToJS(flowMetricDefaultForm);
   }
   return flowMetricJS!;
 };