[8.x] [Streams] Schema Editor advanced field mapping options (#210667) (#211220)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Streams] Schema Editor advanced field mapping options
(#210667)](#210667)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kerry
Gallagher","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-02-14T13:52:18Z","message":"[Streams]
Schema Editor advanced field mapping options (#210667)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/streams-program/issues/88\r\n\r\nAdds JSON
advanced field mapping parameters to the Schema Editor.\r\n\r\nMain
questions here are around the types and data structure. In this
PR\r\nthese are added as an `additionalProperties` property, but we may
also\r\nwant to have all of these parameters top level (like `type`
and\r\n`format`). This version makes separating concerns easier in the
UI and\r\nseparating \"first class\" options vs advanced options, I
could see pros\r\nand cons to both, and also things might be
\"upgraded\" from advanced to\r\nfirst class later on. Also an open
question on whether the\r\n`MappingProperty` type needs to be explicitly
redefined for Zod (ES will\r\nobviously reject anything that isn't
supported here).\r\n\r\n![Screenshot 2025-02-12 at 13
01\r\n35](https://github.com/user-attachments/assets/7082fed7-f445-476f-abb7-8f41d693d378)\r\n\r\n\r\n![json_params](https://github.com/user-attachments/assets/521df9bf-cbd4-468a-9385-5787cdd5f906)","sha":"c9dfe9aab1ac2146f3c6ec3810ad35f8b753e6c3","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:version","Feature:Streams","v9.1.0","v8.19.0"],"title":"[Streams]
Schema Editor advanced field mapping
options","number":210667,"url":"https://github.com/elastic/kibana/pull/210667","mergeCommit":{"message":"[Streams]
Schema Editor advanced field mapping options (#210667)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/streams-program/issues/88\r\n\r\nAdds JSON
advanced field mapping parameters to the Schema Editor.\r\n\r\nMain
questions here are around the types and data structure. In this
PR\r\nthese are added as an `additionalProperties` property, but we may
also\r\nwant to have all of these parameters top level (like `type`
and\r\n`format`). This version makes separating concerns easier in the
UI and\r\nseparating \"first class\" options vs advanced options, I
could see pros\r\nand cons to both, and also things might be
\"upgraded\" from advanced to\r\nfirst class later on. Also an open
question on whether the\r\n`MappingProperty` type needs to be explicitly
redefined for Zod (ES will\r\nobviously reject anything that isn't
supported here).\r\n\r\n![Screenshot 2025-02-12 at 13
01\r\n35](https://github.com/user-attachments/assets/7082fed7-f445-476f-abb7-8f41d693d378)\r\n\r\n\r\n![json_params](https://github.com/user-attachments/assets/521df9bf-cbd4-468a-9385-5787cdd5f906)","sha":"c9dfe9aab1ac2146f3c6ec3810ad35f8b753e6c3"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/210667","number":210667,"mergeCommit":{"message":"[Streams]
Schema Editor advanced field mapping options (#210667)\n\n##
Summary\r\n\r\nCloses
https://github.com/elastic/streams-program/issues/88\r\n\r\nAdds JSON
advanced field mapping parameters to the Schema Editor.\r\n\r\nMain
questions here are around the types and data structure. In this
PR\r\nthese are added as an `additionalProperties` property, but we may
also\r\nwant to have all of these parameters top level (like `type`
and\r\n`format`). This version makes separating concerns easier in the
UI and\r\nseparating \"first class\" options vs advanced options, I
could see pros\r\nand cons to both, and also things might be
\"upgraded\" from advanced to\r\nfirst class later on. Also an open
question on whether the\r\n`MappingProperty` type needs to be explicitly
redefined for Zod (ES will\r\nobviously reject anything that isn't
supported here).\r\n\r\n![Screenshot 2025-02-12 at 13
01\r\n35](https://github.com/user-attachments/assets/7082fed7-f445-476f-abb7-8f41d693d378)\r\n\r\n\r\n![json_params](https://github.com/user-attachments/assets/521df9bf-cbd4-468a-9385-5787cdd5f906)","sha":"c9dfe9aab1ac2146f3c6ec3810ad35f8b753e6c3"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Kerry Gallagher <[email protected]>

Author: kibanamachine

x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { omit } from 'lodash';
+import { FieldDefinitionConfig } from '../models';
+
+// Parameters that we consider first class and provide a curated experience for
+const FIRST_CLASS_PARAMETERS = ['type', 'format'];
+
+// Advanced parameters that we provide a generic experience (JSON blob) for
+export const getAdvancedParameters = (fieldName: string, fieldConfig: FieldDefinitionConfig) => {
+  // @timestamp can't ignore malformed dates as it's used for sorting in logsdb
+  const additionalOmissions = fieldName === '@timestamp' ? ['ignore_malformed'] : [];
+  return omit(fieldConfig, FIRST_CLASS_PARAMETERS.concat(additionalOmissions));
+};

x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts

@@ -10,3 +10,4 @@ export * from './hierarchy';
 export * from './lifecycle';
 export * from './condition_fields';
 export * from './condition_to_query_dsl';
+export * from './field_definition';

x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
 import { z } from '@kbn/zod';
 import { NonEmptyString } from '@kbn/zod-helpers';
 
@@ -20,15 +21,25 @@ export const FIELD_DEFINITION_TYPES = [
 
 export type FieldDefinitionType = (typeof FIELD_DEFINITION_TYPES)[number];
 
-export interface FieldDefinitionConfig {
+// We redefine "first class" parameters
+export type FieldDefinitionConfig = MappingProperty & {
   type: FieldDefinitionType;
   format?: string;
-}
+};
+
+// Parameters that we provide a generic (JSON blob) experience for
+export type FieldDefinitionConfigAdvancedParameters = Omit<
+  FieldDefinitionConfig,
+  'type' | 'format'
+>;
 
-export const fieldDefinitionConfigSchema: z.Schema<FieldDefinitionConfig> = z.object({
-  type: z.enum(FIELD_DEFINITION_TYPES),
-  format: z.optional(NonEmptyString),
-});
+export const fieldDefinitionConfigSchema: z.Schema<FieldDefinitionConfig> = z.intersection(
+  z.record(z.string(), z.unknown()),
+  z.object({
+    type: z.enum(FIELD_DEFINITION_TYPES),
+    format: z.optional(NonEmptyString),
+  })
+);
 
 export interface FieldDefinition {
   [x: string]: FieldDefinitionConfig;
@@ -39,9 +50,9 @@ export const fieldDefinitionSchema: z.Schema<FieldDefinition> = z.record(
   fieldDefinitionConfigSchema
 );
 
-export interface InheritedFieldDefinitionConfig extends FieldDefinitionConfig {
+export type InheritedFieldDefinitionConfig = FieldDefinitionConfig & {
   from: string;
-}
+};
 
 export interface InheritedFieldDefinition {
   [x: string]: InheritedFieldDefinitionConfig;
@@ -52,9 +63,9 @@ export const inheritedFieldDefinitionSchema: z.Schema<InheritedFieldDefinition>
   z.intersection(fieldDefinitionConfigSchema, z.object({ from: NonEmptyString }))
 );
 
-export interface NamedFieldDefinitionConfig extends FieldDefinitionConfig {
+export type NamedFieldDefinitionConfig = FieldDefinitionConfig & {
   name: string;
-}
+};
 
 export const namedFieldDefinitionConfigSchema: z.Schema<NamedFieldDefinitionConfig> =
   z.intersection(

x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts

@@ -10,7 +10,13 @@ import {
   MappingDateProperty,
   MappingProperty,
 } from '@elastic/elasticsearch/lib/api/types';
-import { WiredStreamDefinition, isDslLifecycle, isIlmLifecycle, isRoot } from '@kbn/streams-schema';
+import {
+  WiredStreamDefinition,
+  getAdvancedParameters,
+  isDslLifecycle,
+  isIlmLifecycle,
+  isRoot,
+} from '@kbn/streams-schema';
 import { ASSET_VERSION } from '../../../../common/constants';
 import { logsSettings } from './logs_layer';
 import { getComponentTemplateName } from './name';
@@ -25,13 +31,21 @@ export function generateLayer(
     const property: MappingProperty = {
       type: props.type,
     };
+
+    const advancedParameters = getAdvancedParameters(field, props);
+
+    if (Object.keys(advancedParameters).length > 0) {
+      Object.assign(property, advancedParameters);
+    }
+
     if (field === '@timestamp') {
       // @timestamp can't ignore malformed dates as it's used for sorting in logsdb
       (property as MappingDateProperty).ignore_malformed = false;
     }
     if (props.type === 'date' && props.format) {
       (property as MappingDateProperty).format = props.format;
     }
+
     properties[field] = property;
   });
 

x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts

@@ -165,7 +165,10 @@ export const schemaFieldsSimulationRoute = createServerRoute({
     const propertiesForSimulation = Object.fromEntries(
       params.body.field_definitions.map((field) => [
         field.name,
-        { type: field.type, ...(field.format ? { format: field.format } : {}) },
+        {
+          type: field.type,
+          ...(field.format ? { format: field.format } : {}),
+        },
       ])
     );
 

x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx

@@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  EuiAccordion,
+  EuiCodeBlock,
+  EuiLink,
+  EuiPanel,
+  EuiSpacer,
+  EuiText,
+  useGeneratedHtmlId,
+} from '@elastic/eui';
+import { CodeEditor } from '@kbn/code-editor';
+import { i18n } from '@kbn/i18n';
+import React, { useMemo } from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { getAdvancedParameters } from '@kbn/streams-schema';
+import { SchemaField } from '../types';
+import { useKibana } from '../../../hooks/use_kibana';
+
+const label = i18n.translate('xpack.streams.advancedFieldMappingOptions.label', {
+  defaultMessage: 'Advanced field mapping parameters',
+});
+
+export const AdvancedFieldMappingOptions = ({
+  field,
+  onChange,
+  isEditing,
+}: {
+  field: SchemaField;
+  onChange: (field: Partial<SchemaField>) => void;
+  isEditing: boolean;
+}) => {
+  const { core } = useKibana();
+
+  const accordionId = useGeneratedHtmlId({ prefix: 'accordionID' });
+
+  const jsonOptions = useMemo(
+    () => (field.additionalParameters ? JSON.stringify(field.additionalParameters, null, 2) : ''),
+    [field.additionalParameters]
+  );
+
+  return (
+    <EuiAccordion id={accordionId} buttonContent={label}>
+      <EuiPanel color="subdued">
+        <EuiText size="xs">
+          <FormattedMessage
+            id="xpack.streams.advancedFieldMappingOptions.docs.label"
+            defaultMessage="Parameters can be defined with JSON. {link}"
+            values={{
+              link: (
+                <EuiLink
+                  data-test-subj="streamsAppAdvancedFieldMappingOptionsViewDocumentationLink"
+                  href={core.docLinks.links.elasticsearch.docsBase.concat('mapping-params.html')}
+                  target="_blank"
+                  external
+                >
+                  <FormattedMessage
+                    id="xpack.streams.indexPattern.randomSampling.learnMore"
+                    defaultMessage="View documentation."
+                  />
+                </EuiLink>
+              ),
+            }}
+          />
+        </EuiText>
+        <EuiSpacer size="s" />
+        {isEditing ? (
+          <CodeEditor
+            height={120}
+            languageId="json"
+            value={jsonOptions}
+            onChange={(value) => {
+              try {
+                onChange({
+                  additionalParameters:
+                    value === '' ? undefined : getAdvancedParameters(field.name, JSON.parse(value)),
+                });
+              } catch (error: unknown) {
+                // do nothing
+              }
+            }}
+          />
+        ) : (
+          <EuiCodeBlock language="json" isCopyable>
+            {jsonOptions ?? ''}
+          </EuiCodeBlock>
+        )}
+      </EuiPanel>
+    </EuiAccordion>
+  );
+};

x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx

@@ -16,7 +16,6 @@ import {
 } from '@elastic/eui';
 import React from 'react';
 import { i18n } from '@kbn/i18n';
-import useToggle from 'react-use/lib/useToggle';
 import { WiredStreamDefinition } from '@kbn/streams-schema';
 import { useStreamsAppRouter } from '../../../hooks/use_streams_app_router';
 import { FieldParent } from '../field_parent';
@@ -56,18 +55,17 @@ const FIELD_SUMMARIES = {
 
 interface FieldSummaryProps {
   field: SchemaField;
-  isEditingByDefault: boolean;
+  isEditing: boolean;
+  toggleEditMode: () => void;
   stream: WiredStreamDefinition;
   onChange: (field: Partial<SchemaField>) => void;
 }
 
 export const FieldSummary = (props: FieldSummaryProps) => {
-  const { field, isEditingByDefault, onChange, stream } = props;
+  const { field, isEditing, toggleEditMode, onChange, stream } = props;
 
   const router = useStreamsAppRouter();
 
-  const [isEditing, toggleEditMode] = useToggle(isEditingByDefault);
-
   return (
     <>
       <EuiFlexGroup direction="column" gutterSize="s">

x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx

@@ -19,9 +19,11 @@ import React, { useReducer } from 'react';
 import { i18n } from '@kbn/i18n';
 import { WiredStreamDefinition } from '@kbn/streams-schema';
 import useAsyncFn from 'react-use/lib/useAsyncFn';
+import useToggle from 'react-use/lib/useToggle';
 import { SamplePreviewTable } from './sample_preview_table';
 import { FieldSummary } from './field_summary';
 import { SchemaField } from '../types';
+import { AdvancedFieldMappingOptions } from './advanced_field_mapping_options';
 
 export interface SchemaEditorFlyoutProps {
   field: SchemaField;
@@ -40,6 +42,8 @@ export const SchemaEditorFlyout = ({
   isEditingByDefault = false,
   withFieldSimulation = false,
 }: SchemaEditorFlyoutProps) => {
+  const [isEditing, toggleEditMode] = useToggle(isEditingByDefault);
+
   const [nextField, setNextField] = useReducer(
     (prev: SchemaField, updated: Partial<SchemaField>) =>
       ({
@@ -66,10 +70,16 @@ export const SchemaEditorFlyout = ({
         <EuiFlexGroup direction="column">
           <FieldSummary
             field={nextField}
-            isEditingByDefault={isEditingByDefault}
+            isEditing={isEditing}
+            toggleEditMode={toggleEditMode}
             onChange={setNextField}
             stream={stream}
           />
+          <AdvancedFieldMappingOptions
+            field={nextField}
+            onChange={setNextField}
+            isEditing={isEditing}
+          />
           {withFieldSimulation && (
             <EuiFlexItem grow={false}>
               <SamplePreviewTable stream={stream} nextField={nextField} />

x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts

@@ -7,7 +7,11 @@
 
 import { i18n } from '@kbn/i18n';
 import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller';
-import { NamedFieldDefinitionConfig, WiredStreamGetResponse } from '@kbn/streams-schema';
+import {
+  NamedFieldDefinitionConfig,
+  WiredStreamGetResponse,
+  getAdvancedParameters,
+} from '@kbn/streams-schema';
 import { isEqual, omit } from 'lodash';
 import { useMemo, useCallback } from 'react';
 import { useStreamsAppFetch } from '../../../hooks/use_streams_app_fetch';
@@ -59,6 +63,7 @@ export const useSchemaFields = ({
         name,
         type: field.type,
         format: field.format,
+        additionalParameters: getAdvancedParameters(name, field),
         parent: field.from,
         status: 'inherited',
       })
@@ -69,6 +74,7 @@ export const useSchemaFields = ({
         name,
         type: field.type,
         format: field.format,
+        additionalParameters: getAdvancedParameters(name, field),
         parent: definition.stream.name,
         status: 'mapped',
       })
@@ -132,12 +138,12 @@ export const useSchemaFields = ({
 
         refreshFields();
       } catch (error) {
-        toasts.addError(error, {
+        toasts.addError(new Error(error.body.message), {
           title: i18n.translate('xpack.streams.streamDetailSchemaEditorEditErrorToast', {
             defaultMessage: 'Something went wrong editing the {field} field',
             values: { field: field.name },
           }),
-          toastMessage: error.message,
+          toastMessage: error.body.message,
           toastLifeTimeMs: 5000,
         });
       }

x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx

@@ -107,7 +107,7 @@ const createCellRenderer =
       return <FieldStatusBadge status={status} />;
     }
 
-    return field[columnId as keyof SchemaField] || EMPTY_CONTENT;
+    return <>{field[columnId as keyof SchemaField] || EMPTY_CONTENT}</>;
   };
 
 const createFieldActionsCellRenderer = (fields: SchemaField[]): EuiDataGridControlColumn => ({