From 391b86932a780eecc19e96cf240404df4e7e4ba5 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 23 May 2025 14:17:42 +0200 Subject: [PATCH 01/12] ilm for classic streams --- .../data_streams/manage_data_streams.ts | 15 ++++++++++++++- .../streams/unwired_stream.ts | 13 ------------- .../stream_detail_lifecycle/index.tsx | 19 ++++++++----------- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts index 7212adf9781ef..b9f138e058f07 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -151,7 +151,20 @@ export async function updateDataStreamsLifecycle({ const isIlm = isIlmLifecycle(lifecycle); for (const dataStream of dataStreams.data_streams) { - logger.debug(`updating settings for data stream ${dataStream.name} backing indices`); + logger.debug(`updating settings for data stream ${dataStream.name}`); + + await retryTransientEsErrors( + () => + esClient.transport.request({ + method: 'PUT', + path: `/_data_stream/${dataStream.name}/_settings`, + body: { + 'index.lifecycle.name': isIlm ? lifecycle.ilm.policy : null, + }, + }), + { logger } + ); + await retryTransientEsErrors( () => esClient.indices.putSettings({ diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts index 60f44cf6ed598..9c801a55871c2 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts @@ -134,19 +134,6 @@ export class UnwiredStream extends StreamActiveRecord { const actions: Array<{ name: string; action: LifecycleEditAction }> = []; - const isWired = Streams.WiredStream.GetResponse.is(definition); const isUnwired = Streams.UnwiredStream.GetResponse.is(definition); const isIlm = isIlmLifecycle(definition.effective_lifecycle); - if (isWired || (isUnwired && !isIlm)) { - actions.push({ - name: i18n.translate('xpack.streams.streamDetailLifecycle.setRetentionDays', { - defaultMessage: 'Set specific retention days', - }), - action: 'dsl', - }); - } + actions.push({ + name: i18n.translate('xpack.streams.streamDetailLifecycle.setRetentionDays', { + defaultMessage: 'Set specific retention days', + }), + action: 'dsl', + }); - if (isWired && !isServerless) { + if (!isServerless) { actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.setLifecyclePolicy', { defaultMessage: 'Use a lifecycle policy', @@ -54,7 +51,7 @@ function useLifecycleState({ }); } - if (!isRoot(definition.stream.name) || (isUnwired && !isIlm)) { + if (!isRoot(definition.stream.name) && !isUnwired) { actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.resetToDefault', { defaultMessage: 'Reset to default', From 532759b664144d36b0b90a03e128fa9dd8338d3a Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 26 May 2025 10:58:32 +0200 Subject: [PATCH 02/12] reset to default for unwired streams --- .../data_management/stream_detail_lifecycle/index.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx index 5b457b34fff27..a4084cc392962 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx @@ -32,8 +32,6 @@ function useLifecycleState({ const lifecycleActions = useMemo(() => { const actions: Array<{ name: string; action: LifecycleEditAction }> = []; - const isUnwired = Streams.UnwiredStream.GetResponse.is(definition); - const isIlm = isIlmLifecycle(definition.effective_lifecycle); actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.setRetentionDays', { @@ -51,7 +49,7 @@ function useLifecycleState({ }); } - if (!isRoot(definition.stream.name) && !isUnwired) { + if (!isRoot(definition.stream.name)) { actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.resetToDefault', { defaultMessage: 'Reset to default', From 8619062aa391e1bb0d067b5c16ff135f3f8d5189 Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 26 May 2025 11:00:49 +0200 Subject: [PATCH 03/12] api test --- .../apis/observability/streams/lifecycle.ts | 87 ++++++++++--------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts index b530ec5617e9c..88252ba799b2b 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts @@ -412,6 +412,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { queries: [], }; + let clean: () => Promise; + afterEach(() => clean?.()); + const createDataStream = async (name: string, lifecycle: IngestStreamLifecycle) => { await esClient.indices.putIndexTemplate({ name, @@ -437,7 +440,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); await esClient.indices.createDataStream({ name }); - return async () => { + clean = async () => { await esClient.indices.deleteDataStream({ name }); await esClient.indices.deleteIndexTemplate({ name }); }; @@ -445,21 +448,36 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { it('noop when inherit lifecycle', async () => { const indexName = 'unwired-stream-inherit'; - const clean = await createDataStream(indexName, { dsl: { data_retention: '77d' } }); + await createDataStream(indexName, { dsl: { data_retention: '77d' } }); - try { - await putStream(apiClient, indexName, unwiredPutBody); - await expectLifecycle([indexName], { dsl: { data_retention: '77d' } }); - } finally { - await clean(); - } + await putStream(apiClient, indexName, unwiredPutBody); + await expectLifecycle([indexName], { dsl: { data_retention: '77d' } }); }); it('overrides dsl retention', async () => { const indexName = 'unwired-stream-override-dsl'; - const clean = await createDataStream(indexName, { dsl: { data_retention: '77d' } }); + await createDataStream(indexName, { dsl: { data_retention: '77d' } }); + + await putStream(apiClient, indexName, { + dashboards: [], + queries: [], + stream: { + description: '', + ingest: { + ...unwiredPutBody.stream.ingest, + lifecycle: { dsl: { data_retention: '11d' } }, + }, + }, + }); + + await expectLifecycle([indexName], { dsl: { data_retention: '11d' } }); + }); + + if (!isServerless) { + it('updates from ilm to dsl', async () => { + const indexName = 'unwired-stream-ilm-to-dsl'; + await createDataStream(indexName, { ilm: { policy: 'my-policy' } }); - try { await putStream(apiClient, indexName, { dashboards: [], queries: [], @@ -467,42 +485,31 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { description: '', ingest: { ...unwiredPutBody.stream.ingest, - lifecycle: { dsl: { data_retention: '11d' } }, + lifecycle: { dsl: { data_retention: '1d' } }, }, }, }); - await expectLifecycle([indexName], { dsl: { data_retention: '11d' } }); - } finally { - await clean(); - } - }); + await expectLifecycle([indexName], { dsl: { data_retention: '1d' } }); + }); - if (!isServerless) { - it('does not allow dsl lifecycle if the data stream is managed by ilm', async () => { - const indexName = 'unwired-stream-ilm-to-dsl'; - const clean = await createDataStream(indexName, { ilm: { policy: 'my-policy' } }); - - try { - await putStream( - apiClient, - indexName, - { - dashboards: [], - queries: [], - stream: { - description: '', - ingest: { - ...unwiredPutBody.stream.ingest, - lifecycle: { dsl: { data_retention: '1d' } }, - }, - }, + it('updates from dsl to ilm', async () => { + const indexName = 'unwired-stream-dsl-to-ilm'; + await createDataStream(indexName, { dsl: { data_retention: '10d' } }); + + await putStream(apiClient, indexName, { + dashboards: [], + queries: [], + stream: { + description: '', + ingest: { + ...unwiredPutBody.stream.ingest, + lifecycle: { ilm: { policy: 'my-policy' } }, }, - 400 - ); - } finally { - await clean(); - } + }, + }); + + await expectLifecycle([indexName], { ilm: { policy: 'my-policy' } }); }); } }); From 8566f7f79738c463d4e468ebeb5bcc9617d2f163 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 26 May 2025 09:32:18 +0000 Subject: [PATCH 04/12] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../lib/streams/state_management/streams/unwired_stream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts index 9c801a55871c2..183c64eb930f4 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts @@ -10,7 +10,7 @@ import type { IngestProcessorContainer, } from '@elastic/elasticsearch/lib/api/types'; import type { IngestStreamLifecycle } from '@kbn/streams-schema'; -import { isDslLifecycle, isInheritLifecycle, Streams } from '@kbn/streams-schema'; +import { isInheritLifecycle, Streams } from '@kbn/streams-schema'; import _, { cloneDeep } from 'lodash'; import { isNotFoundError } from '@kbn/es-errors'; import { StatusError } from '../../errors/status_error'; From 0e2169ec8a22070669808320db5d0da66627b67a Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 28 May 2025 17:44:19 +0200 Subject: [PATCH 05/12] pass prefer_ilm to _settings --- .../data_streams/manage_data_streams.ts | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts index b9f138e058f07..afe2828d30198 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -138,45 +138,27 @@ export async function updateDataStreamsLifecycle({ { logger } ); - // if we transition from ilm to dlm or vice versa, the rolled over backing + // if we transition from ilm to dsl or vice versa, the rolled over backing // indices need to be updated or they'll retain the lifecycle configuration // set at the time of creation. - // this is not needed for serverless since only dlm is allowed but in stateful - // we update every indices while not always necessary. this should be optimized + // this is not needed for serverless since only dsl is allowed. if (isServerless) { return; } - const dataStreams = await esClient.indices.getDataStream({ name: names }); const isIlm = isIlmLifecycle(lifecycle); - - for (const dataStream of dataStreams.data_streams) { - logger.debug(`updating settings for data stream ${dataStream.name}`); - - await retryTransientEsErrors( - () => - esClient.transport.request({ - method: 'PUT', - path: `/_data_stream/${dataStream.name}/_settings`, - body: { - 'index.lifecycle.name': isIlm ? lifecycle.ilm.policy : null, - }, - }), - { logger } - ); - - await retryTransientEsErrors( - () => - esClient.indices.putSettings({ - index: dataStream.indices.map((index) => index.index_name), - settings: { - 'lifecycle.prefer_ilm': isIlm, - 'lifecycle.name': isIlm ? lifecycle.ilm.policy : null, - }, - }), - { logger } - ); - } + await retryTransientEsErrors( + () => + esClient.transport.request({ + method: 'PUT', + path: `/_data_stream/${names.join(',')}/_settings`, + body: { + 'index.lifecycle.name': isIlm ? lifecycle.ilm.policy : null, + 'index.lifecycle.prefer_ilm': isIlm, + }, + }), + { logger } + ); } catch (err: any) { logger.error(`Error updating data stream lifecycle: ${err.message}`); throw err; From e8f352bb7b34e1c0fe949f22977c52bb8b7a84dd Mon Sep 17 00:00:00 2001 From: klacabane Date: Wed, 28 May 2025 19:28:36 +0200 Subject: [PATCH 06/12] important lowercasing --- .../data_streams/manage_data_streams.ts | 1 + .../streams/unwired_stream.ts | 22 ++++++------- .../state_management/streams/wired_stream.ts | 32 +++++++++---------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts index afe2828d30198..2732470de9689 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/data_streams/manage_data_streams.ts @@ -149,6 +149,7 @@ export async function updateDataStreamsLifecycle({ const isIlm = isIlmLifecycle(lifecycle); await retryTransientEsErrors( () => + // TODO: use client method once available esClient.transport.request({ method: 'PUT', path: `/_data_stream/${names.join(',')}/_settings`, diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts index 183c64eb930f4..00736da2c9180 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts @@ -28,7 +28,7 @@ import { StreamActiveRecord, PrintableStream } from '../stream_active_record/str export class UnwiredStream extends StreamActiveRecord { private _processingChanged: boolean = false; - private _lifeCycleChanged: boolean = false; + private _lifecycleChanged: boolean = false; constructor(definition: Streams.UnwiredStream.Definition, dependencies: StateDependencies) { super(definition, dependencies); @@ -42,7 +42,7 @@ export class UnwiredStream extends StreamActiveRecord { // Check for conflicts - if (this._lifeCycleChanged || this._processingChanged) { + if (this._lifecycleChanged || this._processingChanged) { try { const dataStreamResult = await this.dependencies.scopedClusterClient.asCurrentUser.indices.getDataStream({ @@ -153,12 +153,12 @@ export class UnwiredStream extends StreamActiveRecord 0) { actions.push(...(await this.createUpsertPipelineActions())); } - if (!isInheritLifecycle(this.getLifeCycle())) { + if (!isInheritLifecycle(this.getLifecycle())) { actions.push({ type: 'update_lifecycle', request: { name: this._definition.name, - lifecycle: this.getLifeCycle(), + lifecycle: this.getLifecycle(), }, }); } @@ -169,11 +169,11 @@ export class UnwiredStream extends StreamActiveRecord { const actions: ElasticsearchAction[] = []; - if (this.hasChangedFields() || this.hasChangedLifeCycle()) { + if (this.hasChangedFields() || this.hasChangedLifecycle()) { actions.push({ type: 'upsert_component_template', request: generateLayer( @@ -539,22 +539,22 @@ export class WiredStream extends StreamActiveRecord Date: Mon, 2 Jun 2025 18:28:42 +0200 Subject: [PATCH 07/12] remove redundant template settings --- .../component_templates/generate_layer.ts | 56 +------------------ .../state_management/streams/wired_stream.ts | 2 +- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.ts index fe9002d366369..078cec9d85362 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.ts @@ -10,14 +10,7 @@ import { MappingDateProperty, MappingProperty, } from '@elastic/elasticsearch/lib/api/types'; -import { - Streams, - getAdvancedParameters, - isDslLifecycle, - isIlmLifecycle, - isRoot, - namespacePrefixes, -} from '@kbn/streams-schema'; +import { Streams, getAdvancedParameters, isRoot, namespacePrefixes } from '@kbn/streams-schema'; import { ASSET_VERSION } from '../../../../common/constants'; import { logsSettings } from './logs_layer'; import { getComponentTemplateName } from './name'; @@ -65,7 +58,6 @@ export function generateLayer( return { name: getComponentTemplateName(name), template: { - lifecycle: getTemplateLifecycle(definition, isServerless), settings: getTemplateSettings(definition, isServerless), mappings: { dynamic: false, @@ -85,53 +77,7 @@ export function generateLayer( }; } -function getTemplateLifecycle(definition: Streams.WiredStream.Definition, isServerless: boolean) { - const lifecycle = definition.ingest.lifecycle; - if (isServerless) { - // dlm cannot be disabled in serverless - return { - data_retention: isDslLifecycle(lifecycle) ? lifecycle.dsl.data_retention : undefined, - }; - } - - if (isIlmLifecycle(lifecycle)) { - return { enabled: false }; - } - - if (isDslLifecycle(lifecycle)) { - return { - enabled: true, - data_retention: lifecycle.dsl.data_retention, - }; - } - - return undefined; -} - function getTemplateSettings(definition: Streams.WiredStream.Definition, isServerless: boolean) { const baseSettings = isRoot(definition.name) ? logsSettings : {}; - const lifecycle = definition.ingest.lifecycle; - - if (isServerless) { - return baseSettings; - } - - if (isIlmLifecycle(lifecycle)) { - return { - ...baseSettings, - 'index.lifecycle.prefer_ilm': true, - 'index.lifecycle.name': lifecycle.ilm.policy, - }; - } - - if (isDslLifecycle(lifecycle)) { - return { - ...baseSettings, - 'index.lifecycle.prefer_ilm': false, - 'index.lifecycle.name': undefined, - }; - } - - // don't specify any lifecycle property when lifecyle is disabled or inherited return baseSettings; } diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/wired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/wired_stream.ts index 944cfc7f4d3bd..24a714bd7c809 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/wired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/wired_stream.ts @@ -500,7 +500,7 @@ export class WiredStream extends StreamActiveRecord { const actions: ElasticsearchAction[] = []; - if (this.hasChangedFields() || this.hasChangedLifecycle()) { + if (this.hasChangedFields()) { actions.push({ type: 'upsert_component_template', request: generateLayer( From 7b62adbf1144a3db05a7ede826906c42d73db382 Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 2 Jun 2025 21:07:56 +0200 Subject: [PATCH 08/12] fix tests --- .../component_templates/generate_layer.test.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.test.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.test.ts index 552196892503c..6ea527c666a32 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.test.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/component_templates/generate_layer.test.ts @@ -39,10 +39,6 @@ describe('generateLayer', () => { }, "name": "logs.abc@stream.layer", "template": Object { - "lifecycle": Object { - "data_retention": "30d", - "enabled": true, - }, "mappings": Object { "dynamic": false, "properties": Object { @@ -63,10 +59,7 @@ describe('generateLayer', () => { }, }, }, - "settings": Object { - "index.lifecycle.name": undefined, - "index.lifecycle.prefer_ilm": false, - }, + "settings": Object {}, }, "version": 1, } @@ -83,9 +76,6 @@ describe('generateLayer', () => { }, "name": "logs@stream.layer", "template": Object { - "lifecycle": Object { - "data_retention": "30d", - }, "mappings": Object { "dynamic": false, "properties": Object { From addc150106ab1aad9287e48b7f47d1588e6145d0 Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 2 Jun 2025 21:52:03 +0200 Subject: [PATCH 09/12] rename dlm --- .../apis/observability/streams/lifecycle.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts index 88252ba799b2b..4b10bf9fa2898 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts @@ -153,7 +153,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { ); }); - it('inherits dlm', async () => { + it('inherits dsl', async () => { // create two branches, one that inherits from root and // another one that overrides the root lifecycle await putStream(apiClient, 'logs.inherits.lifecycle', wiredPutBody); @@ -320,7 +320,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); }); - it('updates when transitioning from ilm to dlm', async () => { + it('updates when transitioning from ilm to dsl', async () => { const name = 'logs.ilm-with-backing-indices'; await putStream(apiClient, name, { dashboards: [], @@ -356,8 +356,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await expectLifecycle([name], { dsl: { data_retention: '7d' }, from: name }); }); - it('updates when transitioning from dlm to ilm', async () => { - const name = 'logs.dlm-with-backing-indices'; + it('updates when transitioning from dsl to ilm', async () => { + const name = 'logs.dsl-with-backing-indices'; await putStream(apiClient, name, { dashboards: [], queries: [], From 0c5ebb5b8f1e7b70bf4961f4c507ca5ec96e14b7 Mon Sep 17 00:00:00 2001 From: klacabane Date: Mon, 2 Jun 2025 22:35:45 +0200 Subject: [PATCH 10/12] fix unwired inherit --- .../state_management/streams/unwired_stream.ts | 2 +- .../stream_detail_lifecycle/index.tsx | 3 ++- .../apis/observability/streams/lifecycle.ts | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts index 00736da2c9180..815e267951603 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts @@ -210,7 +210,7 @@ export class UnwiredStream extends StreamActiveRecord { const actions: Array<{ name: string; action: LifecycleEditAction }> = []; + const isUnwired = Streams.UnwiredStream.GetResponse.is(definition); actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.setRetentionDays', { @@ -49,7 +50,7 @@ function useLifecycleState({ }); } - if (!isRoot(definition.stream.name)) { + if (isUnwired || !isRoot(definition.stream.name)) { actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.resetToDefault', { defaultMessage: 'Reset to default', diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts index 4b10bf9fa2898..d7625ad37b349 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts @@ -452,6 +452,23 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await putStream(apiClient, indexName, unwiredPutBody); await expectLifecycle([indexName], { dsl: { data_retention: '77d' } }); + + await putStream(apiClient, indexName, { + dashboards: [], + queries: [], + stream: { + description: '', + ingest: { + ...unwiredPutBody.stream.ingest, + lifecycle: { ilm: { policy: 'my-policy' } }, + }, + }, + }); + await expectLifecycle([indexName], { ilm: { policy: 'my-policy' } }); + + // lifecycle remains unchanged after going back to inherit + await putStream(apiClient, indexName, unwiredPutBody); + await expectLifecycle([indexName], { ilm: { policy: 'my-policy' } }); }); it('overrides dsl retention', async () => { From 996da44853a92657f0fb40f7d87a4b514411daf5 Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 3 Jun 2025 11:42:40 +0200 Subject: [PATCH 11/12] only use dsl for serverless tests --- .../apis/observability/streams/lifecycle.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts index d7625ad37b349..202695b0f6bd8 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts @@ -460,15 +460,15 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { description: '', ingest: { ...unwiredPutBody.stream.ingest, - lifecycle: { ilm: { policy: 'my-policy' } }, + lifecycle: { dsl: { data_retention: '2d' } }, }, }, }); - await expectLifecycle([indexName], { ilm: { policy: 'my-policy' } }); + await expectLifecycle([indexName], { dsl: { data_retention: '2d' } }); // lifecycle remains unchanged after going back to inherit await putStream(apiClient, indexName, unwiredPutBody); - await expectLifecycle([indexName], { ilm: { policy: 'my-policy' } }); + await expectLifecycle([indexName], { dsl: { data_retention: '2d' } }); }); it('overrides dsl retention', async () => { From 1d1ddc346ec0cb021ccc732a2d47d18628ad5dab Mon Sep 17 00:00:00 2001 From: klacabane Date: Tue, 3 Jun 2025 20:31:40 +0200 Subject: [PATCH 12/12] dont allow inherit after update --- .../streams/unwired_stream.ts | 18 ++++++++++- .../stream_detail_lifecycle/index.tsx | 4 +-- .../apis/observability/streams/lifecycle.ts | 32 ++++++++++++++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts index 815e267951603..c49d0f5c6ff93 100644 --- a/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts +++ b/x-pack/platform/plugins/shared/streams/server/lib/streams/state_management/streams/unwired_stream.ts @@ -10,7 +10,7 @@ import type { IngestProcessorContainer, } from '@elastic/elasticsearch/lib/api/types'; import type { IngestStreamLifecycle } from '@kbn/streams-schema'; -import { isInheritLifecycle, Streams } from '@kbn/streams-schema'; +import { isIlmLifecycle, isInheritLifecycle, Streams } from '@kbn/streams-schema'; import _, { cloneDeep } from 'lodash'; import { isNotFoundError } from '@kbn/es-errors'; import { StatusError } from '../../errors/status_error'; @@ -100,6 +100,22 @@ export class UnwiredStream extends StreamActiveRecord { + if (this.dependencies.isServerless && isIlmLifecycle(this.getLifecycle())) { + return { isValid: false, errors: [new Error('Using ILM is not supported in Serverless')] }; + } + + if ( + startingState.get(this._definition.name)?.definition && + this._lifecycleChanged && + isInheritLifecycle(this.getLifecycle()) + ) { + // temporary until https://github.com/elastic/kibana/issues/222440 is resolved + return { + isValid: false, + errors: [new Error('Cannot revert to default lifecycle once updated')], + }; + } + // Check for conflicts if (this._lifecycleChanged || this._processingChanged) { try { diff --git a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx index 6186222df66e1..5deb0991c1eb3 100644 --- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx +++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_lifecycle/index.tsx @@ -32,7 +32,7 @@ function useLifecycleState({ const lifecycleActions = useMemo(() => { const actions: Array<{ name: string; action: LifecycleEditAction }> = []; - const isUnwired = Streams.UnwiredStream.GetResponse.is(definition); + const isWired = Streams.WiredStream.GetResponse.is(definition); actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.setRetentionDays', { @@ -50,7 +50,7 @@ function useLifecycleState({ }); } - if (isUnwired || !isRoot(definition.stream.name)) { + if (isWired && !isRoot(definition.stream.name)) { actions.push({ name: i18n.translate('xpack.streams.streamDetailLifecycle.resetToDefault', { defaultMessage: 'Reset to default', diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts index 202695b0f6bd8..38d9ba888bc95 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/lifecycle.ts @@ -446,13 +446,15 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }; }; - it('noop when inherit lifecycle', async () => { + it('cannot update to inherit lifecycle', async () => { const indexName = 'unwired-stream-inherit'; await createDataStream(indexName, { dsl: { data_retention: '77d' } }); + // initially set to inherit which is a noop await putStream(apiClient, indexName, unwiredPutBody); await expectLifecycle([indexName], { dsl: { data_retention: '77d' } }); + // update to dsl await putStream(apiClient, indexName, { dashboards: [], queries: [], @@ -466,8 +468,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { }); await expectLifecycle([indexName], { dsl: { data_retention: '2d' } }); - // lifecycle remains unchanged after going back to inherit - await putStream(apiClient, indexName, unwiredPutBody); + // fail to set inherit + await putStream(apiClient, indexName, unwiredPutBody, 400); await expectLifecycle([indexName], { dsl: { data_retention: '2d' } }); }); @@ -490,7 +492,29 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { await expectLifecycle([indexName], { dsl: { data_retention: '11d' } }); }); - if (!isServerless) { + if (isServerless) { + it('does not support ilm', async () => { + const indexName = 'unwired-stream-no-ilm'; + await createDataStream(indexName, { dsl: { data_retention: '2d' } }); + + await putStream( + apiClient, + indexName, + { + dashboards: [], + queries: [], + stream: { + description: '', + ingest: { + ...wiredPutBody.stream.ingest, + lifecycle: { ilm: { policy: 'my-policy' } }, + }, + }, + }, + 400 + ); + }); + } else { it('updates from ilm to dsl', async () => { const indexName = 'unwired-stream-ilm-to-dsl'; await createDataStream(indexName, { ilm: { policy: 'my-policy' } });