Skip to content

Commit 6f09403

Browse files
kibanamachineflash1293
kibanamachine
and
flash1293
authored
[8.x] 🌊 Streams: Fix listing page for orphaned streams (elastic#217854) (elastic#218092)
# Backport This will backport the following commits from `main` to `8.x`: - [🌊 Streams: Fix listing page for orphaned streams (elastic#217854)](elastic#217854) <!--- Backport version: 9.6.6 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Joe Reuter","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-04-14T11:24:33Z","message":"🌊 Streams: Fix listing page for orphaned streams (elastic#217854)\n\nThe listing page didn't handle \"orphaned\" streams properly (classic data\nstreams which have a configuration on the stream level but the\nunderlying data stream is not available because it got deleted).\n\nThis PR fixes that and adds an integration test for it\n\n<img width=\"774\" alt=\"Screenshot 2025-04-10 at 16 07 15\"\nsrc=\"https://github.com/user-attachments/assets/da15c56b-7dbd-4070-ab6d-4235132da8ed\"\n/>\n\nIn this picture, `logs-test-default` is orphaned.\n\nTo test:\n* Create a new classic stream (e.g. via executing\n```\nPOST logs-mylogs-default/_doc\n{ \"message\": \"Test\" }\n```\n* Go into the streams UI and add a processor for this stream\n* Delete the data stream via stack management or via\n```\nDELETE _data_stream/logs-mylogs-default\n```\n* Go to the streams listing page","sha":"f3042efa8f22a8bfa680c33a4999d541639436ff","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team:obs-ux-logs","backport:version","Feature:Streams","v9.1.0","v8.19.0"],"title":"🌊 Streams: Fix listing page for orphaned streams","number":217854,"url":"https://github.com/elastic/kibana/pull/217854","mergeCommit":{"message":"🌊 Streams: Fix listing page for orphaned streams (elastic#217854)\n\nThe listing page didn't handle \"orphaned\" streams properly (classic data\nstreams which have a configuration on the stream level but the\nunderlying data stream is not available because it got deleted).\n\nThis PR fixes that and adds an integration test for it\n\n<img width=\"774\" alt=\"Screenshot 2025-04-10 at 16 07 15\"\nsrc=\"https://github.com/user-attachments/assets/da15c56b-7dbd-4070-ab6d-4235132da8ed\"\n/>\n\nIn this picture, `logs-test-default` is orphaned.\n\nTo test:\n* Create a new classic stream (e.g. via executing\n```\nPOST logs-mylogs-default/_doc\n{ \"message\": \"Test\" }\n```\n* Go into the streams UI and add a processor for this stream\n* Delete the data stream via stack management or via\n```\nDELETE _data_stream/logs-mylogs-default\n```\n* Go to the streams listing page","sha":"f3042efa8f22a8bfa680c33a4999d541639436ff"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/217854","number":217854,"mergeCommit":{"message":"🌊 Streams: Fix listing page for orphaned streams (elastic#217854)\n\nThe listing page didn't handle \"orphaned\" streams properly (classic data\nstreams which have a configuration on the stream level but the\nunderlying data stream is not available because it got deleted).\n\nThis PR fixes that and adds an integration test for it\n\n<img width=\"774\" alt=\"Screenshot 2025-04-10 at 16 07 15\"\nsrc=\"https://github.com/user-attachments/assets/da15c56b-7dbd-4070-ab6d-4235132da8ed\"\n/>\n\nIn this picture, `logs-test-default` is orphaned.\n\nTo test:\n* Create a new classic stream (e.g. via executing\n```\nPOST logs-mylogs-default/_doc\n{ \"message\": \"Test\" }\n```\n* Go into the streams UI and add a processor for this stream\n* Delete the data stream via stack management or via\n```\nDELETE _data_stream/logs-mylogs-default\n```\n* Go to the streams listing page","sha":"f3042efa8f22a8bfa680c33a4999d541639436ff"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Joe Reuter <[email protected]>
1 parent 2904428 commit 6f09403

File tree

8 files changed

+96
-36
lines changed

8 files changed

+96
-36
lines changed

‎x-pack/platform/plugins/shared/streams/server/lib/streams/client.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -495,18 +495,33 @@ export class StreamsClient {
495495
* Lists both managed and unmanaged streams
496496
*/
497497
async listStreams(): Promise<StreamDefinition[]> {
498+
const streams = await this.listStreamsWithDataStreamExistence();
499+
return streams.map((stream) => {
500+
const { data_stream_exists: _, ...rest } = stream;
501+
return rest;
502+
});
503+
}
504+
505+
async listStreamsWithDataStreamExistence(): Promise<
506+
Array<StreamDefinition & { data_stream_exists: boolean }>
507+
> {
498508
const [managedStreams, unmanagedStreams] = await Promise.all([
499509
this.getManagedStreams(),
500510
this.getUnmanagedDataStreams(),
501511
]);
502512

503-
const allDefinitionsById = new Map<string, StreamDefinition>(
504-
managedStreams.map((stream) => [stream.name, stream])
513+
const allDefinitionsById = new Map<string, StreamDefinition & { data_stream_exists: boolean }>(
514+
managedStreams.map((stream) => [stream.name, { ...stream, data_stream_exists: false }])
505515
);
506516

507517
unmanagedStreams.forEach((stream) => {
508518
if (!allDefinitionsById.get(stream.name)) {
509-
allDefinitionsById.set(stream.name, stream);
519+
allDefinitionsById.set(stream.name, { ...stream, data_stream_exists: true });
520+
} else {
521+
allDefinitionsById.set(stream.name, {
522+
...allDefinitionsById.get(stream.name)!,
523+
data_stream_exists: true,
524+
});
510525
}
511526
});
512527

‎x-pack/platform/plugins/shared/streams/server/routes/internal/streams/crud/route.ts‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { getDataStreamLifecycle } from '../../../../lib/streams/stream_crud';
1616
export interface ListStreamDetail {
1717
stream: StreamDefinition;
1818
effective_lifecycle: UnwiredIngestStreamEffectiveLifecycle;
19-
data_stream: estypes.IndicesDataStream;
19+
data_stream?: estypes.IndicesDataStream;
2020
}
2121

2222
export const listStreamsRoute = createServerRoute({
@@ -27,20 +27,18 @@ export const listStreamsRoute = createServerRoute({
2727
params: z.object({}),
2828
handler: async ({ request, getScopedClients }): Promise<{ streams: ListStreamDetail[] }> => {
2929
const { streamsClient, scopedClusterClient } = await getScopedClients({ request });
30-
const streams = await streamsClient.listStreams();
30+
const streams = await streamsClient.listStreamsWithDataStreamExistence();
3131
const dataStreams = await scopedClusterClient.asCurrentUser.indices.getDataStream({
32-
name: streams.map((stream) => stream.name),
32+
name: streams.filter((stream) => stream.data_stream_exists).map((stream) => stream.name),
3333
});
3434

3535
const enrichedStreams = streams.reduce<ListStreamDetail[]>((acc, stream) => {
3636
const match = dataStreams.data_streams.find((dataStream) => dataStream.name === stream.name);
37-
if (match) {
38-
acc.push({
39-
stream,
40-
effective_lifecycle: getDataStreamLifecycle(match),
41-
data_stream: match,
42-
});
43-
}
37+
acc.push({
38+
stream,
39+
effective_lifecycle: getDataStreamLifecycle(match ?? null),
40+
data_stream: match,
41+
});
4442
return acc;
4543
}, []);
4644

‎x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_management/classic.tsx‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,16 @@
66
*/
77
import React from 'react';
88
import { i18n } from '@kbn/i18n';
9-
import { UnwiredStreamGetResponse } from '@kbn/streams-schema';
10-
import { EuiCallOut, EuiFlexGroup } from '@elastic/eui';
9+
import { UnwiredStreamGetResponse, isUnwiredStreamDefinition } from '@kbn/streams-schema';
10+
import { EuiBadgeGroup, EuiCallOut, EuiFlexGroup } from '@elastic/eui';
1111
import { useStreamsAppParams } from '../../../hooks/use_streams_app_params';
1212
import { RedirectTo } from '../../redirect_to';
1313
import { StreamDetailEnrichment } from '../stream_detail_enrichment';
1414
import { ManagementTabs, Wrapper } from './wrapper';
1515
import { StreamDetailLifecycle } from '../stream_detail_lifecycle';
1616
import { UnmanagedElasticsearchAssets } from './unmanaged_elasticsearch_assets';
17+
import { StreamsAppPageTemplate } from '../../streams_app_page_template';
18+
import { ClassicStreamBadge, LifecycleBadge } from '../../stream_badges';
1719

1820
const classicStreamManagementSubTabs = ['enrich', 'advanced', 'lifecycle'] as const;
1921

@@ -36,22 +38,42 @@ export function ClassicStreamDetailManagement({
3638

3739
if (!definition.data_stream_exists) {
3840
return (
39-
<EuiFlexGroup direction="column">
40-
<EuiCallOut
41-
title={i18n.translate('xpack.streams.unmanagedStreamOverview.missingDatastream.title', {
42-
defaultMessage: 'Data stream missing',
43-
})}
44-
color="danger"
45-
iconType="error"
46-
>
47-
<p>
48-
{i18n.translate('xpack.streams.unmanagedStreamOverview.missingDatastream.description', {
49-
defaultMessage:
50-
'The underlying Elasticsearch data stream for this classic stream is missing. Recreate the data stream to restore the stream by sending data before using the management features.',
41+
<>
42+
<StreamsAppPageTemplate.Header
43+
bottomBorder="extended"
44+
pageTitle={
45+
<EuiFlexGroup gutterSize="s" alignItems="center">
46+
{i18n.translate('xpack.streams.entityDetailViewWithoutParams.manageStreamTitle', {
47+
defaultMessage: 'Manage stream {streamId}',
48+
values: { streamId: key },
49+
})}
50+
<EuiBadgeGroup gutterSize="s">
51+
{isUnwiredStreamDefinition(definition.stream) && <ClassicStreamBadge />}
52+
<LifecycleBadge lifecycle={definition.effective_lifecycle} />
53+
</EuiBadgeGroup>
54+
</EuiFlexGroup>
55+
}
56+
/>
57+
<StreamsAppPageTemplate.Body>
58+
<EuiCallOut
59+
title={i18n.translate('xpack.streams.unmanagedStreamOverview.missingDatastream.title', {
60+
defaultMessage: 'Data stream missing',
5161
})}
52-
</p>
53-
</EuiCallOut>
54-
</EuiFlexGroup>
62+
color="danger"
63+
iconType="error"
64+
>
65+
<p>
66+
{i18n.translate(
67+
'xpack.streams.unmanagedStreamOverview.missingDatastream.description',
68+
{
69+
defaultMessage:
70+
'The underlying Elasticsearch data stream for this classic stream is missing. Recreate the data stream to restore the stream by sending data before using the management features.',
71+
}
72+
)}
73+
</p>
74+
</EuiCallOut>
75+
</StreamsAppPageTemplate.Body>
76+
</>
5577
);
5678
}
5779

‎x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_overview/components/stream_chart_panel.tsx‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
import { css } from '@emotion/css';
1616
import { i18n } from '@kbn/i18n';
1717
import React, { useMemo } from 'react';
18-
import { IngestStreamGetResponse } from '@kbn/streams-schema';
18+
import { IngestStreamGetResponse, isWiredStreamGetResponse } from '@kbn/streams-schema';
1919
import { computeInterval } from '@kbn/visualization-utils';
2020
import moment, { DurationInputArg1, DurationInputArg2 } from 'moment';
2121
import { useKibana } from '../../../hooks/use_kibana';
@@ -143,6 +143,8 @@ export function StreamChartPanel({ definition }: StreamChartPanelProps) {
143143
const docCount = docCountFetch?.value?.details.count;
144144
const formattedDocCount = docCount ? formatNumber(docCount, 'decimal0') : '0';
145145

146+
const dataStreamExists = isWiredStreamGetResponse(definition) || definition.data_stream_exists;
147+
146148
return (
147149
<EuiPanel hasShadow={false} hasBorder>
148150
<EuiFlexGroup
@@ -170,7 +172,7 @@ export function StreamChartPanel({ definition }: StreamChartPanelProps) {
170172
data-test-subj="streamsDetailOverviewOpenInDiscoverButton"
171173
iconType="discoverApp"
172174
href={discoverLink}
173-
isDisabled={!discoverLink}
175+
isDisabled={!discoverLink || !dataStreamExists}
174176
>
175177
{i18n.translate('xpack.streams.streamDetailOverview.openInDiscoverButtonLabel', {
176178
defaultMessage: 'Open in Discover',

‎x-pack/platform/plugins/shared/streams_app/public/components/stream_detail_overview/stream_detail_overview.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@ export function StreamDetailOverview({ definition }: { definition: IngestStreamG
4949

5050
<EuiFlexItem grow>
5151
<EuiFlexGroup direction="row" gutterSize="m">
52-
<EuiFlexItem grow={4}>{definition && <TabsPanel tabs={tabs} />}</EuiFlexItem>
52+
<EuiFlexItem grow={4}>
53+
<TabsPanel tabs={tabs} />
54+
</EuiFlexItem>
5355
<EuiFlexItem grow={8}>
5456
<StreamChartPanel definition={definition} />
5557
</EuiFlexItem>

‎x-pack/platform/plugins/shared/streams_app/public/components/stream_list_view/retention_column.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ export function RetentionColumn({ lifecycle }: { lifecycle: IngestStreamEffectiv
2626
const ilmLocator = share.url.locators.get<IlmLocatorParams>(ILM_LOCATOR_ID);
2727

2828
if (isErrorLifecycle(lifecycle)) {
29-
return null;
29+
return <EuiBadge color="hollow">{lifecycle.error.message}</EuiBadge>;
3030
}
3131

3232
if (isIlmLifecycle(lifecycle)) {

‎x-pack/platform/plugins/shared/streams_app/public/components/stream_list_view/tree_table.tsx‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,10 @@ export function StreamsTreeTable({
7676
defaultMessage: 'Documents',
7777
}),
7878
width: '40%',
79-
render: (_, item) => <DocumentsColumn indexPattern={item.name} numDataPoints={25} />,
79+
render: (_, item) =>
80+
item.data_stream ? (
81+
<DocumentsColumn indexPattern={item.name} numDataPoints={25} />
82+
) : null,
8083
},
8184
{
8285
field: 'effective_lifecycle',

‎x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/classic.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
*/
77

88
import expect from '@kbn/expect';
9-
import { asUnwiredStreamGetResponse } from '@kbn/streams-schema';
9+
import { asUnwiredStreamGetResponse, isUnwiredStreamDefinition } from '@kbn/streams-schema';
1010
import { isNotFoundError } from '@kbn/es-errors';
1111
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
1212
import {
@@ -567,6 +567,24 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
567567
expect(getDetailsResponse.status).to.eql(404);
568568
});
569569

570+
it('should still return the stream on public listing API', async () => {
571+
const getResponse = await apiClient.fetch('GET /api/streams 2023-10-31');
572+
expect(getResponse.status).to.eql(200);
573+
const classicStream = getResponse.body.streams.find(
574+
(stream) => stream.name === ORPHANED_STREAM_NAME
575+
);
576+
expect(isUnwiredStreamDefinition(classicStream!)).to.be(true);
577+
});
578+
579+
it('should still return the stream on internal listing API', async () => {
580+
const getResponse = await apiClient.fetch('GET /internal/streams');
581+
expect(getResponse.status).to.eql(200);
582+
const classicStream = getResponse.body.streams.find(
583+
(stream) => stream.stream.name === ORPHANED_STREAM_NAME
584+
);
585+
expect(isUnwiredStreamDefinition(classicStream!.stream)).to.be(true);
586+
});
587+
570588
after(async () => {
571589
await apiClient.fetch('DELETE /api/streams/{name} 2023-10-31', {
572590
params: {

0 commit comments

Comments
 (0)