diff --git a/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts b/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts index 305ea8ac57a53..11e9bffed0667 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts @@ -34,6 +34,7 @@ export const dataStreamStatRt = rt.intersection([ integration: rt.string, totalDocs: rt.number, creationDate: rt.number, + hasFailureStore: rt.boolean, }), ]); @@ -236,6 +237,7 @@ export const dataStreamSettingsRt = rt.partial({ export type DataStreamSettings = rt.TypeOf; export const dataStreamDetailsRt = rt.partial({ + hasFailureStore: rt.boolean, lastActivity: rt.number, degradedDocsCount: rt.number, failedDocsCount: rt.number, diff --git a/x-pack/platform/plugins/shared/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/platform/plugins/shared/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 9c551a7b5dcc2..05e3efb7782cb 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -32,6 +32,7 @@ export class DataStreamStat { docsInTimeRange?: number; degradedDocs: QualityStat; failedDocs: QualityStat; + hasFailureStore?: DataStreamStatType['hasFailureStore']; private constructor(dataStreamStat: DataStreamStat) { this.rawName = dataStreamStat.rawName; @@ -49,6 +50,7 @@ export class DataStreamStat { this.docsInTimeRange = dataStreamStat.docsInTimeRange; this.degradedDocs = dataStreamStat.degradedDocs; this.failedDocs = dataStreamStat.failedDocs; + this.hasFailureStore = dataStreamStat.hasFailureStore; } public static create(dataStreamStat: DataStreamStatType) { @@ -56,6 +58,7 @@ export class DataStreamStat { const dataStreamStatProps = { rawName: dataStreamStat.name, + hasFailureStore: dataStreamStat.hasFailureStore, type, name: dataset, title: dataset, @@ -79,17 +82,20 @@ export class DataStreamStat { failedDocStat, datasetIntegrationMap, totalDocs, + hasFailureStore, }: { datasetName: string; degradedDocStat: QualityStat; failedDocStat: QualityStat; datasetIntegrationMap: Record; totalDocs: number; + hasFailureStore?: boolean; }) { const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName); const dataStreamStatProps = { rawName: datasetName, + hasFailureStore, type, name: dataset, title: datasetIntegrationMap[dataset]?.title || dataset, diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/columns.tsx b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/columns.tsx index 71f11a2b00895..1637eafccd5d7 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/columns.tsx +++ b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/columns.tsx @@ -355,29 +355,69 @@ export const getDatasetQualityTableColumns = ({ ), field: 'failedDocs.percentage', sortable: true, - render: (_: any, dataStreamStat: DataStreamStat) => ( - - - i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', { - defaultMessage: '{failedDocsCount} failed docs in this data set.', - values: { - failedDocsCount, - }, - }) - } - dataTestSubj="datasetQualityFailedDocsPercentageLink" - /> - - ), + render: (_: any, dataStreamStat: DataStreamStat) => { + if (!dataStreamStat.hasFailureStore) { + const FailureStoreHoverLink = () => { + const [hovered, setHovered] = React.useState(false); + const locator = urlService.locators.get('INDEX_MANAGEMENT_LOCATOR_ID'); + const params = { + page: 'data_streams_details', + dataStreamName: dataStreamStat.rawName, + } as const; + + return ( + + setHovered(true)} + onMouseLeave={() => setHovered(false)} + css={{ fontWeight: 'normal' }} + > + {hovered + ? i18n.translate('xpack.datasetQuality.failureStore.enable', { + defaultMessage: 'Set failure store', + }) + : i18n.translate('xpack.datasetQuality.failureStore.notAvailable', { + defaultMessage: 'N/A', + })} + + + ); + }; + return ; + } + return ( + + + i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', { + defaultMessage: '{failedDocsCount} failed docs in this data set.', + values: { + failedDocsCount, + }, + }) + } + dataTestSubj="datasetQualityFailedDocsPercentageLink" + /> + + ); + }, width: '140px', }, ] diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/table.tsx b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/table.tsx index a7524198aa24a..96f786cad6947 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/table.tsx +++ b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/table.tsx @@ -75,7 +75,7 @@ export const Table = () => { - + - {canUserReadFailureStore && ( + {canShowFailureStoreInfo && ( import('./header')); const Summary = dynamic(() => import('./summary')); @@ -22,9 +24,20 @@ export function Overview() { dataStream, isNonAggregatable, canUserReadFailureStore, + hasFailureStore, updateTimeRange, loadingState: { dataStreamSettingsLoading }, } = useDatasetQualityDetailsState(); + + const { + services: { + share: { url: urlService }, + }, + } = useKibanaContextForPlugin(); + + const locator = urlService.locators.get('INDEX_MANAGEMENT_LOCATOR_ID'); + const locatorParams = { page: 'data_streams_details', dataStreamName: dataStream } as const; + const [lastReloadTime, setLastReloadTime] = useState(Date.now()); const handleRefresh = useCallback( @@ -39,6 +52,31 @@ export function Overview() { {isNonAggregatable && } + {!dataStreamSettingsLoading && !hasFailureStore && canUserReadFailureStore && ( +
+ + {i18n.translate('xpack.datasetQuality.noFailureStoreTitle', { + defaultMessage: 'Failure store is not enabled for this data stream. ', + })} + + {i18n.translate('xpack.datasetQuality.enableFailureStore', { + defaultMessage: 'Enable failure store', + })} + + + } + /> +
+ )} + {!dataStreamSettingsLoading && !canUserReadFailureStore && ( diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx index bfbb405dfd30c..f11828f16f825 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx +++ b/x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx @@ -48,7 +48,7 @@ const failedDocsColumnTooltip = ( // Allow for lazy loading // eslint-disable-next-line import/no-default-export export default function Summary() { - const { canUserReadFailureStore } = useDatasetQualityDetailsState(); + const { canShowFailureStoreInfo } = useDatasetQualityDetailsState(); const { isSummaryPanelLoading, totalDocsCount, @@ -103,7 +103,7 @@ export default function Summary() { isLoading={isSummaryPanelLoading} tooltip={degradedDocsTooltip} /> - {canUserReadFailureStore && ( + {canShowFailureStoreInfo && ( { [service] ); + const hasFailureStore = Boolean(dataStreamDetails?.hasFailureStore); + const canShowFailureStoreInfo = canUserReadFailureStore && hasFailureStore; + return { service, telemetryClient, @@ -182,6 +185,8 @@ export const useDatasetQualityDetailsState = () => { canUserAccessDashboards, canUserViewIntegrations, canUserReadFailureStore, + hasFailureStore, + canShowFailureStoreInfo, expandedQualityIssue, isQualityIssueFlyoutOpen, }; diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.test.ts index c24d9941c8919..8b8cc31838c57 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.test.ts @@ -36,6 +36,7 @@ describe('generateDatasets', () => { const dataStreamStats: DataStreamStatType[] = [ { + hasFailureStore: true, name: 'logs-system.application-default', lastActivity: 1712911241117, size: '82.1kb', @@ -48,6 +49,7 @@ describe('generateDatasets', () => { }, }, { + hasFailureStore: false, name: 'logs-synth-default', lastActivity: 1712911241117, size: '62.5kb', @@ -123,6 +125,7 @@ describe('generateDatasets', () => { percentage: 1.9607843137254901, count: 2, }, + hasFailureStore: true, }, { name: 'synth', @@ -149,6 +152,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); @@ -188,6 +192,7 @@ describe('generateDatasets', () => { percentage: 100, count: 2, }, + hasFailureStore: true, }, { name: 'synth', @@ -214,6 +219,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); @@ -244,6 +250,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, { name: 'synth', @@ -267,6 +274,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); @@ -300,6 +308,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, { name: 'synth', @@ -323,6 +332,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, { name: 'another', @@ -346,6 +356,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); @@ -379,6 +390,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: true, }, { name: 'synth', @@ -405,12 +417,14 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); it('merges integration information with dataStreamStats when dataset is not an integration default one', () => { const nonDefaultDataset = { + hasFailureStore: false, name: 'logs-system.custom-default', lastActivity: 1712911241117, size: '82.1kb', @@ -451,6 +465,7 @@ describe('generateDatasets', () => { percentage: 0, count: 0, }, + hasFailureStore: false, }, ]); }); diff --git a/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.ts b/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.ts index c4a82170b3afd..4828adea9f3cd 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.ts @@ -74,6 +74,10 @@ export function generateDatasets( {} ); + const datasetsWithFailureStore = new Set( + dataStreamStats.filter(({ hasFailureStore }) => hasFailureStore).map(({ name }) => name) + ); + const degradedMap: Record< DataStreamDocsStat['dataset'], { @@ -110,12 +114,15 @@ export function generateDatasets( failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS, datasetIntegrationMap, totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0), + hasFailureStore: datasetsWithFailureStore.has(dataset), }) ); } return dataStreamStats?.map((dataStream) => { - const dataset = DataStreamStat.create(dataStream); + const dataset = DataStreamStat.create({ + ...dataStream, + }); const degradedDocs = degradedMap[dataset.rawName] || dataset.degradedDocs; const failedDocs = failedMap[dataset.rawName] || dataset.failedDocs; const qualityStats = [degradedDocs.percentage, failedDocs.percentage]; diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts index 5ff09db33ba37..e71be20e01a8a 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts @@ -86,6 +86,7 @@ export async function getDataStreamDetails({ ...dataStreamSummaryStats, failedDocsCount: failedDocs?.count, sizeBytes, + hasFailureStore: esDataStream?.hasFailureStore, lastActivity: esDataStream?.lastActivity, userPrivileges: { canMonitor: dataStreamPrivileges.monitor, diff --git a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts index 6e0d84d15c26c..376d2923220ce 100644 --- a/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts +++ b/x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts @@ -68,6 +68,7 @@ export async function getDataStreams(options: { canMonitor: dataStreamsPrivileges[dataStream.name].monitor, canReadFailureStore: dataStreamsPrivileges[dataStream.name][FAILURE_STORE_PRIVILEGE], }, + hasFailureStore: dataStream.failure_store?.enabled, })); return { diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_streams_failure_store.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_streams_failure_store.ts new file mode 100644 index 0000000000000..afaa30fc55e56 --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_streams_failure_store.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const synthtrace = getService('synthtrace'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const es = getService('es'); + + const start = '2025-01-01T00:00:00.000Z'; + const end = '2025-01-01T00:01:00.000Z'; + + const enabledDs = 'logs-synth.fs-default'; + const disabledDs = 'logs-synth.no-default'; + + async function callDetails(supertestApi: any, ds: string) { + return supertestApi + .get(`/internal/dataset_quality/data_streams/${encodeURIComponent(ds)}/details`) + .query({ start, end }); + } + + describe('Failure-store flag on data-streams', () => { + let client: LogsSynthtraceEsClient; + let supertestAdmin: any; + + before(async () => { + client = await synthtrace.createLogsSynthtraceEsClient(); + + await client.createComponentTemplate({ + name: 'logs-failure-enabled@mappings', + dataStreamOptions: { failure_store: { enabled: true } }, + }); + await es.indices.putIndexTemplate({ + name: enabledDs, + index_patterns: [enabledDs], + composed_of: [ + 'logs-failure-enabled@mappings', + 'logs@mappings', + 'logs@settings', + 'ecs@mappings', + ], + priority: 500, + allow_auto_create: true, + data_stream: { hidden: false }, + }); + + await client.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((ts) => log.create().timestamp(ts).dataset('synth.fs')), + timerange(start, end) + .interval('1m') + .rate(1) + .generator((ts) => log.create().timestamp(ts).dataset('synth.no')), + ]); + await client.refresh(); + + supertestAdmin = await roleScopedSupertest.getSupertestWithRoleScope('admin', { + useCookieHeader: true, + withInternalHeaders: true, + }); + }); + + after(async () => { + await es.indices.deleteIndexTemplate({ name: enabledDs }); + await client.deleteComponentTemplate('logs-failure-enabled@mappings'); + await client.clean(); + }); + + it('details API reports correct hasFailureStore flag', async () => { + const enabled = await callDetails(supertestAdmin, enabledDs); + const disabled = await callDetails(supertestAdmin, disabledDs); + expect(enabled.body.hasFailureStore).to.be(true); + expect(disabled.body.hasFailureStore).to.be(false); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 81878a916b6c1..e33156b4900ee 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -25,5 +25,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./integration_dashboards')); loadTestFile(require.resolve('./integrations')); loadTestFile(require.resolve('./update_field_limit')); + loadTestFile(require.resolve('./data_streams_failure_store')); }); } diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts index 7f3d525e3ed8d..9f9bfca4cdc56 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts @@ -28,6 +28,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'datasetQuality', ]); const retry = getService('retry'); + const testSubjects = getService('testSubjects'); const synthtrace = getService('logSynthtraceEsClient'); const to = '2024-01-01T12:00:00.000Z'; const apacheAccessDatasetName = 'apache.access'; @@ -203,7 +204,25 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const failedDocsCol = cols[PageObjects.datasetQuality.texts.datasetFailedDocsColumn]; const failedDocsColCellTexts = await failedDocsCol.getCellTexts(); - expect(failedDocsColCellTexts).to.eql(['0%', '0%', '20%', '0%']); + expect(failedDocsColCellTexts).to.eql(['N/A', 'N/A', '20%', 'N/A']); + }); + + it('changes link text on hover when failure store is not enabled', async () => { + const linkSelector = 'datasetQualitySetFailureStoreLink'; + const links = await testSubjects.findAll(linkSelector); + expect(links.length).to.be.greaterThan(0); + const link = links[links.length - 1]; + + expect(await link.getVisibleText()).to.eql('N/A'); + + await link.moveMouseTo(); + + await retry.try(async () => { + expect(await link.getVisibleText()).to.eql('Set failure store'); + }); + + const table = await PageObjects.datasetQuality.getDatasetsTable(); + await table.moveMouseTo(); }); }); }); diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index f560daa03a9af..0d6d4040806ff 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -411,6 +411,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv }, async toggleShowFullDatasetNames() { + await find.waitForDeletedByCssSelector('.euiToolTipPopover', 5 * 1000); return find.clickByCssSelector(selectors.showFullDatasetNamesSwitch); }, @@ -473,12 +474,16 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ].filter((item) => !excludeKeys.includes(item.key)); const kpiTexts = await Promise.all( - kpiTitleAndKeys.map(async ({ title, key }) => ({ - key, - value: await testSubjects.getVisibleText( - `${testSubjectSelectors.datasetQualityDetailsSummaryKpiValue}-${title}` - ), - })) + kpiTitleAndKeys.map(async ({ title, key }) => { + const selector = `${testSubjectSelectors.datasetQualityDetailsSummaryKpiValue}-${title}`; + + const exists = await testSubjects.exists(selector); + if (!exists) { + return { key, value: undefined } as { key: string; value: string | undefined }; + } + + return { key, value: await testSubjects.getVisibleText(selector) }; + }) ); return kpiTexts.reduce( @@ -490,10 +495,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ); }, - /** - * Selects a breakdown field from the unified histogram breakdown selector - * @param fieldText The text of the field to select. Use 'No breakdown' to clear the selection - */ async selectBreakdownField(fieldText: string) { return euiSelectable.searchAndSelectOption( testSubjectSelectors.unifiedHistogramBreakdownSelectorButton, @@ -532,10 +533,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv const fieldExpandButton = expandButtons[testDatasetRowIndex]; - // Check if 'title' attribute is "Expand" or "Collapse" const isCollapsed = (await fieldExpandButton.getAttribute('title')) === 'Expand'; - // Open if collapsed if (isCollapsed) { await fieldExpandButton.click(); } @@ -563,10 +562,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv const fieldExpandButton = expandButtons[testDatasetRowIndex]; - // Check if 'title' attribute is "Expand" or "Collapse" const isCollapsed = (await fieldExpandButton.getAttribute('title')) === 'Expand'; - // Open if collapsed if (isCollapsed) { await fieldExpandButton.click(); }