[NOT MERGE] PoC: Storybook a11y testing in CI

.buildkite/pipelines/pull_request/base.yml

@@ -101,6 +101,11 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/storybooks/storybook_tests.sh
+    label: 'Run Storybook Tests'
+    env:
+      STORYBOOK_URL: 'http://127.0.0.1:9001'
+
   - wait
 
   - command: .buildkite/scripts/steps/ci_stats_ready.sh

.buildkite/pipelines/pull_request/storybooks.yml

@@ -6,3 +6,9 @@ steps:
       preemptible: true
     key: storybooks
     timeout_in_minutes: 80
+
+  # - command: ".buildkite/scripts/steps/storybooks/storybook_tests.sh"
+  #   label: "Run Storybook Tests"
+  #   soft_fail: true
+  #   env:
+  #     STORYBOOK_URL: 'http://localhost:9001'

.buildkite/scripts/steps/storybooks/storybook_tests.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+echo "Starting Storybook server..."
+yarn storybook apm & 
+STORYBOOK_PID=$!
+
+echo "Waiting for Storybook to be ready..."
+sleep 60
+
+echo "Running Storybook tests..."
+yarn test:storybook --url "$STORYBOOK_URL"
+
+kill $STORYBOOK_PID

package.json

@@ -66,6 +66,7 @@
     "test:ftr:server": "node scripts/functional_tests_server",
     "test:jest": "node scripts/jest",
     "test:jest_integration": "node scripts/jest_integration",
+    "test:storybook": "npx test-storybook",
     "test:type_check": "node scripts/type_check"
   },
   "repository": {
@@ -1140,6 +1141,7 @@
     "archiver": "^7.0.1",
     "async": "^3.2.6",
     "aws4": "^1.13.2",
+    "axe-playwright": "^2.1.0",
     "axios": "^1.8.3",
     "base64-js": "^1.3.1",
     "bitmap-sdf": "^1.0.3",
@@ -1620,6 +1622,7 @@
     "@storybook/react": "^8.6.3",
     "@storybook/react-webpack5": "^8.6.3",
     "@storybook/test": "^8.6.3",
+    "@storybook/test-runner": "^0.22.0",
     "@storybook/theming": "^8.6.3",
     "@storybook/types": "^8.6.3",
     "@testing-library/dom": "^10.4.0",
@@ -1879,7 +1882,7 @@
     "pirates": "^4.0.1",
     "piscina": "^3.2.0",
     "pixelmatch": "^5.3.0",
-    "playwright": "1.49.0",
+    "playwright": "1.51.1",
     "playwright-chromium": "1.49.0",
     "pngjs": "^7.0.0",
     "postcss": "^8.4.31",

src/platform/packages/shared/kbn-storybook/src/lib/preview.ts

@@ -57,6 +57,9 @@ const preview: Preview = {
       },
     },
   },
+  parameters: {
+    a11y: { test: 'error' },
+  },
 };
 
 // eslint-disable-next-line import/no-default-export

x-pack/solutions/observability/plugins/apm/.storybook/test-runner.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { getStoryContext, type TestRunnerConfig } from '@storybook/test-runner';
+import { injectAxe, checkA11y } from 'axe-playwright';
+
+/*
+ * See https://storybook.js.org/docs/writing-tests/test-runner#test-hook-api
+ * to learn more about the test-runner hooks API.
+ */
+const config: TestRunnerConfig = {
+  tags: {
+    skip: ['skip-test'],
+  },
+  async preVisit(page) {
+    await injectAxe(page);
+  },
+  async postVisit(page, context) {
+    const storyContext = await getStoryContext(page, context);
+    // // Do not run a11y tests on disabled stories.
+    if (storyContext.parameters?.a11y?.disable) {
+      return;
+    }
+    await checkA11y(page, 'body', {
+      detailedReport: true,
+      detailedReportOptions: {
+        html: true,
+      },
+    });
+  },
+};
+// eslint-disable-next-line import/no-default-export
+export default config;

x-pack/solutions/observability/plugins/apm/public/components/alerting/rule_types/error_count_rule_type/index.stories.tsx

@@ -110,6 +110,10 @@ export const CreatingInApmFromService: StoryObj<Args> = {
   render: ({ ruleParams, metadata }) => (
     <CreatingInApmFromServiceComponent ruleParams={ruleParams} metadata={metadata} />
   ),
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
 
   args: {
     ruleParams: {},
@@ -143,7 +147,10 @@ export const EditingInStackManagement: StoryObj<Args> = {
   render: ({ ruleParams, metadata }) => (
     <EditingInStackManagementComponent ruleParams={ruleParams} metadata={metadata} />
   ),
-
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
   args: {
     ruleParams: {
       environment: 'testEnvironment',
@@ -177,6 +184,10 @@ export const CreatingInStackManagement: StoryObj<Args> = {
   render: ({ ruleParams, metadata }) => (
     <CreatingInStackManagementComponent ruleParams={ruleParams} metadata={metadata} />
   ),
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
 
   args: {
     ruleParams: {},

x-pack/solutions/observability/plugins/apm/public/components/alerting/rule_types/transaction_duration_rule_type/index.stories.tsx

@@ -28,6 +28,10 @@ interface Args {
 export default {
   title: 'alerting/TransactionDurationRuleType',
   component: TransactionDurationRuleType,
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
   decorators: [
     (StoryComponent: ComponentType) => {
       return (

x-pack/solutions/observability/plugins/apm/public/components/alerting/rule_types/transaction_error_rate_rule_type/index.stories.tsx

@@ -100,7 +100,10 @@ export const CreatingInStackManagement: StoryObj<Args> = {
   render: ({ ruleParams, metadata }) => (
     <CreatingInStackManagementComponent ruleParams={ruleParams} metadata={metadata} />
   ),
-
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
   args: {
     ruleParams: {
       environment: 'testEnvironment',

x-pack/solutions/observability/plugins/apm/public/components/app/error_group_details/distribution/index.stories.tsx

@@ -5,86 +5,94 @@
  * 2.0.
  */
 
-import type { ComponentType } from 'react';
+import type { ComponentProps } from 'react';
 import React from 'react';
+import type { Meta, StoryObj } from '@storybook/react';
 import { ErrorDistribution } from '.';
 import { MockApmPluginStorybook } from '../../../../context/apm_plugin/mock_apm_plugin_storybook';
 import { FETCH_STATUS } from '../../../../hooks/use_fetcher';
 
-export default {
+type Args = ComponentProps<typeof ErrorDistribution>;
+
+const stories: Meta<Args> = {
   title: 'app/ErrorGroupDetails/distribution',
   component: ErrorDistribution,
+  tags: ['skip-test'],
   decorators: [
-    (Story: ComponentType) => {
+    (StoryComponent) => {
       return (
         <MockApmPluginStorybook routePath="/services/{serviceName}/errors/{groupId}?kuery=&rangeFrom=now-15m&rangeTo=now&environment=ENVIRONMENT_ALL&serviceGroup=&comparisonEnabled=true&transactionType=request&offset=1d">
-          <Story />
+          <StoryComponent />
         </MockApmPluginStorybook>
       );
     },
   ],
 };
 
-export function Example() {
-  const distribution = {
-    bucketSize: 62350,
-    currentPeriod: [
-      { x: 1624279912350, y: 6 },
-      { x: 1624279974700, y: 1 },
-      { x: 1624280037050, y: 2 },
-      { x: 1624280099400, y: 3 },
-      { x: 1624280161750, y: 13 },
-      { x: 1624280224100, y: 1 },
-      { x: 1624280286450, y: 2 },
-      { x: 1624280348800, y: 0 },
-      { x: 1624280411150, y: 4 },
-      { x: 1624280473500, y: 4 },
-      { x: 1624280535850, y: 1 },
-      { x: 1624280598200, y: 4 },
-      { x: 1624280660550, y: 0 },
-      { x: 1624280722900, y: 2 },
-      { x: 1624280785250, y: 3 },
-      { x: 1624280847600, y: 0 },
-    ],
-    previousPeriod: [
-      { x: 1624279912350, y: 6 },
-      { x: 1624279974700, y: 1 },
-      { x: 1624280037050, y: 2 },
-      { x: 1624280099400, y: 3 },
-      { x: 1624280161750, y: 13 },
-      { x: 1624280224100, y: 1 },
-      { x: 1624280286450, y: 2 },
-      { x: 1624280348800, y: 0 },
-      { x: 1624280411150, y: 4 },
-      { x: 1624280473500, y: 4 },
-      { x: 1624280535850, y: 1 },
-      { x: 1624280598200, y: 4 },
-      { x: 1624280660550, y: 0 },
-      { x: 1624280722900, y: 2 },
-      { x: 1624280785250, y: 3 },
-      { x: 1624280847600, y: 0 },
-    ],
-  };
+export default stories;
 
-  return (
-    <ErrorDistribution
-      fetchStatus={FETCH_STATUS.SUCCESS}
-      distribution={distribution}
-      title="Foo title"
-    />
-  );
-}
+export const Example: StoryObj<Args> = {
+  render: (args) => {
+    return <ErrorDistribution {...args} />;
+  },
+  args: {
+    distribution: {
+      bucketSize: 62350,
+      currentPeriod: [
+        { x: 1624279912350, y: 6 },
+        { x: 1624279974700, y: 1 },
+        { x: 1624280037050, y: 2 },
+        { x: 1624280099400, y: 3 },
+        { x: 1624280161750, y: 13 },
+        { x: 1624280224100, y: 1 },
+        { x: 1624280286450, y: 2 },
+        { x: 1624280348800, y: 0 },
+        { x: 1624280411150, y: 4 },
+        { x: 1624280473500, y: 4 },
+        { x: 1624280535850, y: 1 },
+        { x: 1624280598200, y: 4 },
+        { x: 1624280660550, y: 0 },
+        { x: 1624280722900, y: 2 },
+        { x: 1624280785250, y: 3 },
+        { x: 1624280847600, y: 0 },
+      ],
+      previousPeriod: [
+        { x: 1624279912350, y: 6 },
+        { x: 1624279974700, y: 1 },
+        { x: 1624280037050, y: 2 },
+        { x: 1624280099400, y: 3 },
+        { x: 1624280161750, y: 13 },
+        { x: 1624280224100, y: 1 },
+        { x: 1624280286450, y: 2 },
+        { x: 1624280348800, y: 0 },
+        { x: 1624280411150, y: 4 },
+        { x: 1624280473500, y: 4 },
+        { x: 1624280535850, y: 1 },
+        { x: 1624280598200, y: 4 },
+        { x: 1624280660550, y: 0 },
+        { x: 1624280722900, y: 2 },
+        { x: 1624280785250, y: 3 },
+        { x: 1624280847600, y: 0 },
+      ],
+    },
+    fetchStatus: FETCH_STATUS.SUCCESS,
+    title: 'Foo title',
+  },
+  tags: ['skip-test'],
+};
 
-export function EmptyState() {
-  return (
-    <ErrorDistribution
-      fetchStatus={FETCH_STATUS.SUCCESS}
-      distribution={{
-        bucketSize: 10,
-        currentPeriod: [],
-        previousPeriod: [],
-      }}
-      title="Foo title"
-    />
-  );
-}
+export const EmptyState: StoryObj<Args> = {
+  render: (args) => {
+    return <ErrorDistribution {...args} />;
+  },
+  args: {
+    fetchStatus: FETCH_STATUS.SUCCESS,
+    distribution: {
+      bucketSize: 10,
+      currentPeriod: [],
+      previousPeriod: [],
+    },
+    title: 'Foo title',
+  },
+  tags: ['skip-test'],
+};

x-pack/solutions/observability/plugins/apm/public/components/app/error_group_overview/error_group_list/error_group_list.stories.tsx

@@ -18,6 +18,7 @@ type Args = ComponentProps<typeof ErrorGroupList>;
 const stories: Meta<Args> = {
   title: 'app/ErrorGroupOverview/ErrorGroupList',
   component: ErrorGroupList,
+  tags: ['skip-test'],
   decorators: [
     (StoryComponent, { args }) => {
       const coreMock = {
@@ -110,7 +111,7 @@ export const Example: StoryObj<Args> = {
   render: (args) => {
     return <ErrorGroupList {...args} />;
   },
-
+  tags: ['skip-test'],
   args: {
     serviceName: 'test service',
     initialPageSize: 5,

x-pack/solutions/observability/plugins/apm/public/components/app/mobile/errors_and_crashes_group_details/shared/distribution/index.stories.tsx

@@ -14,6 +14,7 @@ import { FETCH_STATUS } from '../../../../../../hooks/use_fetcher';
 export default {
   title: 'app/ErrorsAndCrashGroupDetails/distribution',
   component: ErrorDistribution,
+  tags: ['skip-test'],
   decorators: [
     (Story: ComponentType) => {
       return (

x-pack/solutions/observability/plugins/apm/public/components/app/mobile/errors_and_crashes_overview/crash_group_list/crash_group_list.stories.tsx

@@ -100,6 +100,10 @@ export const EmptyState: StoryObj<Args> = {
   render: (args) => {
     return <MobileCrashGroupList {...args} />;
   },
+  // TODO: fix a11y errors
+  // parameters: {
+  //   a11y: { disable: true },
+  // },
 
   args: {
     mainStatistics: [],

x-pack/solutions/observability/plugins/apm/public/components/app/mobile/errors_and_crashes_overview/error_group_list/error_group_list.stories.tsx

@@ -19,6 +19,11 @@ type Args = ComponentProps<typeof MobileErrorGroupList>;
 const stories: Meta<Args> = {
   title: 'app/ErrorGroupOverview/MobileErrorGroupList',
   component: MobileErrorGroupList,
+  // TODO: fix a11y errors
+  parameters: {
+    a11y: { disable: true },
+  },
+
   decorators: [
     (StoryComponent) => {
       return (

x-pack/solutions/observability/plugins/apm/public/components/app/service_inventory/service_inventory.stories.tsx

@@ -18,6 +18,7 @@ import { FETCH_STATUS } from '../../../hooks/use_fetcher';
 const stories: Meta<{}> = {
   title: 'app/ServiceInventory',
   component: ServiceInventory,
+  tags: ['skip-test'],
   decorators: [
     (StoryComponent) => {
       const coreMock = {