[Detection Engine] Fix infinite-looping bug related to bootstrapping lists resources (#241052)

# Summary

Addresses #229018.

## Problem Statement
When the users visits the alerting page, we check to see whether the
lists indices exist, and if not, and they have the appropriate
privileges, we try to create it on behalf of the user.

However, if the user has enough privileges to _ask_ about the lists
indices status, but has insufficient privileges to _create_ the lists
indices, the frontend will enter an infinite loop. With sufficient
privileges, this loop is exited when the frontend creates the lists
indices.

## Explanation
The looping is due to a) the fact that the "read list index" API
responds with a 404 if either of the indices is not present, and b) how
404s are handled in the frontend. By default, a 404 will cause the http
client to throw an error. We use useQuery to cache this query, but
because the request throws an error the request is never cached, and
thus on the next hook render the cycle renews.

## Solution
By manually catching and parsing the 404 API response, we can both a)
ensure proper caching and b) present more information to the consumer of
that API. For backwards compatibility reasons, we cannot update the
semantics of the API itself.

Practically, since the response to a "list index not found" response is
to call the (generic) "create list index" API, this additional
information is not strictly necessary, the additional information could
e.g. be used to provide a more helpful error message on the client in
the future.

## Further Improvements
In these cases where the lists indices do not exist, and the user does
not have permission to create them, we do not currently inform the user
of this fact. We could improve the user experience by providing a more
helpful error message in this scenario.


# How to review
1. Create a user/role in kibana with _almost_ all of the [necessary
privileges to create the list
indices](https://www.elastic.co/docs/solutions/security/detect-and-alert/detections-requirements#security-detections-requirements-custom-role-privileges),
save for the "manage" cluster privilege (which is easily overlooked, due
to its being a single word in a column on that table):

    ```json
    almost-lists-manager:
      cluster: []
      indices:
        - names:
            - ".alerts-security.alerts-default"
            - ".internal.alerts-security.alerts-default-*"
            - ".siem-signals-default"
            - ".lists-default"
            - ".items-default"
          privileges: ["manage", "read", "write", "view_index_metadata"]
          field_security:
            grant: ["*"]
            except: []
          allow_restricted_indices: false
        - names:
            - ".preview.alerts-security.alerts-default"
            - ".internal.preview.alerts-security.alerts-default-*"
          privileges: ["read"]
          field_security:
            grant: ["*"]
            except: []
          allow_restricted_indices: false
      applications:
        - application: "kibana-.kibana"
privileges: ["feature_indexPatterns.all", "feature_siemV4.all",
"feature_actions.all", "feature_savedObjectsManagement.all"]
          resources: ["space:default"]
      run_as: []
      metadata: {}
      transient_metadata:
        enabled: true
      description: ""
    ```

1. Navigate to the alerts page, and confirm that the page loads
successfully, and does not loop.
1. (optional) Confirm that navigating to the alerts page on `main`
reproduces the infinite loop.
1. (optional) Confirm that adding the "manage" cluster privilege results
in the lists indices being created successfully.

### Checklist


- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
- [x] Review the [backport
guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing)
and apply applicable `backport:*` labels.

---------

Co-authored-by: kibanamachine <[email protected]>

Author: rylnd

x-pack/solutions/security/packages/kbn-securitysolution-list-hooks/src/mocks/response/read_list_index_schema.mock.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ListItemIndexExistSchema } from '@kbn/securitysolution-io-ts-list-types';
+
+export const getListIndexExistSchemaMock = (
+  overrides: Partial<ListItemIndexExistSchema> = {}
+): ListItemIndexExistSchema => ({
+  list_index: true,
+  list_item_index: true,
+  ...overrides,
+});

x-pack/solutions/security/packages/kbn-securitysolution-list-hooks/src/use_read_list_index/index.test.tsx

@@ -0,0 +1,252 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { QueryClient, QueryClientProvider } from '@kbn/react-query';
+import { waitFor, renderHook } from '@testing-library/react';
+
+import type { SecurityAppError } from '@kbn/securitysolution-t-grid';
+import { httpServiceMock } from '@kbn/core-http-browser-mocks';
+import * as Api from '@kbn/securitysolution-list-api';
+import { getListIndexExistSchemaMock } from '../mocks/response/read_list_index_schema.mock';
+import { useReadListIndex } from '.';
+
+jest.mock('@kbn/securitysolution-list-api');
+
+const customQueryProviderWrapper: React.FC<React.PropsWithChildren<{}>> = ({ children }) => {
+  const mockLogger = { error: jest.fn(), log: jest.fn(), warn: jest.fn() };
+  const queryClient = new QueryClient({ logger: mockLogger });
+  return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>;
+};
+
+describe('useReadListIndex', () => {
+  let httpMock: ReturnType<typeof httpServiceMock.createStartContract>;
+
+  beforeEach(() => {
+    httpMock = httpServiceMock.createStartContract();
+  });
+
+  describe('when both indices exist', () => {
+    beforeEach(() => {
+      (Api.readListIndex as jest.Mock).mockResolvedValue(getListIndexExistSchemaMock());
+    });
+
+    it('returns a response indicating both indices exist', async () => {
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: { list_index: true, list_item_index: true },
+            error: null,
+          })
+        );
+      });
+    });
+  });
+
+  describe('error conditions', () => {
+    let mockOnError: jest.Mock;
+
+    beforeEach(() => {
+      mockOnError = jest.fn();
+    });
+
+    it('does not call onError for an expected 404 response', async () => {
+      const notFoundError: SecurityAppError = {
+        message: 'Error',
+        name: 'Error',
+        body: {
+          message: 'data stream .lists-default does not exist',
+          status_code: 404,
+        },
+      };
+      (Api.readListIndex as jest.Mock).mockRejectedValue(notFoundError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+            onError: mockOnError,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            error: null,
+          })
+        );
+      });
+
+      expect(mockOnError).not.toHaveBeenCalled();
+    });
+
+    it('returns the appropriate response when neither list index exists', async () => {
+      const neitherFoundError: SecurityAppError = {
+        message: 'Error',
+        name: 'Error',
+        body: {
+          message: 'data stream .lists-default and data stream .items-default does not exist',
+          status_code: 404,
+        },
+      };
+      (Api.readListIndex as jest.Mock).mockRejectedValue(neitherFoundError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: { list_index: false, list_item_index: false },
+            error: null,
+          })
+        );
+      });
+    });
+
+    it('returns the appropriate response when only the lists index exists', async () => {
+      const neitherFoundError: SecurityAppError = {
+        message: 'Error',
+        name: 'Error',
+        body: {
+          message: 'data stream .items-default does not exist',
+          status_code: 404,
+        },
+      };
+      (Api.readListIndex as jest.Mock).mockRejectedValue(neitherFoundError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: { list_index: true, list_item_index: false },
+            error: null,
+          })
+        );
+      });
+    });
+
+    it('returns the appropriate response when only the items index exists', async () => {
+      const neitherFoundError: SecurityAppError = {
+        message: 'Error',
+        name: 'Error',
+        body: {
+          message: 'data stream .lists-default does not exist',
+          status_code: 404,
+        },
+      };
+      (Api.readListIndex as jest.Mock).mockRejectedValue(neitherFoundError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: { list_index: false, list_item_index: true },
+            error: null,
+          })
+        );
+      });
+    });
+
+    it('calls onError with and returns a generic error', async () => {
+      const genericError = new Error('Generic error');
+      (Api.readListIndex as jest.Mock).mockRejectedValue(genericError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+            onError: mockOnError,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: undefined,
+            error: genericError,
+          })
+        );
+      });
+      expect(mockOnError).toHaveBeenCalledWith(genericError);
+    });
+
+    it('calls onError with and returns a 404 error that does not mention an index not being found', async () => {
+      const genericNotFoundError: SecurityAppError = {
+        message: 'Error',
+        name: 'Error',
+        body: {
+          message: 'Not found',
+          status_code: 404,
+        },
+      };
+      (Api.readListIndex as jest.Mock).mockRejectedValue(genericNotFoundError);
+
+      const { result } = renderHook(
+        () =>
+          useReadListIndex({
+            http: httpMock,
+            isEnabled: true,
+            onError: mockOnError,
+          }),
+        { wrapper: customQueryProviderWrapper }
+      );
+
+      await waitFor(() => {
+        expect(result.current).toEqual(
+          expect.objectContaining({
+            loading: false,
+            result: undefined,
+            error: genericNotFoundError,
+          })
+        );
+      });
+      expect(mockOnError).toHaveBeenCalledWith(genericNotFoundError);
+    });
+  });
+});

x-pack/solutions/security/packages/kbn-securitysolution-list-hooks/src/use_read_list_index/index.ts

@@ -10,6 +10,11 @@ import { useQuery } from '@kbn/react-query';
 import type { ApiParams } from '@kbn/securitysolution-list-api';
 import { readListIndex } from '@kbn/securitysolution-list-api';
 import { withOptionalSignal } from '@kbn/securitysolution-hook-utils';
+import {
+  type SecurityAppError,
+  isSecurityAppError,
+  isNotFoundError,
+} from '@kbn/securitysolution-t-grid';
 
 import { READ_INDEX_QUERY_KEY } from '../constants';
 
@@ -31,7 +36,14 @@ export const useReadListIndex = ({
         return null;
       }
 
-      return readListIndexWithOptionalSignal({ http, signal });
+      try {
+        return await readListIndexWithOptionalSignal({ http, signal });
+      } catch (err) {
+        if (isIndexNotCreatedError(err)) {
+          return parseReadIndexResultFrom404Error(err);
+        }
+        return Promise.reject(err);
+      }
     },
     {
       onError,
@@ -48,3 +60,38 @@ export const useReadListIndex = ({
     error: query.error,
   };
 };
+
+const errorMentionsListsIndex = (error: SecurityAppError): boolean =>
+  error.body.message.includes('.lists-');
+const errorMentionsItemsIndex = (error: SecurityAppError): boolean =>
+  error.body.message.includes('.items-');
+
+/**
+ * Determines whether an error response from the `readListIndex`
+ * API call indicates that the index is not yet created.
+ */
+export const isIndexNotCreatedError = (err: unknown): err is SecurityAppError => {
+  return (
+    isSecurityAppError(err) &&
+    isNotFoundError(err) &&
+    (errorMentionsListsIndex(err) || errorMentionsItemsIndex(err))
+  );
+};
+
+/**
+ * Parses the result of a 404 error from the read list index API into an actionable response
+ *
+ * This endpoint returns a 404 if either of the underlying indices do
+ * not exist. By default, this causes the http client to throw an error,
+ * but as this is a valid result from the endpoint, we want to catch that error
+ * and present a result that is actionable to the consumer.
+ * @param error The 404 IndexNotCreated error returned from the read list index API
+ */
+export const parseReadIndexResultFrom404Error = (
+  error: SecurityAppError
+): Awaited<ReturnType<typeof readListIndexWithOptionalSignal>> => {
+  return {
+    list_index: !errorMentionsListsIndex(error),
+    list_item_index: !errorMentionsItemsIndex(error),
+  };
+};

x-pack/solutions/security/packages/kbn-securitysolution-list-hooks/tsconfig.json

@@ -9,6 +9,7 @@
   },
   "include": [
     "**/*.ts",
+    "**/*.tsx",
   ],
   "kbn_references": [
     "@kbn/securitysolution-hook-utils",
@@ -19,6 +20,7 @@
     "@kbn/core-http-browser-mocks",
     "@kbn/core-http-browser",
     "@kbn/react-query",
+    "@kbn/securitysolution-t-grid",
   ],
   "exclude": [
     "target/**/*",