[DO NOT MERGE]Debug assistant test

.github/workflows/assistant-release-e2e-workflow.yml

@@ -24,7 +24,7 @@ jobs:
       test-name: dashboards assistant
       test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js'
       osd-serve-args: --assistant.chat.enabled=true
-      artifact-name-suffix: "-with-security"
+      artifact-name-suffix: '-with-security'
 
   tests-without-security:
     needs: changes
@@ -35,7 +35,7 @@ jobs:
       test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js'
       osd-serve-args: --assistant.chat.enabled=true
       security-enabled: false
-      artifact-name-suffix: "-without-security"
+      artifact-name-suffix: '-without-security'
       multi-opensearch-enabled: false
 
   tests-with-multiple-data-source-and-disabled-local-cluster:
@@ -47,7 +47,7 @@ jobs:
       test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_chatbot*.js'
       osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true
       security-enabled: true
-      artifact-name-suffix: "-with-security-and-mds"
+      artifact-name-suffix: '-with-security-and-mds'
 
   tests-with-query-enhancements:
     needs: changes
@@ -58,5 +58,15 @@ jobs:
       test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_query_enhancements*.js'
       osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true --assistant.alertInsight.enabled=true --assistant.smartAnomalyDetector.enabled=true --queryEnhancements.queryAssist.summary.enabled=true --uiSettings.overrides.home:useNewHomePage=true --uiSettings.overrides.query:enhancements:enabled=true --workspace.enabled=true --savedObjects.permission.enabled=true --assistant.text2viz.enabled=true  --opensearchDashboards.dashboardAdmin.users='["admin"]' --opensearch_security.multitenancy.enabled=false
       security-enabled: true
-      artifact-name-suffix: "-with-query-enhancements"
+      artifact-name-suffix: '-with-query-enhancements'
 
+  tests-with-assistant-feature-flag:
+    needs: changes
+    if: ${{ needs.changes.outputs.tests == 'true' }}
+    uses: ./.github/workflows/release-e2e-workflow-template.yml
+    with:
+      test-name: dashboards assistant
+      test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js'
+      osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true --assistant.alertInsight.enabled=true --assistant.smartAnomalyDetector.enabled=true --queryEnhancements.queryAssist.summary.enabled=true --uiSettings.overrides.home:useNewHomePage=true --uiSettings.overrides.query:enhancements:enabled=true --workspace.enabled=true --savedObjects.permission.enabled=true --assistant.text2viz.enabled=true  --opensearchDashboards.dashboardAdmin.users='["admin","workspace-test"]' --opensearch_security.multitenancy.enabled=false
+      security-enabled: true
+      artifact-name-suffix: '-with-assistant-feature-flag'

cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json

@@ -120,7 +120,7 @@
           },
           "user_inputs": {
             "parameters": {
-              "prompt": "\n\nHuman:\" turn\" You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]}\n\n. \n\nHuman:\" turn\":You will be given a chat history between OpenSearch Assistant and a Human.\nUse the context provided to generate follow up questions the Human would ask to the Assistant.\nThe Assistant can answer general questions about logs, traces and metrics.\nAssistant can access a set of tools listed below to answer questions given by the Human:\nQuestion suggestions generator tool\nHere's the chat history between the human and the Assistant.\n${parameters.AgentTool.output}\nUse the following steps to generate follow up questions Human may ask after the response of the Assistant:\nStep 1. Use the chat history to understand what human is trying to search and explore.\nStep 2. Understand what capabilities the assistant has with the set of tools it has access to.\nStep 3. Use the above context and generate follow up questions.Step4:You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]} \n \n----------------\n\nAssistant:"
+              "prompt": "You are an AI that only speaks JSON"
             },
             "description": "A general tool to answer any question.",
             "alias": "language_model_tool",

cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js

@@ -0,0 +1,316 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { BASE_PATH } from '../../../utils/constants';
+import sampleQueryLevelMonitorForAlertSummary from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_query_level_monitor_for_alert_summary.json';
+import { ADMIN_AUTH } from '../../../utils/commands';
+import workspaceTestUser from '../../../fixtures/dashboard/opensearch_dashboards/workspace/workspaceTestUser.json';
+import workspaceTestRole from '../../../fixtures/dashboard/opensearch_dashboards/workspace/workspaceTestRole.json';
+
+const workspaceName = `test_workspace_${Math.random()
+  .toString(36)
+  .substring(7)}`;
+
+const toggleOnAIFeature = () => {
+  cy.getElementByTestId('advancedSetting-resetField-enableAIFeatures').click();
+  cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').should(
+    'have.attr',
+    'aria-checked',
+    'true'
+  );
+  cy.getElementByTestId('advancedSetting-saveButton')
+    .should('be.visible')
+    .click();
+};
+
+const toggleOffAIFeature = () => {
+  cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').click();
+  cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').should(
+    'have.attr',
+    'aria-checked',
+    'false'
+  );
+  cy.getElementByTestId('advancedSetting-saveButton')
+    .should('be.visible')
+    .click();
+};
+
+function dashboardsAssistantFeatureFlagTestCases() {
+  describe('Dashboards assistant feature flag', () => {
+    let workspaceId;
+    let dataSourceId;
+    before(() => {
+      cy.createDataSourceNoAuth().then((result) => {
+        dataSourceId = result[0];
+        cy.createWorkspace({
+          name: workspaceName,
+          description: 'Workspace for cypress testing',
+          features: ['use-case-all'],
+          settings: {
+            permissions: {
+              library_write: { users: ['%me%'] },
+              write: { users: ['%me%'] },
+            },
+            dataSources: [dataSourceId],
+          },
+        })
+          .then((wsId) => {
+            workspaceId = wsId;
+          })
+          .then(() =>
+            cy.loadSampleDataForWorkspace(
+              'ecommerce',
+              workspaceId,
+              dataSourceId
+            )
+          );
+      });
+    });
+
+    after(() => {
+      if (workspaceId) {
+        if (dataSourceId) {
+          cy.removeSampleDataForWorkspace(
+            'ecommerce',
+            workspaceId,
+            dataSourceId
+          );
+        }
+        cy.deleteWorkspaceById(workspaceId);
+      }
+      if (dataSourceId) {
+        cy.deleteDataSource(dataSourceId);
+      }
+    });
+
+    describe('AI feature flag enabled', () => {
+      it('AI feature flag toggle on advanced settings page should be enabled', () => {
+        cy.visit(`${BASE_PATH}/app/settings`);
+
+        cy.getElementByTestId(
+          'advancedSetting-editField-enableAIFeatures'
+        ).then(($el) => {
+          if ($el.attr('aria-checked') === 'false') {
+            toggleOnAIFeature();
+          } else {
+            cy.getElementByTestId(
+              'advancedSetting-resetField-enableAIFeatures'
+            ).should('not.exist');
+          }
+        });
+      });
+
+      it('Data2summary and suggested AD should be available on the discover', () => {
+        cy.visit(`w/${workspaceId}/app/discover`);
+
+        cy.get('.languageSelector__button').should('be.visible').click();
+        cy.contains('button', 'PPL').should('be.visible').click();
+        cy.getElementByTestId('languageReferenceButton')
+          .should('be.visible')
+          .click();
+        cy.getElementByTestId('queryAssist__summary').should('be.visible');
+
+        cy.get('button[aria-label="OpenSearch assistant trigger button', {
+          timeout: 60000,
+        }).click();
+        cy.contains('Suggest anomaly detector').click();
+        cy.get('.add-anomaly-detector').should('be.visible');
+      });
+
+      it('Chatbot should be available on the overview page', () => {
+        cy.visit(`w/${workspaceId}/app/all_overview`);
+
+        cy.openAssistantChatbot();
+      });
+
+      it('Alert summary should be available on the alert', () => {
+        const dataSourceUrl = Cypress.env('remoteDataSourceBasicAuthUrl');
+        cy.createMonitor(sampleQueryLevelMonitorForAlertSummary, dataSourceUrl);
+
+        cy.wait(80000);
+
+        cy.visit(`w/${workspaceId}/app/alerts`);
+        cy.get('.incontextInsightAnchorIcon', { timeout: 60000 }).should(
+          'be.visible'
+        );
+
+        cy.deleteAllAlerts(dataSourceUrl);
+        cy.deleteAllMonitors(dataSourceUrl);
+      });
+    });
+
+    describe('AI feature flag disabled', () => {
+      after(() => {
+        cy.visit(`${BASE_PATH}/app/settings`);
+
+        cy.reload();
+
+        cy.getElementByTestId('advancedSetting-editField-enableAIFeatures')
+          .should('exist')
+          .and('be.visible')
+          .click();
+        cy.getElementByTestId('advancedSetting-saveButton')
+          .should('be.visible')
+          .click();
+      });
+
+      it('AI feature flag on advanced settings page can be toggle off', () => {
+        cy.visit(`${BASE_PATH}/app/settings`);
+
+        cy.contains('Enable AI features');
+        cy.getElementByTestId(
+          'advancedSetting-editField-enableAIFeatures'
+        ).then(($el) => {
+          if ($el.attr('aria-checked') === 'true') {
+            toggleOffAIFeature();
+          } else {
+            cy.getElementByTestId(
+              'advancedSetting-resetField-enableAIFeatures'
+            ).should('be.visible');
+          }
+        });
+        cy.getElementByTestId(
+          'advancedSetting-resetField-enableAIFeatures'
+        ).should('be.visible');
+      });
+
+      it('Data2summary and suggested AD should be unavailable on the discover', () => {
+        cy.visit(`w/${workspaceId}/app/discover`);
+
+        cy.get('.languageSelector__button').should('be.visible').click();
+        cy.contains('button', 'PPL').should('be.visible').click();
+        cy.getElementByTestId('languageReferenceButton')
+          .should('be.visible')
+          .click();
+
+        cy.getElementByTestId('queryAssist__summary').should('not.exist');
+        cy.get('button[aria-label="OpenSearch assistant trigger button').should(
+          'not.exist'
+        );
+      });
+
+      it('Chatbot should be unavailable on the overview page', () => {
+        cy.visit(`w/${workspaceId}/app/all_overview`);
+
+        cy.get('button[aria-label="toggle chat flyout icon').should(
+          'not.exist'
+        );
+      });
+
+      it('Alert summary should be unavailable on the alert', () => {
+        const dataSourceUrl = Cypress.env('remoteDataSourceBasicAuthUrl');
+        cy.createMonitor(sampleQueryLevelMonitorForAlertSummary);
+
+        cy.wait(80000);
+
+        cy.visit(`w/${workspaceId}/app/alerts`);
+        cy.get('.euiTableRow').should('exist').and('be.visible');
+        cy.get('.incontextInsightAnchorIcon').should('not.exist');
+
+        cy.deleteAllAlerts(dataSourceUrl);
+        cy.deleteAllMonitors(dataSourceUrl);
+      });
+    });
+  });
+}
+
+function dashboardAdminUiSettingsTestCases() {
+  describe('Dashboard admin UI settings', () => {
+    describe('Dashboard amin user', () => {
+      it('can toggle AI feature flag', () => {
+        cy.visit(`${BASE_PATH}/app/settings`);
+
+        cy.contains('Enable AI features');
+
+        cy.getElementByTestId(
+          'advancedSetting-editField-enableAIFeatures'
+        ).then(($el) => {
+          if ($el.attr('aria-checked') === 'false') {
+            toggleOnAIFeature();
+
+            cy.reload();
+          }
+          toggleOffAIFeature();
+        });
+      });
+
+      it('can delete dashboard admin settings saved object', () => {
+        cy.visit(`${BASE_PATH}/app/objects`);
+
+        cy.getElementByTestId('savedObjectsTableRow row-_dashboard_admin')
+          .contains('_dashboard_admin')
+          .getElementByTestId('checkboxSelectRow-_dashboard_admin')
+          .click();
+
+        cy.getElementByTestId('savedObjectsManagementDelete')
+          .should('exist')
+          .click();
+        cy.getElementByTestId('confirmModalConfirmButton')
+          .should('exist')
+          .click();
+
+        cy.visit(`${BASE_PATH}/app/settings`);
+      });
+    });
+
+    describe('Non-dashboard admin user', () => {
+      const NONE_DASHBOARDS_ADMIN_USERNAME = 'workspace-test';
+      const WORKSPACE_TEST_ROLE_NAME = 'workspace-test-role';
+      const originalUser = ADMIN_AUTH.username;
+      const originalPassword = ADMIN_AUTH.password;
+
+      before(() => {
+        cy.createInternalUser(
+          NONE_DASHBOARDS_ADMIN_USERNAME,
+          workspaceTestUser
+        );
+        cy.createRole(WORKSPACE_TEST_ROLE_NAME, workspaceTestRole);
+        cy.createRoleMapping(WORKSPACE_TEST_ROLE_NAME, {
+          users: [NONE_DASHBOARDS_ADMIN_USERNAME],
+        });
+      });
+
+      beforeEach(() => {
+        // Login as non OSD admin user
+        ADMIN_AUTH.newUser = NONE_DASHBOARDS_ADMIN_USERNAME;
+        ADMIN_AUTH.newPassword = workspaceTestUser.password;
+      });
+
+      after(() => {
+        ADMIN_AUTH.newUser = originalUser;
+        ADMIN_AUTH.newPassword = originalPassword;
+        cy.deleteRoleMapping(WORKSPACE_TEST_ROLE_NAME);
+        cy.deleteInternalUser(NONE_DASHBOARDS_ADMIN_USERNAME);
+        cy.deleteRole(WORKSPACE_TEST_ROLE_NAME);
+      });
+
+      it('cannot toggle AI feature flag', () => {
+        cy.visit(`${BASE_PATH}/app/settings`);
+
+        cy.contains('Enable AI features');
+        cy.getElementByTestId(
+          'advancedSetting-editField-enableAIFeatures'
+        ).should('be.disabled');
+      });
+
+      it('cannot see admin settings saved object in assets page', () => {
+        cy.visit(`${BASE_PATH}/app/objects`);
+
+        cy.getElementByTestId('savedObjectsTable').within(() => {
+          cy.get('.euiTableRow').should('be.visible');
+          cy.contains('_dashboard_admin').should('not.exist');
+        });
+      });
+    });
+  });
+}
+
+if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
+  dashboardsAssistantFeatureFlagTestCases();
+
+  if (Cypress.env('SECURITY_ENABLED')) {
+    dashboardAdminUiSettingsTestCases();
+  }
+}