diff --git a/.github/workflows/assistant-release-e2e-workflow.yml b/.github/workflows/assistant-release-e2e-workflow.yml index 6cf479adb..7ac25b3ae 100644 --- a/.github/workflows/assistant-release-e2e-workflow.yml +++ b/.github/workflows/assistant-release-e2e-workflow.yml @@ -24,7 +24,7 @@ jobs: test-name: dashboards assistant test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js' osd-serve-args: --assistant.chat.enabled=true - artifact-name-suffix: "-with-security" + artifact-name-suffix: '-with-security' tests-without-security: needs: changes @@ -35,7 +35,7 @@ jobs: test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js' osd-serve-args: --assistant.chat.enabled=true security-enabled: false - artifact-name-suffix: "-without-security" + artifact-name-suffix: '-without-security' multi-opensearch-enabled: false tests-with-multiple-data-source-and-disabled-local-cluster: @@ -47,7 +47,7 @@ jobs: test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_chatbot*.js' osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true security-enabled: true - artifact-name-suffix: "-with-security-and-mds" + artifact-name-suffix: '-with-security-and-mds' tests-with-query-enhancements: needs: changes @@ -58,5 +58,15 @@ jobs: test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_query_enhancements*.js' osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true --assistant.alertInsight.enabled=true --assistant.smartAnomalyDetector.enabled=true --queryEnhancements.queryAssist.summary.enabled=true --uiSettings.overrides.home:useNewHomePage=true --uiSettings.overrides.query:enhancements:enabled=true --workspace.enabled=true --savedObjects.permission.enabled=true --assistant.text2viz.enabled=true --opensearchDashboards.dashboardAdmin.users='["admin"]' --opensearch_security.multitenancy.enabled=false security-enabled: true - artifact-name-suffix: "-with-query-enhancements" + artifact-name-suffix: '-with-query-enhancements' + tests-with-assistant-feature-flag: + needs: changes + if: ${{ needs.changes.outputs.tests == 'true' }} + uses: ./.github/workflows/release-e2e-workflow-template.yml + with: + test-name: dashboards assistant + test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js' + osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true --assistant.alertInsight.enabled=true --assistant.smartAnomalyDetector.enabled=true --queryEnhancements.queryAssist.summary.enabled=true --uiSettings.overrides.home:useNewHomePage=true --uiSettings.overrides.query:enhancements:enabled=true --workspace.enabled=true --savedObjects.permission.enabled=true --assistant.text2viz.enabled=true --opensearchDashboards.dashboardAdmin.users='["admin","workspace-test"]' --opensearch_security.multitenancy.enabled=false + security-enabled: true + artifact-name-suffix: '-with-assistant-feature-flag' diff --git a/cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json b/cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json index accdeaf32..0d6bc9d36 100644 --- a/cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json +++ b/cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json @@ -120,7 +120,7 @@ }, "user_inputs": { "parameters": { - "prompt": "\n\nHuman:\" turn\" You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]}\n\n. \n\nHuman:\" turn\":You will be given a chat history between OpenSearch Assistant and a Human.\nUse the context provided to generate follow up questions the Human would ask to the Assistant.\nThe Assistant can answer general questions about logs, traces and metrics.\nAssistant can access a set of tools listed below to answer questions given by the Human:\nQuestion suggestions generator tool\nHere's the chat history between the human and the Assistant.\n${parameters.AgentTool.output}\nUse the following steps to generate follow up questions Human may ask after the response of the Assistant:\nStep 1. Use the chat history to understand what human is trying to search and explore.\nStep 2. Understand what capabilities the assistant has with the set of tools it has access to.\nStep 3. Use the above context and generate follow up questions.Step4:You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]} \n \n----------------\n\nAssistant:" + "prompt": "You are an AI that only speaks JSON" }, "description": "A general tool to answer any question.", "alias": "language_model_tool", diff --git a/cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js b/cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js new file mode 100644 index 000000000..23183dd8e --- /dev/null +++ b/cypress/integration/plugins/dashboards-assistant/mds_assistant_feature_flag_spec.js @@ -0,0 +1,316 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BASE_PATH } from '../../../utils/constants'; +import sampleQueryLevelMonitorForAlertSummary from '../../../fixtures/plugins/alerting-dashboards-plugin/sample_query_level_monitor_for_alert_summary.json'; +import { ADMIN_AUTH } from '../../../utils/commands'; +import workspaceTestUser from '../../../fixtures/dashboard/opensearch_dashboards/workspace/workspaceTestUser.json'; +import workspaceTestRole from '../../../fixtures/dashboard/opensearch_dashboards/workspace/workspaceTestRole.json'; + +const workspaceName = `test_workspace_${Math.random() + .toString(36) + .substring(7)}`; + +const toggleOnAIFeature = () => { + cy.getElementByTestId('advancedSetting-resetField-enableAIFeatures').click(); + cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').should( + 'have.attr', + 'aria-checked', + 'true' + ); + cy.getElementByTestId('advancedSetting-saveButton') + .should('be.visible') + .click(); +}; + +const toggleOffAIFeature = () => { + cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').click(); + cy.getElementByTestId('advancedSetting-editField-enableAIFeatures').should( + 'have.attr', + 'aria-checked', + 'false' + ); + cy.getElementByTestId('advancedSetting-saveButton') + .should('be.visible') + .click(); +}; + +function dashboardsAssistantFeatureFlagTestCases() { + describe('Dashboards assistant feature flag', () => { + let workspaceId; + let dataSourceId; + before(() => { + cy.createDataSourceNoAuth().then((result) => { + dataSourceId = result[0]; + cy.createWorkspace({ + name: workspaceName, + description: 'Workspace for cypress testing', + features: ['use-case-all'], + settings: { + permissions: { + library_write: { users: ['%me%'] }, + write: { users: ['%me%'] }, + }, + dataSources: [dataSourceId], + }, + }) + .then((wsId) => { + workspaceId = wsId; + }) + .then(() => + cy.loadSampleDataForWorkspace( + 'ecommerce', + workspaceId, + dataSourceId + ) + ); + }); + }); + + after(() => { + if (workspaceId) { + if (dataSourceId) { + cy.removeSampleDataForWorkspace( + 'ecommerce', + workspaceId, + dataSourceId + ); + } + cy.deleteWorkspaceById(workspaceId); + } + if (dataSourceId) { + cy.deleteDataSource(dataSourceId); + } + }); + + describe('AI feature flag enabled', () => { + it('AI feature flag toggle on advanced settings page should be enabled', () => { + cy.visit(`${BASE_PATH}/app/settings`); + + cy.getElementByTestId( + 'advancedSetting-editField-enableAIFeatures' + ).then(($el) => { + if ($el.attr('aria-checked') === 'false') { + toggleOnAIFeature(); + } else { + cy.getElementByTestId( + 'advancedSetting-resetField-enableAIFeatures' + ).should('not.exist'); + } + }); + }); + + it('Data2summary and suggested AD should be available on the discover', () => { + cy.visit(`w/${workspaceId}/app/discover`); + + cy.get('.languageSelector__button').should('be.visible').click(); + cy.contains('button', 'PPL').should('be.visible').click(); + cy.getElementByTestId('languageReferenceButton') + .should('be.visible') + .click(); + cy.getElementByTestId('queryAssist__summary').should('be.visible'); + + cy.get('button[aria-label="OpenSearch assistant trigger button', { + timeout: 60000, + }).click(); + cy.contains('Suggest anomaly detector').click(); + cy.get('.add-anomaly-detector').should('be.visible'); + }); + + it('Chatbot should be available on the overview page', () => { + cy.visit(`w/${workspaceId}/app/all_overview`); + + cy.openAssistantChatbot(); + }); + + it('Alert summary should be available on the alert', () => { + const dataSourceUrl = Cypress.env('remoteDataSourceBasicAuthUrl'); + cy.createMonitor(sampleQueryLevelMonitorForAlertSummary, dataSourceUrl); + + cy.wait(80000); + + cy.visit(`w/${workspaceId}/app/alerts`); + cy.get('.incontextInsightAnchorIcon', { timeout: 60000 }).should( + 'be.visible' + ); + + cy.deleteAllAlerts(dataSourceUrl); + cy.deleteAllMonitors(dataSourceUrl); + }); + }); + + describe('AI feature flag disabled', () => { + after(() => { + cy.visit(`${BASE_PATH}/app/settings`); + + cy.reload(); + + cy.getElementByTestId('advancedSetting-editField-enableAIFeatures') + .should('exist') + .and('be.visible') + .click(); + cy.getElementByTestId('advancedSetting-saveButton') + .should('be.visible') + .click(); + }); + + it('AI feature flag on advanced settings page can be toggle off', () => { + cy.visit(`${BASE_PATH}/app/settings`); + + cy.contains('Enable AI features'); + cy.getElementByTestId( + 'advancedSetting-editField-enableAIFeatures' + ).then(($el) => { + if ($el.attr('aria-checked') === 'true') { + toggleOffAIFeature(); + } else { + cy.getElementByTestId( + 'advancedSetting-resetField-enableAIFeatures' + ).should('be.visible'); + } + }); + cy.getElementByTestId( + 'advancedSetting-resetField-enableAIFeatures' + ).should('be.visible'); + }); + + it('Data2summary and suggested AD should be unavailable on the discover', () => { + cy.visit(`w/${workspaceId}/app/discover`); + + cy.get('.languageSelector__button').should('be.visible').click(); + cy.contains('button', 'PPL').should('be.visible').click(); + cy.getElementByTestId('languageReferenceButton') + .should('be.visible') + .click(); + + cy.getElementByTestId('queryAssist__summary').should('not.exist'); + cy.get('button[aria-label="OpenSearch assistant trigger button').should( + 'not.exist' + ); + }); + + it('Chatbot should be unavailable on the overview page', () => { + cy.visit(`w/${workspaceId}/app/all_overview`); + + cy.get('button[aria-label="toggle chat flyout icon').should( + 'not.exist' + ); + }); + + it('Alert summary should be unavailable on the alert', () => { + const dataSourceUrl = Cypress.env('remoteDataSourceBasicAuthUrl'); + cy.createMonitor(sampleQueryLevelMonitorForAlertSummary); + + cy.wait(80000); + + cy.visit(`w/${workspaceId}/app/alerts`); + cy.get('.euiTableRow').should('exist').and('be.visible'); + cy.get('.incontextInsightAnchorIcon').should('not.exist'); + + cy.deleteAllAlerts(dataSourceUrl); + cy.deleteAllMonitors(dataSourceUrl); + }); + }); + }); +} + +function dashboardAdminUiSettingsTestCases() { + describe('Dashboard admin UI settings', () => { + describe('Dashboard amin user', () => { + it('can toggle AI feature flag', () => { + cy.visit(`${BASE_PATH}/app/settings`); + + cy.contains('Enable AI features'); + + cy.getElementByTestId( + 'advancedSetting-editField-enableAIFeatures' + ).then(($el) => { + if ($el.attr('aria-checked') === 'false') { + toggleOnAIFeature(); + + cy.reload(); + } + toggleOffAIFeature(); + }); + }); + + it('can delete dashboard admin settings saved object', () => { + cy.visit(`${BASE_PATH}/app/objects`); + + cy.getElementByTestId('savedObjectsTableRow row-_dashboard_admin') + .contains('_dashboard_admin') + .getElementByTestId('checkboxSelectRow-_dashboard_admin') + .click(); + + cy.getElementByTestId('savedObjectsManagementDelete') + .should('exist') + .click(); + cy.getElementByTestId('confirmModalConfirmButton') + .should('exist') + .click(); + + cy.visit(`${BASE_PATH}/app/settings`); + }); + }); + + describe('Non-dashboard admin user', () => { + const NONE_DASHBOARDS_ADMIN_USERNAME = 'workspace-test'; + const WORKSPACE_TEST_ROLE_NAME = 'workspace-test-role'; + const originalUser = ADMIN_AUTH.username; + const originalPassword = ADMIN_AUTH.password; + + before(() => { + cy.createInternalUser( + NONE_DASHBOARDS_ADMIN_USERNAME, + workspaceTestUser + ); + cy.createRole(WORKSPACE_TEST_ROLE_NAME, workspaceTestRole); + cy.createRoleMapping(WORKSPACE_TEST_ROLE_NAME, { + users: [NONE_DASHBOARDS_ADMIN_USERNAME], + }); + }); + + beforeEach(() => { + // Login as non OSD admin user + ADMIN_AUTH.newUser = NONE_DASHBOARDS_ADMIN_USERNAME; + ADMIN_AUTH.newPassword = workspaceTestUser.password; + }); + + after(() => { + ADMIN_AUTH.newUser = originalUser; + ADMIN_AUTH.newPassword = originalPassword; + cy.deleteRoleMapping(WORKSPACE_TEST_ROLE_NAME); + cy.deleteInternalUser(NONE_DASHBOARDS_ADMIN_USERNAME); + cy.deleteRole(WORKSPACE_TEST_ROLE_NAME); + }); + + it('cannot toggle AI feature flag', () => { + cy.visit(`${BASE_PATH}/app/settings`); + + cy.contains('Enable AI features'); + cy.getElementByTestId( + 'advancedSetting-editField-enableAIFeatures' + ).should('be.disabled'); + }); + + it('cannot see admin settings saved object in assets page', () => { + cy.visit(`${BASE_PATH}/app/objects`); + + cy.getElementByTestId('savedObjectsTable').within(() => { + cy.get('.euiTableRow').should('be.visible'); + cy.contains('_dashboard_admin').should('not.exist'); + }); + }); + }); + }); +} + +if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) { + dashboardsAssistantFeatureFlagTestCases(); + + if (Cypress.env('SECURITY_ENABLED')) { + dashboardAdminUiSettingsTestCases(); + } +} diff --git a/cypress/utils/commands.js b/cypress/utils/commands.js index e7fc067e8..a778c641b 100644 --- a/cypress/utils/commands.js +++ b/cypress/utils/commands.js @@ -27,43 +27,41 @@ export const CURRENT_TENANT = { }; // Overwrite default backend endpoint to customized one, remember set to original value after tests complete. -export const currentBackendEndpoint = (() => { - let currentEndpoint = BACKEND_BASE_PATH; - const DEFAULT_ENDPOINT = BACKEND_BASE_PATH; - const REMOTE_NO_AUTH_ENDPOINT = Cypress.env('remoteDataSourceNoAuthUrl'); - - return Object.freeze({ - DEFAULT: DEFAULT_ENDPOINT, - REMOTE_NO_AUTH: REMOTE_NO_AUTH_ENDPOINT, - /** - * Change current backend endpoint - * @param {*} changedEndPoint - * @param {*} immediately set immediately false to change tenant after all pending promise be invoked, - * useful for reset backend endpoint after all tests run. - */ - set(changedEndPoint, immediately = true) { - if ( - ![DEFAULT_ENDPOINT, REMOTE_NO_AUTH_ENDPOINT].includes(changedEndPoint) - ) { - throw new Error(`Invalid endpoint:${changedEndPoint}`); - } - const updateEndpoint = () => { - currentEndpoint = changedEndPoint; - cy.log( - `Current backend endpoint has been changed to: ${currentEndpoint}` - ); - }; - if (immediately) { - updateEndpoint(); - } else { - cy.wrap().then(updateEndpoint); - } - }, - get() { - return currentEndpoint; - }, - }); -})(); +export const currentBackendEndpoint = Object.freeze({ + DEFAULT: BACKEND_BASE_PATH, + REMOTE_NO_AUTH: Cypress.env('remoteDataSourceNoAuthUrl'), + /** + * Change current backend endpoint + * @param {*} changedEndPoint + * @param {*} immediately set immediately false to change tenant after all pending promise be invoked, + * useful for reset backend endpoint after all tests run. + */ + set(changedEndPoint, immediately = true) { + if ( + ![currentBackendEndpoint.DEFAULT, currentBackendEndpoint.REMOTE_NO_AUTH].includes( + changedEndPoint + ) + ) { + throw new Error(`Invalid endpoint:${changedEndPoint}`); + } + const updateEndpoint = () => { + Cypress.env('currentBackendEndpoint', changedEndPoint); + cy.log( + `Current backend endpoint has been changed to: ${changedEndPoint}` + ); + }; + if (immediately) { + updateEndpoint(); + } else { + cy.wrap().then(updateEndpoint); + } + }, + get() { + return ( + Cypress.env('currentBackendEndpoint') || currentBackendEndpoint.DEFAULT + ); + }, +}); export const supressNoRequestOccurred = () => { cy.on('fail', (err) => {