Skip to content

Commit d233231

Browse files
opensearch-trigger-bot[bot]wanglam
opensearch-trigger-bot[bot]
and
wanglam
authored
Feat add tests for t2v (#1772) (#1781)
* Refactor chatbot agent registration * Add t2v test cases * Add query enhancements workflow job * Fix failed test cases * Fix lint * Fix putAgentIdConfig * Fix lint * Remove duplicate new home flag * Fix CYPRESS_WORKSPACE_ENABLED * Fix should not exists in Chai property --------- (cherry picked from commit e4de45a) Signed-off-by: Lin Wang <[email protected]> Co-authored-by: Lin Wang <[email protected]>
1 parent bf51d4e commit d233231

File tree

16 files changed

+781
-155
lines changed

16 files changed

+781
-155
lines changed

‎.github/workflows/assistant-release-e2e-workflow.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
uses: ./.github/workflows/release-e2e-workflow-template.yml
2323
with:
2424
test-name: dashboards assistant
25-
test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*'
25+
test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js'
2626
osd-serve-args: --assistant.chat.enabled=true
2727
artifact-name-suffix: "-with-security"
2828

@@ -32,7 +32,7 @@ jobs:
3232
uses: ./.github/workflows/release-e2e-workflow-template.yml
3333
with:
3434
test-name: dashboards assistant
35-
test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*'
35+
test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*chatbot*.js'
3636
osd-serve-args: --assistant.chat.enabled=true
3737
security-enabled: false
3838
artifact-name-suffix: "-without-security"
@@ -44,8 +44,19 @@ jobs:
4444
uses: ./.github/workflows/release-e2e-workflow-template.yml
4545
with:
4646
test-name: dashboards assistant
47-
test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds*.js'
47+
test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_chatbot*.js'
4848
osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true
4949
security-enabled: true
5050
artifact-name-suffix: "-with-security-and-mds"
5151

52+
tests-with-query-enhancements:
53+
needs: changes
54+
if: ${{ needs.changes.outputs.tests == 'true' }}
55+
uses: ./.github/workflows/release-e2e-workflow-template.yml
56+
with:
57+
test-name: dashboards assistant
58+
test-command: env CYPRESS_DISABLE_LOCAL_CLUSTER=true CYPRESS_DATASOURCE_MANAGEMENT_ENABLED=true CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true CYPRESS_WORKSPACE_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/mds_query_enhancements*.js'
59+
osd-serve-args: --data_source.enabled=true --data_source.ssl.verificationMode=none --data_source.hideLocalCluster=true --assistant.chat.enabled=true --uiSettings.overrides.home:useNewHomePage=true --uiSettings.overrides.query:enhancements:enabled=true --workspace.enabled=true --savedObjects.permission.enabled=true --assistant.text2viz.enabled=true --opensearchDashboards.dashboardAdmin.users='["admin"]' --opensearch_security.multitenancy.enabled=false
60+
security-enabled: true
61+
artifact-name-suffix: "-with-query-enhancements"
62+

‎cypress/fixtures/plugins/dashboards-assistant/flow-template.json‎ renamed to ‎cypress/fixtures/plugins/dashboards-assistant/flow-templates/chat.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@
130130
}
131131
},
132132
{
133-
"id": "root_agent",
133+
"id": "os_chat",
134134
"type": "register_agent",
135135
"previous_node_inputs": {
136136
"agent_tool": "tools",

‎cypress/fixtures/plugins/dashboards-assistant/flow-templates/query-assist-ppl.json‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"name": "Cypress-register-query-assist-ppl-agent",
3+
"description": "Cypress Flow template",
4+
"use_case": "REGISTER_AGENT",
5+
"version": {
6+
"template": "1.0.0",
7+
"compatibility": ["2.12.0", "3.0.0"]
8+
},
9+
"workflows": {
10+
"provision": {
11+
"user_params": {},
12+
"nodes": [
13+
{
14+
"id": "create_connector_1",
15+
"type": "create_connector",
16+
"previous_node_inputs": {},
17+
"user_inputs": {
18+
"version": "1",
19+
"name": "Claude instant runtime T2V with instructions Connector",
20+
"protocol": "aws_sigv4",
21+
"description": "The connector to BedRock service for claude model",
22+
"actions": [
23+
{
24+
"headers": {
25+
"x-amz-content-sha256": "required",
26+
"content-type": "application/json"
27+
},
28+
"method": "POST",
29+
"request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }",
30+
"action_type": "predict",
31+
"url": "http://127.0.0.1:3000"
32+
}
33+
],
34+
"credential": {
35+
"access_key": "<key>",
36+
"secret_key": "<value>"
37+
},
38+
"parameters": {
39+
"endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
40+
"content_type": "application/json",
41+
"auth": "Sig_V4",
42+
"max_tokens_to_sample": "8000",
43+
"service_name": "bedrock",
44+
"temperature": "0.0001",
45+
"response_filter": "$.completion",
46+
"region": "us-west-2",
47+
"anthropic_version": "bedrock-2023-05-31"
48+
}
49+
}
50+
},
51+
{
52+
"id": "register_model_2",
53+
"type": "register_remote_model",
54+
"previous_node_inputs": {
55+
"create_connector_1": "connector_id"
56+
},
57+
"user_inputs": {
58+
"description": "test model",
59+
"deploy": true,
60+
"name": "claude-instant-t2v-with-instructions"
61+
}
62+
},
63+
{
64+
"id": "ppl_tool",
65+
"type": "create_tool",
66+
"previous_node_inputs": {
67+
"register_model_2": "model_id"
68+
},
69+
"user_inputs": {
70+
"type": "PPLTool",
71+
"name": "TransferQuestionToPPLAndExecuteTool",
72+
"description": "Use this tool to transfer natural language to generate PPL and execute PPL to query inside. Use this tool after you know the index name, otherwise, call IndexRoutingTool first. The input parameters are: {index:IndexName, question:UserQuestion}",
73+
"parameters": {
74+
"model_type": "FINETUNE",
75+
"execute": false
76+
}
77+
}
78+
},
79+
{
80+
"id": "os_query_assist_ppl",
81+
"type": "register_agent",
82+
"previous_node_inputs": {
83+
"register_model_2": "model_id",
84+
"ppl_tool": "tools"
85+
},
86+
"user_inputs": {
87+
"parameters": {},
88+
"type": "flow",
89+
"name": "t2vega with instructions agent",
90+
"description": "this is the t2vega with instructions agent",
91+
"tools_order": ["ppl_tool"]
92+
}
93+
}
94+
]
95+
}
96+
}
97+
}

‎cypress/fixtures/plugins/dashboards-assistant/flow-templates/text2vega-with-instructions.json‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
{
2+
"name": "Cypress-register-text2vega-with-instructions-agent",
3+
"description": "Cypress Flow template",
4+
"use_case": "REGISTER_AGENT",
5+
"version": {
6+
"template": "1.0.0",
7+
"compatibility": ["2.12.0", "3.0.0"]
8+
},
9+
"workflows": {
10+
"provision": {
11+
"user_params": {},
12+
"nodes": [
13+
{
14+
"id": "create_connector_1",
15+
"type": "create_connector",
16+
"previous_node_inputs": {},
17+
"user_inputs": {
18+
"version": "1",
19+
"name": "Claude instant runtime T2V with instructions Connector",
20+
"protocol": "aws_sigv4",
21+
"description": "The connector to BedRock service for claude model",
22+
"actions": [
23+
{
24+
"headers": {
25+
"x-amz-content-sha256": "required",
26+
"content-type": "application/json"
27+
},
28+
"method": "POST",
29+
"request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }",
30+
"action_type": "predict",
31+
"url": "http://127.0.0.1:3000"
32+
}
33+
],
34+
"credential": {
35+
"access_key": "<key>",
36+
"secret_key": "<value>"
37+
},
38+
"parameters": {
39+
"endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
40+
"content_type": "application/json",
41+
"auth": "Sig_V4",
42+
"max_tokens_to_sample": "8000",
43+
"service_name": "bedrock",
44+
"temperature": "0.0001",
45+
"response_filter": "$.completion",
46+
"region": "us-west-2",
47+
"anthropic_version": "bedrock-2023-05-31"
48+
}
49+
}
50+
},
51+
{
52+
"id": "register_model_2",
53+
"type": "register_remote_model",
54+
"previous_node_inputs": {
55+
"create_connector_1": "connector_id"
56+
},
57+
"user_inputs": {
58+
"description": "test model",
59+
"deploy": true,
60+
"name": "claude-instant-t2v-with-instructions"
61+
}
62+
},
63+
{
64+
"id": "ml_model_tool",
65+
"type": "create_tool",
66+
"previous_node_inputs": {
67+
"register_model_2": "model_id"
68+
},
69+
"user_inputs": {
70+
"parameters": {
71+
"prompt": "t2v instruction agent prompt"
72+
},
73+
"description": "A general tool to answer any question.",
74+
"alias": "language_model_tool",
75+
"include_output_in_agent_response": true,
76+
"name": "T2VWithInstructionsModelTool",
77+
"type": "MLModelTool"
78+
}
79+
},
80+
{
81+
"id": "os_text2vega_with_instructions",
82+
"type": "register_agent",
83+
"previous_node_inputs": {
84+
"register_model_2": "model_id",
85+
"ml_model_tool": "tools"
86+
},
87+
"user_inputs": {
88+
"parameters": {},
89+
"type": "flow",
90+
"name": "t2vega with instructions agent",
91+
"description": "this is the t2vega with instructions agent",
92+
"tools_order": ["ml_model_tool"]
93+
}
94+
}
95+
]
96+
}
97+
}
98+
}

‎cypress/fixtures/plugins/dashboards-assistant/flow-templates/text2vega.json‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
{
2+
"name": "Cypress-register-text2vega-agent",
3+
"description": "Cypress Flow template",
4+
"use_case": "REGISTER_AGENT",
5+
"version": {
6+
"template": "1.0.0",
7+
"compatibility": ["2.12.0", "3.0.0"]
8+
},
9+
"workflows": {
10+
"provision": {
11+
"user_params": {},
12+
"nodes": [
13+
{
14+
"id": "create_connector_1",
15+
"type": "create_connector",
16+
"previous_node_inputs": {},
17+
"user_inputs": {
18+
"version": "1",
19+
"name": "Claude instant runtime T2V Connector",
20+
"protocol": "aws_sigv4",
21+
"description": "The connector to BedRock service for claude model",
22+
"actions": [
23+
{
24+
"headers": {
25+
"x-amz-content-sha256": "required",
26+
"content-type": "application/json"
27+
},
28+
"method": "POST",
29+
"request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }",
30+
"action_type": "predict",
31+
"url": "http://127.0.0.1:3000"
32+
}
33+
],
34+
"credential": {
35+
"access_key": "<key>",
36+
"secret_key": "<value>"
37+
},
38+
"parameters": {
39+
"endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
40+
"content_type": "application/json",
41+
"auth": "Sig_V4",
42+
"max_tokens_to_sample": "8000",
43+
"service_name": "bedrock",
44+
"temperature": "0.0001",
45+
"response_filter": "$.completion",
46+
"region": "us-west-2",
47+
"anthropic_version": "bedrock-2023-05-31"
48+
}
49+
}
50+
},
51+
{
52+
"id": "register_model_2",
53+
"type": "register_remote_model",
54+
"previous_node_inputs": {
55+
"create_connector_1": "connector_id"
56+
},
57+
"user_inputs": {
58+
"description": "test model",
59+
"deploy": true,
60+
"name": "claude-instant-t2v"
61+
}
62+
},
63+
{
64+
"id": "ml_model_tool",
65+
"type": "create_tool",
66+
"previous_node_inputs": {
67+
"register_model_2": "model_id"
68+
},
69+
"user_inputs": {
70+
"parameters": {
71+
"prompt": "t2v agent prompt"
72+
},
73+
"description": "A general tool to answer any question.",
74+
"alias": "language_model_tool",
75+
"include_output_in_agent_response": true,
76+
"name": "T2VModelTool",
77+
"type": "MLModelTool"
78+
}
79+
},
80+
{
81+
"id": "os_text2vega",
82+
"type": "register_agent",
83+
"previous_node_inputs": {
84+
"register_model_2": "model_id",
85+
"ml_model_tool": "tools"
86+
},
87+
"user_inputs": {
88+
"parameters": {},
89+
"type": "flow",
90+
"name": "t2vega agent",
91+
"description": "this is the t2vega agent",
92+
"tools_order": ["ml_model_tool"]
93+
}
94+
}
95+
]
96+
}
97+
}
98+
}

‎cypress/fixtures/plugins/dashboards-assistant/query-assist-ppl-response.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"completion":
3+
"source=opensearch_dashboards_sample_data_ecommerce| eval week = WEEK_OF_YEAR(`order_date`) | stats DISTINCT_COUNT(`customer_id`) as visitors by week | sort week",
4+
"stop_reason": "stop_sequence",
5+
"stop": "\n\nHuman:"
6+
}

‎cypress/fixtures/plugins/dashboards-assistant/text2vega-response.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"completion": "{\"title\":\"Number of Visitors by Week\",\"description\":\"Number of visitors grouped by week\",\"mark\":\"bar\",\"encoding\":{\"x\":{\"field\":\"visitors\",\"type\":\"quantitative\"},\"y\":{\"field\":\"week\",\"type\":\"nominal\"}}}",
3+
"stop_reason": "stop_sequence",
4+
"stop": "\n\nHuman:"
5+
}

‎cypress/fixtures/plugins/dashboards-assistant/text2vega-with-instructions-response.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"completion": "{\"title\":\"Number of Visitors by Week\",\"description\":\"A pie chart showing the number of visitors grouped by week.\",\"layer\":[{\"mark\":\"arc\",\"encoding\":{\"theta\":{\"field\":\"visitors\",\"type\":\"quantitative\"},\"color\":{\"field\":\"week\",\"type\":\"ordinal\"}}}]}",
3+
"stop_reason": "stop_sequence",
4+
"stop": "\n\nHuman:"
5+
}

‎cypress/integration/plugins/dashboards-assistant/conversation_save_to_notebook_spec.js‎ renamed to ‎cypress/integration/plugins/dashboards-assistant/conversation_chatbot_save_to_notebook_spec.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,12 @@ if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
4040
timeout: 60000,
4141
}).should('be.length', 1);
4242

43+
// The header may render multiple times, wait for UI to be stable
44+
cy.wait(5000);
4345
// enable to toggle and show Chatbot
44-
cy.get(`button[aria-label="toggle chat flyout icon"]`).click();
46+
cy.get(`button[aria-label="toggle chat flyout icon"]`).click({
47+
force: true,
48+
});
4549

4650
// click suggestions to generate response
4751
cy.contains('What are the indices in my cluster?').click();

‎cypress/integration/plugins/dashboards-assistant/mds_conversation_history_spec.js‎ renamed to ‎cypress/integration/plugins/dashboards-assistant/mds_chatbot_conversation_history_spec.js‎

File renamed without changes.

0 commit comments

Comments
 (0)