Skip to content

Commit 81d3e02

Browse files
jeo02Copilot
jeo02
and
Copilot
authored
Fix issues when timeout occurs in test (#849)
* agent runner with aftereach * UPdate everywhere agent runner is used * update 2 * small fix * clean up when not in test * add back prepare for some reason lol * don't create mardown in test * Update tests/utils/agent-runner.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update tests/utils/agent-runner.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 4ce7e54 commit 81d3e02

File tree

21 files changed

+269
-198
lines changed

21 files changed

+269
-198
lines changed

‎tests/_template/integration.test.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import * as fs from "fs";
1515
import * as path from "path";
1616
import {
17-
run,
17+
useAgentRunner,
1818
isSkillInvoked,
1919
areToolCallsSuccess,
2020
doesAssistantMessageIncludeKeyword,
@@ -28,10 +28,11 @@ const SKILL_NAME = "your-skill-name";
2828
const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe;
2929

3030
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
31-
31+
const agent = useAgentRunner();
32+
3233
// Example test: Verify the skill is invoked for a relevant prompt
3334
test("invokes skill for relevant prompt", async () => {
34-
const agentMetadata = await run({
35+
const agentMetadata = await agent.run({
3536
prompt: "Your test prompt that should trigger this skill"
3637
});
3738

@@ -41,7 +42,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
4142

4243
// Example test: Verify expected content in response
4344
test("response contains expected keywords", async () => {
44-
const agentMetadata = await run({
45+
const agentMetadata = await agent.run({
4546
prompt: "Your test prompt here"
4647
});
4748

@@ -54,7 +55,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
5455

5556
// Example test: Verify MCP tool calls succeed
5657
test("MCP tool calls are successful", async () => {
57-
const agentMetadata = await run({
58+
const agentMetadata = await agent.run({
5859
prompt: "Your test prompt that uses Azure tools"
5960
});
6061

@@ -65,7 +66,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
6566

6667
// Example test with workspace setup
6768
test("works with project files", async () => {
68-
const agentMetadata = await run({
69+
const agentMetadata = await agent.run({
6970
setup: async (workspace: string) => {
7071
// Create any files needed in the workspace
7172
fs.writeFileSync(

‎tests/appinsights-instrumentation/integration.test.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import * as fs from "fs";
1515
import * as path from "path";
1616
import {
17-
run,
17+
useAgentRunner,
1818
isSkillInvoked,
1919
doesAssistantMessageIncludeKeyword,
2020
shouldSkipIntegrationTests,
@@ -38,13 +38,15 @@ if (skipTests && skipReason) {
3838
const describeIntegration = skipTests ? describe.skip : describe;
3939

4040
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
41+
const agent = useAgentRunner();
42+
4143
describe("skill-invocation", () => {
4244
test("invokes skill for App Insights instrumentation request", async () => {
4345
let successCount = 0;
4446

4547
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
4648
try {
47-
const agentMetadata = await run({
49+
const agentMetadata = await agent.run({
4850
prompt: "How do I add Application Insights to my ASP.NET Core web app?"
4951
});
5052

@@ -71,7 +73,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
7173

7274
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
7375
try {
74-
const agentMetadata = await run({
76+
const agentMetadata = await agent.run({
7577
setup: async (workspace: string) => {
7678
// Create a package.json to indicate Node.js project
7779
fs.writeFileSync(
@@ -102,7 +104,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
102104
});
103105

104106
test("response mentions auto-instrumentation for ASP.NET Core App Service app", async () => {
105-
const agentMetadata = await run({
107+
const agentMetadata = await agent.run({
106108
setup: async (workspace: string) => {
107109
fs.cpSync("./appinsights-instrumentation/resources/aspnetcore-app/", workspace, { recursive: true });
108110
},
@@ -119,7 +121,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
119121

120122
test("mentions App Insights in response", async () => {
121123
let workspacePath: string | undefined;
122-
const agentMetadata = await run({
124+
const agentMetadata = await agent.run({
123125
setup: async (workspace: string) => {
124126
workspacePath = workspace;
125127
fs.cpSync("./appinsights-instrumentation/resources/python-app/", workspace, { recursive: true });

‎tests/azure-ai/integration.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111

1212
import {
13-
run,
13+
useAgentRunner,
1414
isSkillInvoked,
1515
shouldSkipIntegrationTests,
1616
getIntegrationSkipReason
@@ -33,13 +33,15 @@ if (skipTests && skipReason) {
3333
const describeIntegration = skipTests ? describe.skip : describe;
3434

3535
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
36+
const agent = useAgentRunner();
37+
3638
describe("skill-invocation", () => {
3739
test("invokes azure-ai skill for AI Search query prompt", async () => {
3840
let successCount = 0;
3941

4042
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
4143
try {
42-
const agentMetadata = await run({
44+
const agentMetadata = await agent.run({
4345
prompt: "How do I create a vector search index in Azure AI Search?"
4446
});
4547

‎tests/azure-aigateway/integration.test.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111

1212
import {
13-
run,
13+
useAgentRunner,
1414
isSkillInvoked,
1515
shouldSkipIntegrationTests,
1616
getIntegrationSkipReason
@@ -33,13 +33,15 @@ if (skipTests && skipReason) {
3333
const describeIntegration = skipTests ? describe.skip : describe;
3434

3535
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
36+
const agent = useAgentRunner();
37+
3638
describe("skill-invocation", () => {
3739
test("invokes azure-aigateway skill for API Management gateway prompt", async () => {
3840
let successCount = 0;
3941

4042
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
4143
try {
42-
const agentMetadata = await run({
44+
const agentMetadata = await agent.run({
4345
prompt: "How do I set up Azure API Management as an AI Gateway for my Azure OpenAI models?"
4446
});
4547

@@ -66,7 +68,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
6668

6769
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
6870
try {
69-
const agentMetadata = await run({
71+
const agentMetadata = await agent.run({
7072
prompt: "How do I add rate limiting and token limits to my AI model requests using APIM?"
7173
});
7274

‎tests/azure-compliance/integration.test.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111

1212
import {
13-
run,
13+
useAgentRunner,
1414
isSkillInvoked,
1515
shouldSkipIntegrationTests,
1616
getIntegrationSkipReason
@@ -33,13 +33,15 @@ if (skipTests && skipReason) {
3333
const describeIntegration = skipTests ? describe.skip : describe;
3434

3535
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
36+
const agent = useAgentRunner();
37+
3638
describe("skill-invocation", () => {
3739
test("invokes azure-compliance skill for comprehensive compliance assessment prompt", async () => {
3840
let successCount = 0;
3941

4042
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
4143
try {
42-
const agentMetadata = await run({
44+
const agentMetadata = await agent.run({
4345
prompt: "Run azqr to check my Azure subscription for compliance against best practices"
4446
});
4547

@@ -66,7 +68,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
6668

6769
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
6870
try {
69-
const agentMetadata = await run({
71+
const agentMetadata = await agent.run({
7072
prompt: "Show me expired certificates and secrets in my Azure Key Vault"
7173
});
7274

@@ -93,7 +95,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
9395

9496
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
9597
try {
96-
const agentMetadata = await run({
98+
const agentMetadata = await agent.run({
9799
prompt: "Run a full compliance audit on my Azure environment including resource best practices and Key Vault expiration checks"
98100
});
99101

@@ -120,7 +122,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
120122

121123
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
122124
try {
123-
const agentMetadata = await run({
125+
const agentMetadata = await agent.run({
124126
prompt: "Check my Azure subscription for orphaned resources and compliance issues"
125127
});
126128

‎tests/azure-cost-optimization/integration.test.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
*/
1111

1212
import {
13-
run,
13+
useAgentRunner,
1414
isSkillInvoked,
1515
shouldSkipIntegrationTests,
1616
getIntegrationSkipReason,
@@ -34,13 +34,15 @@ if (skipTests && skipReason) {
3434
const describeIntegration = skipTests ? describe.skip : describe;
3535

3636
describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
37+
const agent = useAgentRunner();
38+
3739
describe("skill-invocation", () => {
3840
test("invokes azure-cost-optimization skill for cost savings prompt", async () => {
3941
let successCount = 0;
4042

4143
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
4244
try {
43-
const agentMetadata = await run({
45+
const agentMetadata = await agent.run({
4446
prompt: "How can I reduce my Azure spending and find cost savings in my subscription?"
4547
});
4648

@@ -67,7 +69,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
6769

6870
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
6971
try {
70-
const agentMetadata = await run({
72+
const agentMetadata = await agent.run({
7173
prompt: "Find orphaned and unused resources in my Azure subscription that I can delete"
7274
});
7375

@@ -94,7 +96,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
9496

9597
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
9698
try {
97-
const agentMetadata = await run({
99+
const agentMetadata = await agent.run({
98100
prompt: "Rightsize my Azure VMs to reduce costs"
99101
});
100102

@@ -121,7 +123,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
121123

122124
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
123125
try {
124-
const agentMetadata = await run({
126+
const agentMetadata = await agent.run({
125127
prompt: "How can I optimize my Azure Redis costs?"
126128
});
127129

@@ -148,7 +150,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
148150

149151
for (let i = 0; i < RUNS_PER_PROMPT; i++) {
150152
try {
151-
const agentMetadata = await run({
153+
const agentMetadata = await agent.run({
152154
prompt: "Find unused storage accounts to reduce my Azure costs"
153155
});
154156

@@ -173,7 +175,7 @@ describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
173175

174176
test("response mentions Cost Management for cost analysis", async () => {
175177
try {
176-
const agentMetadata = await run({
178+
const agentMetadata = await agent.run({
177179
prompt: "Analyze my Azure costs and show me where I can save money"
178180
});
179181

0 commit comments

Comments
 (0)