Use a more robust way to generate integration test report

JasonYeMSFT · JasonYeMSFT · commit 946e6ba1c555 · 2026-02-17T16:00:51.000-08:00
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -221,22 +221,6 @@ jobs:
           
           echo "All skill files have valid frontmatter"
 
-      - name: Check skill name prefix conflicts
-        if: steps.changed-skills.outputs.any_changed == 'true'
-        run: |
-          # Collect toplevel skill directory names under plugin/skills
-          NAMES=$(ls -d plugin/skills/*/ | xargs -n1 basename)
-          
-          # Check if any name is a prefix of another
-          while IFS= read -r a; do
-            while IFS= read -r b; do
-              if [ "$a" != "$b" ] && [[ "$b" == "$a"* ]]; then
-                echo "error: Skill name '$a' is a prefix of '$b'"
-                exit 1
-              fi
-            done <<< "$NAMES"
-          done <<< "$NAMES"
-
       - name: Skip message
         if: steps.changed-skills.outputs.any_changed != 'true'
         run: echo "No skill files changed - skipping validation"
diff --git a/tests/AGENTS.md b/tests/AGENTS.md
@@ -237,7 +237,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   test('invokes skill for relevant prompt', async () => {
     const agentMetadata = await run({
       prompt: 'What role should I assign for blob storage access?'
diff --git a/tests/_template/integration.test.ts b/tests/_template/integration.test.ts
@@ -27,7 +27,7 @@ const SKILL_NAME = "your-skill-name";
 // Use centralized skip logic from agent-runner
 const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   // Example test: Verify the skill is invoked for a relevant prompt
diff --git a/tests/appinsights-instrumentation/integration.test.ts b/tests/appinsights-instrumentation/integration.test.ts
@@ -37,7 +37,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-ai/integration.test.ts b/tests/azure-ai/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-aigateway/integration.test.ts b/tests/azure-aigateway/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-compliance/integration.test.ts b/tests/azure-compliance/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-cost-optimization/integration.test.ts b/tests/azure-cost-optimization/integration.test.ts
@@ -33,7 +33,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-deploy/integration.test.ts b/tests/azure-deploy/integration.test.ts
@@ -37,7 +37,7 @@ const describeIntegration = skipTests ? describe.skip : describe;
 const deployTestTimeoutMs = 1800000;
 const brownfieldTestTimeoutMs = 2700000;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
   describe("skill-invocation", () => {
     test("invokes azure-deploy skill for deployment prompt", async () => {
diff --git a/tests/azure-diagnostics/integration.test.ts b/tests/azure-diagnostics/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-kusto/integration.test.ts b/tests/azure-kusto/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-observability/integration.test.ts b/tests/azure-observability/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-postgres/integration.test.ts b/tests/azure-postgres/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-prepare/integration.test.ts b/tests/azure-prepare/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-rbac/integration.test.ts b/tests/azure-rbac/integration.test.ts
@@ -34,7 +34,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-resource-lookup/integration.test.ts b/tests/azure-resource-lookup/integration.test.ts
@@ -22,7 +22,7 @@ const SKILL_NAME = "azure-resource-lookup";
 
 const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   // Cross-subscription resource queries (ARG sweet spot)
diff --git a/tests/azure-resource-visualizer/integration.test.ts b/tests/azure-resource-visualizer/integration.test.ts
@@ -51,7 +51,7 @@ function hasArchitectureDiagramFile(agentMetadata: AgentMetadata): boolean {
   });
 }
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-storage/integration.test.ts b/tests/azure-storage/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/azure-validate/integration.test.ts b/tests/azure-validate/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/entra-app-registration/integration.test.ts b/tests/entra-app-registration/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("skill-invocation", () => {
diff --git a/tests/eslint-rules/integration-test-name.mjs b/tests/eslint-rules/integration-test-name.mjs
@@ -8,8 +8,10 @@
  */
 
 // Placeholder regex — update this to match your naming convention.
-// Current pattern: "<lowercase-kebab-case> - Integration Tests"
-const DESCRIBE_NAME_PATTERN = /^[a-z0-9]+(-[a-z0-9]+)* - Integration Tests$/;
+// Current pattern: 
+// - without suffix after skill name "<lowercase-kebab-case-skill-name>_ - Integration Tests"
+// - with suffix after skill name "<lowercase-kebab-case-skill-name>_<test-name-suffix> - Integration Tests"
+const DESCRIBE_NAME_PATTERN = /^[a-z0-9]+(-[a-z0-9]+)_([a-z0-9]+)* - Integration Tests$/;
 
 /** @type {import("eslint").Rule.RuleModule} */
 const rule = {
diff --git a/tests/eslint.config.mjs b/tests/eslint.config.mjs
@@ -120,9 +120,8 @@ export default tseslint.config(
         },
         rules: {
             // Update the pattern option below to change the required format.
-            // Default (in rule file): /^[a-z0-9]+(-[a-z0-9]+)* - Integration Tests$/
             "custom/integration-test-name": ["error", {
-                pattern: "^[a-z0-9]+(-[a-z0-9]+)* - Integration Tests$"
+                pattern: "^[a-z0-9]+(-[a-z0-9]+)*_(-[a-z0-9]+)* - Integration Tests$"
             }],
         },
     }
diff --git a/tests/microsoft-foundry/agent/create/agent-framework/integration.test.ts b/tests/microsoft-foundry/agent/create/agent-framework/integration.test.ts
@@ -37,7 +37,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-agent-framework - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-agent-framework - Integration Tests`, () => {
   const agent = useAgentRunner();
   describe("skill-invocation", () => {
     test("invokes skill for agent creation prompt", async () => {
diff --git a/tests/microsoft-foundry/integration.test.ts b/tests/microsoft-foundry/integration.test.ts
@@ -32,7 +32,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME} - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
   describe("skill-invocation", () => {
     test("invokes microsoft-foundry skill for AI model deployment prompt", async () => {
diff --git a/tests/microsoft-foundry/models/deploy/capacity/integration.test.ts b/tests/microsoft-foundry/models/deploy/capacity/integration.test.ts
@@ -30,7 +30,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-capacity - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-capacity - Integration Tests`, () => {
   const agent = useAgentRunner();
   describe("skill-invocation", () => {
     test("invokes skill for capacity discovery prompt", async () => {
diff --git a/tests/microsoft-foundry/models/deploy/customize-deployment/integration.test.ts b/tests/microsoft-foundry/models/deploy/customize-deployment/integration.test.ts
@@ -30,7 +30,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-customize-deployment - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-customize-deployment - Integration Tests`, () => {
   describe("skill-invocation", () => {
     const agent = useAgentRunner();
     test("invokes skill for custom deployment prompt", async () => {
diff --git a/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/integration.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model-optimal-region/integration.test.ts
@@ -30,7 +30,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-preset - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-preset - Integration Tests`, () => {
   describe("skill-invocation", () => {
     const agent = useAgentRunner();
     test("invokes skill for quick deployment prompt", async () => {
diff --git a/tests/microsoft-foundry/models/deploy/deploy-model/integration.test.ts b/tests/microsoft-foundry/models/deploy/deploy-model/integration.test.ts
@@ -30,7 +30,7 @@ if (skipTests && skipReason) {
 
 const describeIntegration = skipTests ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-deploy-model - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-deploy-model - Integration Tests`, () => {
   const agent = useAgentRunner();
   describe("skill-invocation", () => {
     test("invokes skill for simple model deployment prompt", async () => {
diff --git a/tests/microsoft-foundry/quota/integration.test.ts b/tests/microsoft-foundry/quota/integration.test.ts
@@ -24,7 +24,7 @@ const SKILL_NAME = "microsoft-foundry";
 // Use centralized skip logic from agent-runner
 const describeIntegration = shouldSkipIntegrationTests() ? describe.skip : describe;
 
-describeIntegration(`${SKILL_NAME}-quota - Integration Tests`, () => {
+describeIntegration(`${SKILL_NAME}_-quota - Integration Tests`, () => {
   const agent = useAgentRunner();
 
   describe("View Quota Usage", () => {
diff --git a/tests/microsoft-foundry/resource/create/integration.test.ts b/tests/microsoft-foundry/resource/create/integration.test.ts
@@ -8,7 +8,7 @@ import { loadSkill, type LoadedSkill } from "../../../utils/skill-loader";
 
 const SKILL_NAME = "microsoft-foundry";
 
-describe(`${SKILL_NAME}-resource-create - Integration Tests`, () => {
+describe(`${SKILL_NAME}_-resource-create - Integration Tests`, () => {
   let skill: LoadedSkill;
 
   beforeAll(async () => {
diff --git a/tests/scripts/generate-test-reports.ts b/tests/scripts/generate-test-reports.ts
@@ -60,9 +60,12 @@ function filterSubdirectoriesBySkill(subdirectories: string[], skill: string): s
   return subdirectories.filter(subdir => {
     const subdirName = path.basename(subdir);
 
-    // Directory names for test reports are guaranteed to start with the skill name for the tests.
+    // Skill name in the subdirectory name ends at the first underscore character.
     // See tests/eslint-rules/integration-test-name.mjs for details.
-    return subdirName.startsWith(skill);
+    const terminatorIndex = subdirName.indexOf("_");
+    const skillName = subdirName.substring(0, terminatorIndex);
+    
+    return skillName === skill;
   });
 }
 
diff --git a/tests/utils/agent-runner.ts b/tests/utils/agent-runner.ts
@@ -626,6 +626,5 @@ function sanitizeFileName(name: string): string {
     .replace(/\s+/g, "_")           // Replace spaces with underscores
     .replace(/-+/g, "-")            // Collapse multiple dashes
     .replace(/_+/g, "_")            // Collapse multiple underscores
-    .replace(/_-_/g, "-")
     .substring(0, 200);             // Limit length
 }
