Enhanced Quality Generation

notyashhh · notyashhh · commit acebef04bde3 · 2025-09-21T04:37:43.000+10:00
diff --git a/tools/CreateMappings_rules.json b/tools/CreateMappings_rules.json
@@ -979,5 +979,45 @@
   {
     "module": "DependencyMap",
     "alias": "DependencyMap"
+  },
+  {
+    "alias": "YashMaps",
+    "module": "YashMaps"
+  },
+  {
+    "alias": "YashMaps2",
+    "module": "YashMaps2"
+  },
+  {
+    "alias": "TestArizeAI",
+    "module": "TestArizeAI"
+  },
+  {
+    "alias": "TestData",
+    "module": "TestData"
+  },
+  {
+    "alias": "YashMaps3",
+    "module": "YashMaps3"
+  },
+  {
+    "alias": "YashMaps4",
+    "module": "YashMaps4"
+  },
+  {
+    "alias": "YashMaps5",
+    "module": "YashMaps5"
+  },
+  {
+    "alias": "YashMaps6",
+    "module": "YashMaps6"
+  },
+  {
+    "alias": "TestPinecone",
+    "module": "TestPinecone"
+  },
+  {
+    "module": "YashMaps7",
+    "alias": "YashMaps7"
   }
 ]
diff --git a/tools/Mcp/src/services/toolsService.ts b/tools/Mcp/src/services/toolsService.ts
@@ -161,14 +161,16 @@ export class ToolsService {
                 console.error(`Error eliciting input for example ${name}:`, error);
             }
         }
-        return [exampleSpecsPath, examplePath];
+        const idealExamplePaths = utils.getIdealModuleExamplePaths();
+        return [exampleSpecsPath, examplePath, idealExamplePaths];
     }
 
     createTestsFromSpecs = async <Args extends ZodRawShape>(args: Args): Promise<string[]> => {
         const workingDirectory = z.string().parse(Object.values(args)[0]);
         const testPath = path.join(workingDirectory, "test");
         const exampleSpecsPath = await utils.getExamplesFromSpecs(workingDirectory);
-        return [exampleSpecsPath, testPath];
+        const idealTestPaths = utils.getIdealModuleTestPaths();
+        return [exampleSpecsPath, testPath, idealTestPaths];
     }
 
     setupModuleStructure = async <Args extends ZodRawShape>(args: Args): Promise<string[]> => {
@@ -271,7 +273,7 @@ export class ToolsService {
             });
 
             const moduleNameResponse = await this._server!.elicitInput({
-                message: `Configuration resolved:\n- Service: ${selectedService}\n- Provider: ${selectedProvider}\n- Version: ${selectedVersion} (${selectedStability})\n- Service Name: ${resolved.serviceName}\n- Commit ID: ${resolved.commitId}\n- Service Specs: ${resolved.serviceSpecs}\n- Swagger File: ${resolved.swaggerFileSpecs}`,
+                message: `What would you like call the powershell module? \n\n Configuration resolved:\n- Service: ${selectedService}\n- Provider: ${selectedProvider}\n- Version: ${selectedVersion} (${selectedStability})\n- Service Name: ${resolved.serviceName}\n- Commit ID: ${resolved.commitId}\n- Service Specs: ${resolved.serviceSpecs}\n- Swagger File: ${resolved.swaggerFileSpecs}`,
                 requestedSchema: {
                     type: "object",
                     properties: {
diff --git a/tools/Mcp/src/services/utils.ts b/tools/Mcp/src/services/utils.ts
@@ -3,6 +3,7 @@ import yaml from "js-yaml";
 import { yamlContent } from '../types.js';
 import { execSync } from 'child_process';
 import path from 'path';
+import { Dirent } from 'fs';
 
 const GITHUB_API_BASE = 'https://api.github.com';
 const REST_API_SPECS_OWNER = 'Azure';
@@ -370,6 +371,50 @@ export async function writeFileIfNotExists(filePath: string, content: string): P
     }
 }
 
+export function getIdealModuleExamplePaths(): string {
+    const idealModulesRoot = path.join(process.cwd(), 'src', 'ideal-modules');
+    try {
+        if (!fs.existsSync(idealModulesRoot)) {
+            return '';
+        }
+        const modules: Dirent[] = fs.readdirSync(idealModulesRoot, { withFileTypes: true });
+        const exampleDirs: string[] = [];
+        for (const mod of modules) {
+            if (!mod.isDirectory()) continue;
+            const candidate = path.join(idealModulesRoot, mod.name, 'examples');
+            if (fs.existsSync(candidate)) {
+                exampleDirs.push(candidate);
+            }
+        }
+        return exampleDirs.join(';');
+    } catch (err) {
+        console.error('Error collecting ideal module example paths:', err);
+        return '';
+    }
+}
+
+export function getIdealModuleTestPaths(): string {
+    const idealModulesRoot = path.join(process.cwd(), 'src', 'ideal-modules');
+    try {
+        if (!fs.existsSync(idealModulesRoot)) {
+            return '';
+        }
+        const modules: Dirent[] = fs.readdirSync(idealModulesRoot, { withFileTypes: true });
+        const testDirs: string[] = [];
+        for (const mod of modules) {
+            if (!mod.isDirectory()) continue;
+            const candidate = path.join(idealModulesRoot, mod.name, 'tests');
+            if (fs.existsSync(candidate)) {
+                testDirs.push(candidate);
+            }
+        }
+        return testDirs.join(';');
+    } catch (err) {
+        console.error('Error collecting ideal module test paths:', err);
+        return '';
+    }
+}
+
 
 
 
diff --git a/tools/Mcp/src/specs/prompts/partner-module-workflow.md b/tools/Mcp/src/specs/prompts/partner-module-workflow.md
@@ -1,54 +1,69 @@
-# Execution rules 
-- Do not ask before running the command, just go ahead and run.
-
-# Role and Objective
-- You are an autonomous agent that generates Azure PowerShell modules using Autorest for partners.
-- Execute commands confidently without asking for confirmation.
-- Follow all steps carefully and halt if any MCP tool is missing or fails.
-- Think independently using your full context and reset if needed.
-- No need to provide status updates at each step.
-- FOLLOW ALL THE STEPS. DO NOT SKIP ANY STEPS. DO NOT MISS ANY STEPS.
-- If an mcp tool is not found or the tool fails then please halt execution.
-
-# Instructions
-
-## Stage 1: Interactive specification selection and autorest resolution
-- Call the MCP tool "setupModuleStructure" with no parameters
-- This tool will interactively guide you through:
-  1. Selecting the specification from available azure-rest-api-specs
-  2. Choosing the provider namespace
-  3. Selecting the API version (stable or preview)
-  4. Getting the module name from the user
-  5. Automatically creating the module structure and README.md file
-- The tool will create the folder structure under the correct src directory and generate the README.md with proper autorest configuration
-- Mark Stage 1 complete once the setupModuleStructure tool finishes successfully
-
-## Stage 2: Generating partner powershell module
-- FOLLOW ALL THE STEPS. DO NOT SKIP ANY STEPS.
-- Navigate to the created module directory, you can find this from the input given by the user using ellicitation for the input "moduleName" (should be under `src/<PowerShell module name>/<PowerShell module name>.Autorest`)
-- Use the "generate-autorest" mcp tool to generate the <PowerShell module name> module using the README.md that was created by setupModuleStructure
-- Stage 2 Complete.
-
-## Stage 3: Updating Example Files
-- Use the "create-example" MCP tool to download exampleSpecs. Use the output of this tool as a prompt input/task for you.
-- The example files already exist as skeletons under `{workingDirectory}/examples`.
-- Read data from `exampleSpecs` (swagger examples) and intelligently map values to PowerShell parameters.
-- Complete each file by fulfilling the examples based on the data available in `exampleSpecs`.
-- Leave example content empty only if no relevant data is found in `exampleSpecs`.
-- Once all example files are updated, mark stage 3 as complete.
-
-## Stage 4: Updating Test Files
-- Use the "create-test" MCP tool to download exampleSpecs. Use the output of this tool as a prompt input/task for you.
-- Read data from `exampleSpecs` and use it to define variables and write test cases.
-- Define setup variables inside `setupEnv` in `utils.ps1`, inferred from `exampleSpecs`.
-- Use those variables in the actual test case content.
-- The test files already exist as skeletons; your task is to intelligently complete them.
-- Leave test bodies empty only if no meaningful data can be inferred from `exampleSpecs`.
-- Once all test files are updated, mark stage 4 as complete.
-
-## Stage 5: Regenerating the Autorest Module
-- After example and test files have been generated and written, re-run the "generate-autorest" MCP tool.
-- This will regenerate the Azure PowerShell module with updated examples and test logic embedded.
-- Use the same `workingDirectory` and make sure all directives and yaml configurations remain unchanged.
-- This is a mandatory finalization step before pushing to GitHub.
-- Do not skip this regeneration even if the module was generated earlier.
+# Partner Module Workflow (Simplified)
+
+Goal: Generate an Azure PowerShell module via Autorest, then populate examples and tests deterministically with minimal ambiguity.
+
+Core Principle: Derive the working directory once and reuse it. Do not guess or fabricate paths.
+
+## 1. Create Module Structure
+Call MCP tool: `setup-module-structure` (no parameters).
+It returns `{0}` = `ModuleName` (from user input inside the tool).
+Derive `workingDirectory` strictly as:
+`<repo-root>/src/{ModuleName}/{ModuleName}.Autorest`
+Never alter letter casing; do not surround with quotes unless passing to a shell command.
+
+## 2. Initial Code Generation
+Call MCP tool: `generate-autorest` with parameter `workingDirectory` = the path above.
+Do not call Autorest directly; always use the MCP tool.
+If generation fails, STOP.
+
+## 3. Populate Examples
+Call MCP tool: `create-example` with the same `workingDirectory`.
+Tool returns:
+  `{0}` = specs example source dir (downloaded JSON)
+  `{1}` = target examples dir in module
+  `{2}` = reference ideal examples (semicolon separated; may be empty)
+Use reference examples (if any) for style: parameter ordering (required first), consistent variable placeholder patterns, and advanced options inclusion.
+Leave an example file empty only if no relevant swagger example data exists.
+Quality guidance (concise):
+  - Provide both minimal (required params only) and enriched (common optional params) variants if the skeleton expects multiple examples.
+  - Use stable placeholder naming: `<ResourceGroupName>`, `<Location>`, `<WorkspaceName>` etc., unless a deterministic value is required by an existing test.
+  - Do not invent unsupported parameters; omit deprecated or preview-only params unless the swagger version explicitly requires them.
+  - Maintain consistent casing and ordering: required first, then alphabetized optional.
+  - Prefer realistic value shapes (GUID formats, URIs) over lorem ipsum.
+MANDATORY: After the tool call, TREAT the returned response text itself (instructions referencing {0} {1} {2}) AS AN ACTION PLAN. Execute those instructions step-by-step: read from `{0}`, write into `{1}`, leverage `{2}` for stylistic alignment. Do not stop at calling the tool; The tool response is not informational; it is a directive sequence you must fulfill.
+
+## 4. Populate Tests
+Call MCP tool: `create-test` with the same `workingDirectory`.
+Tool returns:
+  `{0}` = specs example source dir
+  `{1}` = target test dir
+  `{2}` = reference ideal test dirs
+Fill test stubs using variables defined in `utils.ps1` `setupEnv` function. 
+IMPORTANT: You can completely ignore the generated Test files as they only have the skeleton. Instead, you can group these test files into groups of 1 particular resource & then write 1 new file combining these 3-4 test file in the CRUD format.
+Add: create -> get -> assert -> (optional cleanup). Add a negative test when feasible.
+Leave a test empty only if genuinely no meaningful data can be inferred.
+Quality guidance (concise):
+  - CRUD Pattern: Create (or Update) => Get => Assert essential fields => (List => Assert presence) => (Delete => Confirm absence) when resource lifecycle is supported.
+  - Assertions: Prefer specific property checks (Name, Id segment patterns, ProvisioningState) over broad object dumps.
+  - Negative Case: Intentionally supply one invalid parameter or missing required field; assert error category/message pattern.
+  - Reuse Variables: All names & IDs defined once in `setupEnv` to avoid duplication and enable future parameterization.
+  - Idempotency: Avoid destructive operations unless they clean up what this test created.
+MANDATORY: After the tool call, EXECUTE the returned test instructions: interpret response text referencing `{0}`, `{1}`, `{2}`, open each stub under `{1}`, populate bodies using data from `{0}`, and mirror style from `{2}`. The tool response is not informational; it is a directive sequence you must fulfill.
+
+## 5. Regenerate Module
+Call `generate-autorest` again with identical `workingDirectory` to ensure examples/tests are integrated. Do not modify the README.yaml block except via directives inserted earlier.
+
+## 6. Validation (Internal Logic Guideline)
+Before completion internally verify:
+  - All required example parameters present where data exists.
+  - No unknown parameters introduced.
+  - Tests assert at least one key property per created resource.
+If any check fails, refine the affected file(s) then proceed.
+
+## Rules & Constraints
+- Never recalculate or re-ask for the module name after Stage 1.
+- Never invent alternative directory paths.
+- Do not skip steps 1–5.
+- Halt immediately if an MCP tool is unavailable or errors.
+
+End of workflow.
diff --git a/tools/Mcp/src/specs/prompts/workflow-old.md b/tools/Mcp/src/specs/prompts/workflow-old.md
@@ -0,0 +1,54 @@
+# Execution rules 
+- Do not ask before running the command, just go ahead and run.
+
+# Role and Objective
+- You are an autonomous agent that generates Azure PowerShell modules using Autorest for partners.
+- Execute commands confidently without asking for confirmation.
+- Follow all steps carefully and halt if any MCP tool is missing or fails.
+- Think independently using your full context and reset if needed.
+- No need to provide status updates at each step.
+- FOLLOW ALL THE STEPS. DO NOT SKIP ANY STEPS. DO NOT MISS ANY STEPS.
+- If an mcp tool is not found or the tool fails then please halt execution.
+
+# Instructions
+
+## Stage 1: Interactive specification selection and autorest resolution
+- Call the MCP tool "setupModuleStructure" with no parameters
+- This tool will interactively guide you through:
+  1. Selecting the specification from available azure-rest-api-specs
+  2. Choosing the provider namespace
+  3. Selecting the API version (stable or preview)
+  4. Getting the module name from the user
+  5. Automatically creating the module structure and README.md file
+- The tool will create the folder structure under the correct src directory and generate the README.md with proper autorest configuration
+- Mark Stage 1 complete once the setupModuleStructure tool finishes successfully
+
+## Stage 2: Generating partner powershell module
+- FOLLOW ALL THE STEPS. DO NOT SKIP ANY STEPS.
+- Navigate to the created module directory, you can find this from the input given by the user using ellicitation for the input "moduleName" (should be under `src/<PowerShell module name>/<PowerShell module name>.Autorest`)
+- Use the "generate-autorest" mcp tool to generate the <PowerShell module name> module using the README.md that was created by setupModuleStructure
+- Stage 2 Complete.
+
+## Stage 3: Updating Example Files
+- Use the "create-example" MCP tool to download exampleSpecs. Use the output of this tool as a prompt input/task for you.
+- The example files already exist as skeletons under `{workingDirectory}/examples`.
+- Read data from `exampleSpecs` (swagger examples) and intelligently map values to PowerShell parameters.
+- Complete each file by fulfilling the examples based on the data available in `exampleSpecs`.
+- Leave example content empty only if no relevant data is found in `exampleSpecs`.
+- Once all example files are updated, mark stage 3 as complete.
+
+## Stage 4: Updating Test Files
+- Use the "create-test" MCP tool to download exampleSpecs. Use the output of this tool as a prompt input/task for you.
+- Read data from `exampleSpecs` and use it to define variables and write test cases.
+- Define setup variables inside `setupEnv` in `utils.ps1`, inferred from `exampleSpecs`.
+- Use those variables in the actual test case content.
+- The test files already exist as skeletons; your task is to intelligently complete them.
+- Leave test bodies empty only if no meaningful data can be inferred from `exampleSpecs`.
+- Once all test files are updated, mark stage 4 as complete.
+
+## Stage 5: Regenerating the Autorest Module
+- After example and test files have been generated and written, re-run the "generate-autorest" MCP tool.
+- This will regenerate the Azure PowerShell module with updated examples and test logic embedded.
+- Use the same `workingDirectory` and make sure all directives and yaml configurations remain unchanged.
+- This is a mandatory finalization step before pushing to GitHub.
+- Do not skip this regeneration even if the module was generated earlier.
diff --git a/tools/Mcp/src/specs/responses.json b/tools/Mcp/src/specs/responses.json
@@ -22,12 +22,12 @@
     {
         "name": "create-example",
         "type": "tool",
-        "text": "Read examples from specs under {0}. Fulfill examples under {1}. You are expert in Azure-PowerShell and Autorest.PowerShell. Leave example as empty if you don't find any matches. You know how to map data from {0} to {1}"
+        "text": "Read examples from specs under {0}. Fulfill examples under {1}. Also leverage high-quality reference examples located in directories: {2} (semicolon-separated). When generating, mirror parameter naming, structure, and advanced option usage patterns seen in those reference examples when applicable. Produce minimal yet complete runnable examples; omit unknown or deprecated params. Leave example empty only if no relevant mapping exists."
     },
     {
         "name": "create-test",
         "type": "tool",
-        "text": "Read examples from specs are under {0}. Implement empty test stubs under {1}. Test stubs are named as '.Test.ps1'. Define variables in function 'setupEnv' in 'utils.ps1' under {1}, and use these variables for test cases. Value of these variables are from {0}. Leave test cases as empty if you don't find any matches. You are expert in Azure-PowerShell and Autorest.PowerShell, You know how to map data from {0} to {1}. "
+        "text": "Read examples from specs under {0}. Implement test stubs under {1}. Reference high-quality existing tests from directories: {2} (semicolon-separated) to replicate assertion style, variable patterns, and setup/teardown conventions. Test stubs are named '.Test.ps1'. Populate 'setupEnv' in 'utils.ps1' with variables derived from {0} examples; reuse them across tests. For each CRUD operation: (1) Create/Update then Get and assert key properties, (2) List and validate presence, (3) Clean up if destructive. Add one negative test if feasible (invalid parameter) asserting specific error type/message. Leave a stub empty only if absolutely no relevant example data exists."
     },
     {
         "name": "setup-module-structure",

Original file line number	Diff line number	Diff line change
`@@ -161,14 +161,16 @@ export class ToolsService {`
`161`	`161`	console.error(`Error eliciting input for example ${name}:`, error);
`162`	`162`	`}`
`163`	`163`	`}`
`164`		`- return [exampleSpecsPath, examplePath];`
	`164`	`+ const idealExamplePaths = utils.getIdealModuleExamplePaths();`
	`165`	`+ return [exampleSpecsPath, examplePath, idealExamplePaths];`
`165`	`166`	`}`
`166`	`167`
`167`	`168`	`createTestsFromSpecs = async <Args extends ZodRawShape>(args: Args): Promise<string[]> => {`
`168`	`169`	`const workingDirectory = z.string().parse(Object.values(args)[0]);`
`169`	`170`	`const testPath = path.join(workingDirectory, "test");`
`170`	`171`	`const exampleSpecsPath = await utils.getExamplesFromSpecs(workingDirectory);`
`171`		`- return [exampleSpecsPath, testPath];`
	`172`	`+ const idealTestPaths = utils.getIdealModuleTestPaths();`
	`173`	`+ return [exampleSpecsPath, testPath, idealTestPaths];`
`172`	`174`	`}`
`173`	`175`
`174`	`176`	`setupModuleStructure = async <Args extends ZodRawShape>(args: Args): Promise<string[]> => {`
`@@ -271,7 +273,7 @@ export class ToolsService {`
`271`	`273`	`});`
`272`	`274`
`273`	`275`	`const moduleNameResponse = await this._server!.elicitInput({`
`274`		- message: `Configuration resolved:\n- Service: ${selectedService}\n- Provider: ${selectedProvider}\n- Version: ${selectedVersion} (${selectedStability})\n- Service Name: ${resolved.serviceName}\n- Commit ID: ${resolved.commitId}\n- Service Specs: ${resolved.serviceSpecs}\n- Swagger File: ${resolved.swaggerFileSpecs}`,
	`276`	+ message: `What would you like call the powershell module? \n\n Configuration resolved:\n- Service: ${selectedService}\n- Provider: ${selectedProvider}\n- Version: ${selectedVersion} (${selectedStability})\n- Service Name: ${resolved.serviceName}\n- Commit ID: ${resolved.commitId}\n- Service Specs: ${resolved.serviceSpecs}\n- Swagger File: ${resolved.swaggerFileSpecs}`,
`275`	`277`	`requestedSchema: {`
`276`	`278`	`type: "object",`
`277`	`279`	`properties: {`
Original file line number	Diff line number	Diff line change
`@@ -22,12 +22,12 @@`
`22`	`22`	`{`
`23`	`23`	`"name": "create-example",`
`24`	`24`	`"type": "tool",`
`25`		`- "text": "Read examples from specs under {0}. Fulfill examples under {1}. You are expert in Azure-PowerShell and Autorest.PowerShell. Leave example as empty if you don't find any matches. You know how to map data from {0} to {1}"`
	`25`	`+ "text": "Read examples from specs under {0}. Fulfill examples under {1}. Also leverage high-quality reference examples located in directories: {2} (semicolon-separated). When generating, mirror parameter naming, structure, and advanced option usage patterns seen in those reference examples when applicable. Produce minimal yet complete runnable examples; omit unknown or deprecated params. Leave example empty only if no relevant mapping exists."`
`26`	`26`	`},`
`27`	`27`	`{`
`28`	`28`	`"name": "create-test",`
`29`	`29`	`"type": "tool",`
`30`		`- "text": "Read examples from specs are under {0}. Implement empty test stubs under {1}. Test stubs are named as '.Test.ps1'. Define variables in function 'setupEnv' in 'utils.ps1' under {1}, and use these variables for test cases. Value of these variables are from {0}. Leave test cases as empty if you don't find any matches. You are expert in Azure-PowerShell and Autorest.PowerShell, You know how to map data from {0} to {1}. "`
	`30`	+ "text": "Read examples from specs under {0}. Implement test stubs under {1}. Reference high-quality existing tests from directories: {2} (semicolon-separated) to replicate assertion style, variable patterns, and setup/teardown conventions. Test stubs are named '.Test.ps1'. Populate 'setupEnv' in 'utils.ps1' with variables derived from {0} examples; reuse them across tests. For each CRUD operation: (1) Create/Update then Get and assert key properties, (2) List and validate presence, (3) Clean up if destructive. Add one negative test if feasible (invalid parameter) asserting specific error type/message. Leave a stub empty only if absolutely no relevant example data exists."
`31`	`31`	`},`
`32`	`32`	`{`
`33`	`33`	`"name": "setup-module-structure",`