ada conversion in progress

Author: naman-msft

tools/abc.py

@@ -1,75 +1,87 @@
-from pathlib import Path
-import sys
 import os
-import shutil
 import re
+from pathlib import Path
+# filepath: [abc.py](http://_vscodecontentref_/1)
+def count_code_blocks(file_path):
+    """Count the number of code blocks (```) in a markdown file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        # Count opening triple backticks
+        # This regex matches ``` with optional leading whitespace
+        code_blocks = re.findall(r'^\s*```', content, re.MULTILINE)
+        return len(code_blocks)
+    except Exception as e:
+        print(f"Error reading {file_path}: {e}")
+        return -1
 
-def check_and_rename_folders():
-    # Get the directory where the script is located
-    script_dir = Path(sys.path[0])
+def find_markdown_files(root_dir):
+    """Find all markdown files in the directory tree."""
+    markdown_files = []
 
-    # Define the untested folder path
-    untested_dir = script_dir / "untested"
+    for root, dirs, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith('.md'):
+                file_path = os.path.join(root, file)
+                markdown_files.append(file_path)
 
-    if not untested_dir.exists():
-        print(f"Error: Untested directory not found at {untested_dir}")
+    return markdown_files
+
+def main():
+    # Get the untested folder path
+    untested_dir = "untested"
+    
+    if not os.path.exists(untested_dir):
+        print(f"Error: '{untested_dir}' folder not found!")
         return
-        
-    print(f"Checking folders in {untested_dir}")
 
-    # Get all subfolders
-    subfolders = [f for f in untested_dir.iterdir() if f.is_dir()]
+    # Find all markdown files
+    print(f"Scanning for markdown files in '{untested_dir}'...")
+    markdown_files = find_markdown_files(untested_dir)
 
-    if not subfolders:
-        print("No subfolders found in the untested directory.")
+    if not markdown_files:
+        print("No markdown files found!")
         return
 
-    print(f"Found {len(subfolders)} subfolders.")
+    print(f"Found {len(markdown_files)} markdown files\n")
 
-    # Process each subfolder
-    for folder in subfolders:
-        original_name = folder.name
-        
-        # Check if folder is empty
-        files = list(folder.iterdir())
-        if not files:
-            print(f"⚠️ Empty folder detected: {original_name}")
-            continue
-            
-        # Remove the "..." from folder name if present
-        if original_name.endswith("..."):
-            new_name = original_name[:-3]  # Remove the trailing "..."
-            new_folder_path = folder.parent / new_name
-            
-            try:
-                # Create temporary folder name to avoid potential conflicts
-                temp_folder_path = folder.parent / f"temp_{original_name}"
-                folder.rename(temp_folder_path)
-                temp_folder_path.rename(new_folder_path)
-                print(f"✓ Renamed folder: {original_name} → {new_name}")
-                # Update the folder reference to the new path
-                folder = new_folder_path
-            except Exception as e:
-                print(f"✗ Error renaming folder {original_name}: {str(e)}")
-                continue
+    # Count code blocks in each file
+    file_stats = []
+    for file_path in markdown_files:
+        count = count_code_blocks(file_path)
+        if count >= 0:  # Only include files that were successfully read
+            # Get relative path for cleaner display
+            relative_path = os.path.relpath(file_path, untested_dir)
+            file_stats.append((relative_path, count))
+    
+    # Sort by code block count (ascending)
+    file_stats.sort(key=lambda x: x[1])
+    
+    # Display all files ranked by code blocks
+    print("All markdown files ranked by number of code blocks (lowest to highest):")
+    print("-" * 100)
+    print(f"{'Rank':<6} {'Code Blocks':<12} {'File Path'}")
+    print("-" * 100)
+    
+    for i, (file_path, count) in enumerate(file_stats, 1):
+        print(f"{i:<6} {count:<12} {file_path}")
+    
+    # Show some statistics
+    print("\n" + "-" * 100)
+    print("Statistics:")
+    print(f"- Total files analyzed: {len(file_stats)}")
+    if file_stats:
+        print(f"- Files with 0 code blocks: {sum(1 for _, count in file_stats if count == 0)}")
+        print(f"- Average code blocks per file: {sum(count for _, count in file_stats) / len(file_stats):.2f}")
+        print(f"- Maximum code blocks in a file: {max(count for _, count in file_stats)}")
 
-        # Rename files within the folder to match folder name
-        for file_path in folder.iterdir():
-            if file_path.is_file():
-                file_extension = file_path.suffix
-                new_file_name = f"{folder.name}{file_extension}"
-                new_file_path = folder / new_file_name
-                
-                # Skip if the filename already matches
-                if file_path.name == new_file_name:
-                    continue
-                    
-                try:
-                    file_path.rename(new_file_path)
-                    print(f"  ✓ Renamed file: {file_path.name} → {new_file_name}")
-                except Exception as e:
-                    print(f"  ✗ Error renaming file {file_path.name}: {str(e)}")
+        # Group by code block count
+        print("\nDistribution:")
+        from collections import Counter
+        count_distribution = Counter(count for _, count in file_stats)
+        for blocks, num_files in sorted(count_distribution.items()):
+            print(f"  {blocks} code blocks: {num_files} file(s)")
 
 if __name__ == "__main__":
-    check_and_rename_folders()
-    print("Processing complete!")
+    main()

tools/ada.py

@@ -186,14 +186,14 @@
     **Example:** 
 
     ```bash  
-    export RANDOM_SUFFIX=$(openssl rand -hex 3)
+    export RANDOM_SUFFIX=$(head -c 3 /dev/urandom | xxd -p)
     export REGION="eastus"
     az group create --name "MyResourceGroup$RANDOM_SUFFIX" --location $REGION
     ```
 
     >**Note:** Add a random suffix to relevant variables that are likely to be unique for each deployment, such as resource group names, VM names, and other resources that need to be uniquely identifiable. However, do not add a random suffix to variables that are constant or environment-specific, such as region, username, or configuration settings that do not change between deployments. 
     
-    >**Note:** You can generate your own random suffix or use the one provided in the example above. The `openssl rand -hex 3` command generates a random 3-character hexadecimal string. This string is then appended to the resource group name to ensure that the resource group name is unique for each deployment.
+    >**Note:** You can generate your own random suffix or use the one provided in the example above. The `head -c 3 /dev/urandom | xxd -p` command generates a random 3-character hexadecimal string. This string is then appended to the resource group name to ensure that the resource group name is unique for each deployment.
 
 14. In Exec Docs, result blocks are distinguished by a custom expected_similarity comment tag followed by a code block. These result blocks indicate to Innovation Engine what the minimum degree of similarity should be between the actual and the expected output of a code block (one which returns something in the terminal that is relevant to benchmark against). Learn More: [Result Blocks](https://github.com/Azure/InnovationEngine/blob/main/README.md#result-blocks). 
 
@@ -227,7 +227,6 @@
                 "type": "Microsoft.Resources/resourceGroups"
             }}
             ```
-        ```
     - If you run into an error while executing a code block or the code block is running in an infinite loop, update the Exec Doc based on the error stack trace, restart/clear Cloudshell, and rerun the command block(s) from the start until you reach that command block. This is done to override any potential issues that may have occurred during the initial run. More guidance is given in the [FAQ section](#frequently-asked-questions-faqs) below.
     
     >**Note:** The expected similarity value is a percentage of similarity between 0 and 1 which specifies how closely the true output needs to match the template output given in the results block - 0 being no similarity, 1 being an exact match. If you are uncertain about the value, it is recommended to set the expected similarity to 0.3 i.e. 30% expected similarity to account for small variations. Once you have run the command multiple times and are confident that the output is consistent, you can adjust the expected similarity value accordingly.
@@ -273,7 +272,7 @@
 17. If the original document lists a prerequisite resource (such as an AKS cluster, VM, storage account, etc.), you MUST NOT add any new commands to create that resource in the Exec Doc.
 
     - **Example:** If the doc says "This article assumes you have an existing AKS cluster," do NOT add `az aks create` or any equivalent cluster creation commands. Only include steps for interacting with or managing the existing resource.
-    - This rule applies to any resource type, not just AKS. Always respect explicit prerequisites and never override them by adding creation steps.
+    - This rule applies to any resource type, not just AKS. Always respect explicit prerequisites and never override them by adding creation steps for that resource.
     - If the prerequisite is stated in any form (e.g., "Before you begin, create a resource group"), treat that resource as pre-existing and do not add creation commands for it.
     - If you are unsure whether a resource should be created, always preserve the prerequisite as stated and avoid introducing creation commands for that resource.
 
@@ -1830,7 +1829,7 @@ def main():
             input_type = 'file'
             with open(user_input, "r") as f:
                 input_content = f.read()
-                input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS. CRITICALLY IMPORTANT: NEVER CHANGE THE LANGUAGE TYPE OF CODE BLOCKS (e.g., from 'shell' to 'bash'). KEEP THE EXACT SAME LANGUAGE IDENTIFIER AFTER TRIPLE BACKTICKS AS IN THE ORIGINAL DOCUMENT:\n\n{input_content}"
+                input_content = f"CONVERT THE FOLLOWING EXISTING DOCUMENT INTO AN EXEC DOC. THIS IS A CONVERSION TASK, NOT CREATION FROM SCRATCH. DON'T EXPLAIN WHAT YOU ARE DOING BEHIND THE SCENES INSIDE THE DOC. PRESERVE ALL ORIGINAL CONTENT, STRUCTURE, AND NARRATIVE OUTSIDE OF CODE BLOCKS. CRITICALLY IMPORTANT: NEVER CHANGE THE LANGUAGE TYPE OF CODE BLOCKS. KEEP THE EXACT SAME LANGUAGE IDENTIFIER AFTER TRIPLE BACKTICKS AS IN THE ORIGINAL DOCUMENT:\n\n{input_content}"
             # We'll generate dependency files later in the process
             dependency_files = []
             generate_deps = input("\nMake new files referenced in the doc for its execution? (y/n): ").lower() == 'y'
@@ -2129,6 +2128,7 @@ def main():
         success = False
         dependency_files_generated = False
         additional_instruction = ""
+        user_edited_content = None  # Add this line to initialize the flag
 
         while attempt <= max_attempts:
             iteration_start_time = time.time()
@@ -2150,6 +2150,7 @@ def main():
                     f"or export MY_LOCATION=\\\"eastus2\\\"). The primary goal is to use the user's *names*."
                 )
 
+
             if attempt == 1:
                 print_header(f"Attempt {attempt}: Generating Exec Doc", "-")
 
@@ -2178,42 +2179,50 @@ def main():
             else:
                 print_header(f"Attempt {attempt}: Fixing Exec Doc", "-")
 
-                # Analyze if the error is in the main doc or in dependency files
-                error_analysis = analyze_error(errors_text, dependency_files)
-                
-                if error_analysis["type"] == "dependency_file" and error_analysis["file"]:
-                    # If error is in a dependency file, try to fix it
-                    dep_file = error_analysis["file"]
-                    print_message(f"\nDetected issue in dependency file: {dep_file['filename']}")
-                    update_dependency_file(dep_file, error_analysis["message"], output_file)
-                    made_dependency_change = True  # Set the flag
-                else:
-                    # If error is in main doc or unknown, update the main doc
-                    user_prompt_for_fix = (
-                        f"The following error(s) have occurred during testing:\n{errors_text}\n{additional_instruction}\n\n"
-                        f"Please carefully analyze these errors and make necessary corrections to the document to prevent them "
-                        f"from happening again. Try to find different solutions if the same errors keep occurring. \n"
-                        f"IMPORTANT: NEVER change the code block language types (e.g., do not change 'shell' to 'bash'). "
-                        f"Keep the exact same language identifier after triple backticks as in the current document."
-                        f"{llm_variable_instruction}" # Add variable instruction here as well
-                        f"\nGiven that context, please think hard and don't hurry. I want you to correct the converted document "
-                        f"in ALL instances where this error has been or can be found. Then, correct ALL other errors apart "
-                        f"from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"
-                    )
-                    response = client.chat.completions.create(
-                        model=deployment_name,
-                        messages=[
-                            {"role": "system", "content": system_prompt},
-                            {"role": "user", "content": input_content},
-                            {"role": "assistant", "content": output_file_content},
-                            {"role": "user", "content": user_prompt_for_fix}
-                        ]
-                    )
-                    output_file_content = response.choices[0].message.content
-
+                # Check if this is a retry after user feedback with document edits only
+                if attempt > 1 and 'user_edited_content' in locals() and user_edited_content:
+                    print_message("\nUsing your directly edited version without AI modifications...")
+                    output_file_content = user_edited_content
                     with open(output_file, "w") as f:
                         f.write(output_file_content)
-                        
+                    # Clear the flag
+                    user_edited_content = None
+                else:
+                    # Analyze if the error is in the main doc or in dependency files
+                    error_analysis = analyze_error(errors_text, dependency_files)
+                    
+                    if error_analysis["type"] == "dependency_file" and error_analysis["file"]:
+                        # If error is in a dependency file, try to fix it
+                        dep_file = error_analysis["file"]
+                        print_message(f"\nDetected issue in dependency file: {dep_file['filename']}")
+                        update_dependency_file(dep_file, error_analysis["message"], output_file)
+                        made_dependency_change = True  # Set the flag
+                    else:
+                        # If error is in main doc or unknown, update the main doc
+                        user_prompt_for_fix = (
+                            f"The following error(s) have occurred during testing:\n{errors_text}\n{additional_instruction}\n\n"
+                            f"Please carefully analyze these errors and make necessary corrections to the document to prevent them "
+                            f"from happening again. Try to find different solutions if the same errors keep occurring. \n"
+                            f"IMPORTANT: NEVER change the code block language types "
+                            f"Keep the exact same language identifier after triple backticks as in the current document."
+                            f"{llm_variable_instruction}" # Add variable instruction here as well
+                            f"\nGiven that context, please think hard and don't hurry. I want you to correct the converted document "
+                            f"in ALL instances where this error has been or can be found. Then, correct ALL other errors apart "
+                            f"from this that you see in the doc. ONLY GIVE THE UPDATED DOC, NOTHING ELSE"
+                        )
+                        response = client.chat.completions.create(
+                            model=deployment_name,
+                            messages=[
+                                {"role": "system", "content": system_prompt},
+                                {"role": "user", "content": input_content},
+                                {"role": "assistant", "content": output_file_content},
+                                {"role": "user", "content": user_prompt_for_fix}
+                            ]
+                        )
+                        output_file_content = response.choices[0].message.content
+
+                        with open(output_file, "w") as f:
+                            f.write(output_file_content)    
                     # Check if we need to regenerate dependency files after updating main doc
                     if generate_deps and dependency_files_generated:
                         # Regenerate dependency files if major changes were made to the main doc
@@ -2235,9 +2244,9 @@ def main():
                 var_names = extract_aks_env_vars(output_file)
                 ie_cmd = [
                     "ie", "execute", output_file,
-                    "--var", f"{var_names['resource_group']}=aks-tc-rged8996",
-                    "--var", f"{var_names['cluster_name']}=aks-tc-clustered8996",
-                    "--var", f"{var_names['region']}=canadacentral"
+                    "--var", f"{var_names['resource_group']}=aks-rg",
+                    "--var", f"{var_names['cluster_name']}=aks-cluster",
+                    "--var", f"{var_names['region']}=eastus2"
                 ]
             else:
                 print_header(f"Running Innovation Engine tests", "-")
@@ -2268,8 +2277,8 @@ def main():
                     iteration_file, 
                     attempt, 
                     "", # No errors in successful run
-                    iteration_start_time, 
-                    True
+                    start_time,
+                    True  # Assume success
                 )
                 all_iterations_data.append(iteration_data)
 
@@ -2368,7 +2377,7 @@ def main():
                     iteration_file, 
                     attempt, 
                     iteration_errors_text,  # Only errors from this iteration
-                    iteration_start_time, 
+                    start_time, 
                     False
                 )
                 all_iterations_data.append(iteration_data)
@@ -2442,6 +2451,9 @@ def main():
                             # Only document edits - no need to call LLM again
                             revised_content = feedback["doc_edit"]
 
+                            # Set a flag for the next iteration to use this content directly
+                            user_edited_content = revised_content
+                            
                             # Just use the user's edited version directly
                             output_file_content = revised_content
 
@@ -2513,5 +2525,5 @@ def main():
 
         print_message(f"\nThe updated file is stored at: {output_file}\n")
 
-# if __name__ == "__main__":
-#     main()
+if __name__ == "__main__":
+    main()