Refactor ContextFilter tests for determinism and logging

Updated ContextFilter-related tests to use -WhatIf mode and PSFramework message inspection for deterministic validation, replacing API-dependent checks. Added verification of deduplication and warning messages, and improved test clarity by focusing on internal logic rather than external tool responses.

Author: potatoqualitee

Tests/aitools.Tests.ps1

@@ -206,6 +206,8 @@ function Get-TestData {
     Context 'ContextFilter Parameter' {
         BeforeAll {
             $script:recipesPath = Join-Path $PSScriptRoot 'recipes'
+            # Set PSFramework message level to capture debug messages
+            Set-PSFConfig -FullName 'PSFramework.Message.Info.Maximum' -Value 9
         }
 
         It 'Should have test recipe files available' {
@@ -214,64 +216,69 @@ function Get-TestData {
             Test-Path (Join-Path $script:recipesPath 'alligator-eggs.fr-ca.md') | Should -Be $true
         }
 
-        It 'Should include English original as dynamic context and Claude should see it' {
+        It 'Should resolve context file from ContextFilter (deterministic)' {
             $frFile = Join-Path $script:recipesPath 'alligator-eggs.fr.md'
-            # Ask Claude to list the context files it can see - the English original should be included
-            $result = Invoke-AITool -Path $frFile -Prompt "List ALL the filenames you can see in this prompt, including any context files. Just list the filenames, one per line." -Tool Claude -ContextFilter { $_ -replace '\.fr\.md$', '.md' }
-
-            $result | Should -Not -BeNullOrEmpty
-            $result.Success | Should -Be $true
-            $result.Tool | Should -Be 'Claude'
-            # Claude should mention the English original file in its response
-            $result.Result | Should -Match 'alligator-eggs\.md'
+            # Clear previous messages and run with -WhatIf to avoid API call
+            Get-PSFMessage | Out-Null
+            $null = Invoke-AITool -Path $frFile -Prompt "test" -Tool Claude -ContextFilter { $_ -replace '\.fr\.md$', '.md' } -WhatIf
+
+            # Check debug messages to verify context file was found
+            $messages = Get-PSFMessage -Last 50 | Where-Object Message -Match 'ContextFilter'
+            $foundMessage = $messages | Where-Object Message -Match 'Found at:.*alligator-eggs\.md'
+            $foundMessage | Should -Not -BeNullOrEmpty -Because "ContextFilter should find alligator-eggs.md"
         }
 
-        It 'Should use ContextFilterBase to find context files in different directory' {
+        It 'Should resolve context file using ContextFilterBase (deterministic)' {
             $frCaFile = Join-Path $script:recipesPath 'alligator-eggs.fr-ca.md'
-            # Ask Claude what files it sees - should include the English original from ContextFilterBase
-            $result = Invoke-AITool -Path $frCaFile -Prompt "What filenames can you see? List them all." -Tool Claude -ContextFilter { [System.IO.Path]::GetFileName($_) -replace '\.fr-ca\.md$', '.md' } -ContextFilterBase $script:recipesPath
-
-            $result | Should -Not -BeNullOrEmpty
-            $result.Success | Should -Be $true
-            # Should see the English original
-            $result.Result | Should -Match 'alligator-eggs\.md'
+            Get-PSFMessage | Out-Null
+            $null = Invoke-AITool -Path $frCaFile -Prompt "test" -Tool Claude `
+                -ContextFilter { [System.IO.Path]::GetFileName($_) -replace '\.fr-ca\.md$', '.md' } `
+                -ContextFilterBase $script:recipesPath -WhatIf
+
+            # Verify it searched in ContextFilterBase and found the file
+            $messages = Get-PSFMessage -Last 50 | Where-Object Message -Match 'ContextFilter'
+            $foundMessage = $messages | Where-Object Message -Match 'Found at:.*alligator-eggs\.md'
+            $foundMessage | Should -Not -BeNullOrEmpty -Because "ContextFilter should find alligator-eggs.md via ContextFilterBase"
         }
 
-        It 'Should handle batch processing with ContextFilter and see context files' {
+        It 'Should deduplicate context files in batch mode (deterministic)' {
+            # Both .fr.md and .fr-ca.md derive the same .md file
             $frFiles = Get-ChildItem -Path $script:recipesPath -Filter '*.fr*.md'
-            # Ask Claude what context files it can see in the batch
-            $result = $frFiles | Invoke-AITool -Prompt "List ALL filenames you can see. Include both the files to process and any context files." -Tool Claude -ContextFilter { $_ -replace '\.fr(-ca)?\.md$', '.md' } -BatchSize 3
-
-            $result | Should -Not -BeNullOrEmpty
-            $result.Success | Should -Be $true
-            # Should see the English original as dynamic context
-            $result.Result | Should -Match 'alligator-eggs\.md'
+            Get-PSFMessage | Out-Null
+            $null = $frFiles | Invoke-AITool -Prompt "test" -Tool Claude `
+                -ContextFilter { $_ -replace '\.fr(-ca)?\.md$', '.md' } -BatchSize 3 -WhatIf
+
+            # Check that deduplication message appears (second file should be skipped)
+            $messages = Get-PSFMessage -Last 50 | Where-Object Message -Match 'ContextFilter'
+            $skipMessage = $messages | Where-Object Message -Match 'Skipping duplicate:.*alligator-eggs\.md'
+            $skipMessage | Should -Not -BeNullOrEmpty -Because "Second derived file should be skipped as duplicate"
         }
 
-        It 'Should warn but not fail when ContextFilter derived file does not exist' {
+        It 'Should warn when ContextFilter derived file does not exist' {
             $frFile = Join-Path $script:recipesPath 'alligator-eggs.fr.md'
-            # This filter will derive a non-existent file
-            $result = Invoke-AITool -Path $frFile -Prompt "Say hello" -Tool Claude -ContextFilter { $_ -replace '\.fr\.md$', '.nonexistent.md' } -WarningVariable warnings
+            Get-PSFMessage | Out-Null
+            $null = Invoke-AITool -Path $frFile -Prompt "test" -Tool Claude `
+                -ContextFilter { $_ -replace '\.fr\.md$', '.nonexistent.md' } -WhatIf -WarningVariable warnings
 
-            $result | Should -Not -BeNullOrEmpty
-            $result.Success | Should -Be $true
             # Should have warned about missing file
-            $warnings | Should -Not -BeNullOrEmpty
+            $messages = Get-PSFMessage -Last 50 | Where-Object Level -eq 'Warning'
+            $warningMessage = $messages | Where-Object Message -Match 'ContextFilter derived file not found.*nonexistent'
+            $warningMessage | Should -Not -BeNullOrEmpty -Because "Should warn about missing derived file"
         }
 
-        It 'Should deduplicate when multiple inputs derive same context file' {
-            # Both .fr.md and .fr-ca.md would derive the same .md file
-            $frFiles = Get-ChildItem -Path $script:recipesPath -Filter '*.fr*.md'
-            # With deduplication, the English original should only appear once even though two files derive it
-            $result = $frFiles | Invoke-AITool -Prompt "How many times does 'alligator-eggs.md' appear as a context file? Answer with just a number." -Tool Claude -ContextFilter { $_ -replace '\.fr(-ca)?\.md$', '.md' } -BatchSize 3
+        It 'Should work end-to-end with ContextFilter (integration)' {
+            $frFile = Join-Path $script:recipesPath 'alligator-eggs.fr.md'
+            # One real API call to verify the whole flow works
+            $result = Invoke-AITool -Path $frFile -Prompt "List ALL the filenames visible in this prompt. Output ONLY the filenames, one per line, nothing else." -Tool Claude -ContextFilter { $_ -replace '\.fr\.md$', '.md' }
 
             $result | Should -Not -BeNullOrEmpty
             $result.Success | Should -Be $true
-            # Should only see the context file once due to deduplication
-            $result.Result | Should -Match '1'
+            $result.Tool | Should -Be 'Claude'
+            # Claude should see and mention both files
+            $result.Result | Should -Match 'alligator-eggs'
         }
 
-        It 'Should work with null ContextFilter (no-op, existing behavior unchanged)' {
+        It 'Should work with null ContextFilter (no-op)' {
             $frFile = Join-Path $script:recipesPath 'alligator-eggs.fr.md'
             $result = Invoke-AITool -Path $frFile -Prompt "Say hello" -Tool Claude