Skip to content
Merged
33 changes: 21 additions & 12 deletions src/Cody.VisualStudio.Tests/SmartApplyTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,57 +38,66 @@ public SmartApplyTests(ITestOutputHelper output) : base(output)
}

[VsFact(Version = VsVersion.VS2022)]
public async Task Apply_Suggestion_Is_Modifying_Point_Document()
public async Task Apply_Suggestion_Is_Modifying_Dummy_Document()
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn’t expecting this to be the fix, nor was it the outcome I aimed for, but it does solve the issue.

{
// given
// TODO: This is a workaround. Without this test, the first SmartApply test fails
// because file context chips don't appear in the chat input, causing the LLM to
// respond without code suggestions. Running this dummy test first somehow fixes it.
// The root cause needs investigation - likely a timing or initialization issue
// in how file context is added to chat on first document open.
await NewChat();
await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Point.cs"));
}

[VsFact(Version = VsVersion.VS2022)]
public async Task Apply_Suggestion_Is_Modifying_Point_Document()
{
await NewChat();
await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Point.cs"));

var originalText = await GetActiveDocumentText();

// when
await ApplyLastSuggestionFor("Suggest improvements");

var modifiedText = await GetActiveDocumentText();

// then
Assert.NotEqual(modifiedText, originalText);
}

[VsFact(Version = VsVersion.VS2022, Skip = "Unstable")]
[VsFact(Version = VsVersion.VS2022)]
public async Task Apply_Suggestion_Is_Modifying_Manager_Document()
{
// given
await NewChat();

await OpenSolution(SolutionsPaths.GetConsoleApp1File("ConsoleApp1.sln"));
await OpenDocument(SolutionsPaths.GetConsoleApp1File(@"ConsoleApp1\Manager.cs"));

var originalText = await GetActiveDocumentText();

// when
await ApplyLastSuggestionFor("Suggest improvements in Print() method");

var modifiedText = await GetActiveDocumentText();

// then
Assert.NotEqual(modifiedText, originalText);
}

private async Task ApplyLastSuggestionFor(string chatText)
{
var contextChipLocator = Page.Locator("[aria-label='Chat message'] span[data-lexical-decorator='true']");
await contextChipLocator.First.WaitForAsync(new() { Timeout = 5000 });

await EnterChatTextAndSend(chatText);

var apply = Page.Locator("span", new() { HasText = "Apply" }).Last;

// checking if Chat window is too narrow to show "Apply" text
await apply.WaitForAsync(new() { Timeout = 60000, State = WaitForSelectorState.Attached });

var hasHiddenClass = await apply.EvaluateAsync<bool>(@"element => element.classList.contains('tw-hidden')");
if (hasHiddenClass)
await apply.EvaluateAsync("element => element.classList.remove('tw-hidden')"); // force shows "Apply" text so it will be possible to click on it
await apply.EvaluateAsync("element => element.classList.remove('tw-hidden')");

await apply.ClickAsync(new() { Force = true });
await apply.ClickAsync(new() { Force = true});

await EditAppliedAsync();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ public Manager2() { }

public void Print()
{
Console.WriteLine("Hello, World!");
var mesage = "Hello, World!";
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The fix is to do something more obviously incorrect here. Before, sometimes Cody suggested no changes (no code snippets). Now we do an explicit typo.

Still not ideal - ideally, we have recordings and the test runs are deterministic.

Console.WriteLine(mesage);


Console.WriteLine("Bye, World!");
Expand Down