Skip to content

Commit 358b0f3

Browse files
TaoChenOSUcrickman
TaoChenOSU
and
crickman
authored
Python: .Net: Magentic orchestration to return the last agent message when limits reached (#12839)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Related to #12687 Related to #12656 Currently, the Magentic orchestration returns a static message if the max reset limit or the max invocation limit is reached. ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Return the last agent response if either of the limit is reached. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Chris <[email protected]>
1 parent c16db27 commit 358b0f3

File tree

3 files changed

+256
-3
lines changed

3 files changed

+256
-3
lines changed

‎dotnet/src/Agents/Magentic/MagenticManagerActor.cs

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,16 @@ private async ValueTask ManageAsync(CancellationToken cancellationToken)
141141

142142
if (this._invocationCount >= this._manager.MaximumInvocationCount)
143143
{
144-
await this.PublishMessageAsync("Maximum number of invocations reached.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
144+
this.Logger.LogMagenticManagerTaskFailed(this.Context.Topic);
145+
try
146+
{
147+
var partialResult = this._chat.Last((message) => message.Role == AuthorRole.Assistant);
148+
await this.PublishMessageAsync(partialResult.AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
149+
}
150+
catch (InvalidOperationException)
151+
{
152+
await this.PublishMessageAsync("I've reaches the maximum number of invocations. No partial result available.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
153+
}
145154
break;
146155
}
147156

@@ -157,7 +166,15 @@ private async ValueTask ManageAsync(CancellationToken cancellationToken)
157166
if (this._retryCount >= this._manager.MaximumResetCount)
158167
{
159168
this.Logger.LogMagenticManagerTaskFailed(this.Context.Topic);
160-
await this.PublishMessageAsync("I've experienced multiple failures and am unable to continue.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
169+
try
170+
{
171+
var partialResult = this._chat.Last((message) => message.Role == AuthorRole.Assistant);
172+
await this.PublishMessageAsync(partialResult.AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
173+
}
174+
catch (InvalidOperationException)
175+
{
176+
await this.PublishMessageAsync("I've experienced multiple failures and am unable to continue. No partial result available.".AsResultMessage(), this._orchestrationType, cancellationToken).ConfigureAwait(false);
177+
}
161178
break;
162179
}
163180

‎dotnet/src/Agents/UnitTests/Magentic/MagenticOrchestrationTests.cs

Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,216 @@ public async Task MagenticOrchestrationWithMultipleAgentsAsync()
5555
Assert.Equal(0, mockAgent3.InvokeCount);
5656
}
5757

58+
[Fact]
59+
public async Task MagenticOrchestrationMaxInvocationCountReached_WithoutPartialResultAsync()
60+
{
61+
// Arrange
62+
await using InProcessRuntime runtime = new();
63+
64+
MockAgent mockAgent1 = CreateMockAgent(1, "abc");
65+
MockAgent mockAgent2 = CreateMockAgent(2, "xyz");
66+
MockAgent mockAgent3 = CreateMockAgent(3, "lmn");
67+
68+
string jsonStatus =
69+
$$"""
70+
{
71+
"Name": "{{mockAgent1.Name}}",
72+
"Instruction":"Proceed",
73+
"Reason":"TestReason",
74+
"IsTaskComplete": {
75+
"Result": false,
76+
"Reason": "Test"
77+
},
78+
"IsTaskProgressing": {
79+
"Result": true,
80+
"Reason": "Test"
81+
},
82+
"IsTaskInLoop": {
83+
"Result": false,
84+
"Reason": "Test"
85+
}
86+
}
87+
""";
88+
Mock<IChatCompletionService> chatServiceMock = CreateMockChatCompletionService(jsonStatus);
89+
90+
FakePromptExecutionSettings settings = new();
91+
StandardMagenticManager manager = new(chatServiceMock.Object, settings)
92+
{
93+
MaximumInvocationCount = 1, // Fast failure for testing
94+
};
95+
96+
MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]);
97+
98+
// Act
99+
await runtime.StartAsync();
100+
101+
const string InitialInput = "123";
102+
OrchestrationResult<string> result = await orchestration.InvokeAsync(InitialInput, runtime);
103+
string response = await result.GetValueAsync(TimeSpan.FromSeconds(20));
104+
105+
// Assert
106+
Assert.NotNull(response);
107+
Assert.Contains("No partial result available.", response);
108+
}
109+
110+
[Fact]
111+
public async Task MagenticOrchestrationMaxInvocationCountReached_WithPartialResultAsync()
112+
{
113+
// Arrange
114+
await using InProcessRuntime runtime = new();
115+
116+
MockAgent mockAgent1 = CreateMockAgent(1, "abc");
117+
MockAgent mockAgent2 = CreateMockAgent(2, "xyz");
118+
MockAgent mockAgent3 = CreateMockAgent(3, "lmn");
119+
120+
string jsonStatus =
121+
$$"""
122+
{
123+
"Name": "{{mockAgent1.Name}}",
124+
"Instruction":"Proceed",
125+
"Reason":"TestReason",
126+
"IsTaskComplete": {
127+
"Result": false,
128+
"Reason": "Test"
129+
},
130+
"IsTaskProgressing": {
131+
"Result": true,
132+
"Reason": "Test"
133+
},
134+
"IsTaskInLoop": {
135+
"Result": false,
136+
"Reason": "Test"
137+
}
138+
}
139+
""";
140+
Mock<IChatCompletionService> chatServiceMock = CreateMockChatCompletionService(jsonStatus);
141+
142+
FakePromptExecutionSettings settings = new();
143+
StandardMagenticManager manager = new(chatServiceMock.Object, settings)
144+
{
145+
MaximumInvocationCount = 2, // Fast failure for testing but at least one invocation
146+
};
147+
148+
MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]);
149+
150+
// Act
151+
await runtime.StartAsync();
152+
153+
const string InitialInput = "123";
154+
OrchestrationResult<string> result = await orchestration.InvokeAsync(InitialInput, runtime);
155+
string response = await result.GetValueAsync(TimeSpan.FromSeconds(20));
156+
157+
// Assert
158+
Assert.NotNull(response);
159+
Assert.Equal("abc", response);
160+
}
161+
162+
[Fact]
163+
public async Task MagenticOrchestrationMaxResetCountReached_WithoutPartialResultAsync()
164+
{
165+
// Arrange
166+
await using InProcessRuntime runtime = new();
167+
168+
MockAgent mockAgent1 = CreateMockAgent(1, "abc");
169+
MockAgent mockAgent2 = CreateMockAgent(2, "xyz");
170+
MockAgent mockAgent3 = CreateMockAgent(3, "lmn");
171+
172+
string jsonStatus =
173+
$$"""
174+
{
175+
"Name": "{{mockAgent1.Name}}",
176+
"Instruction":"Proceed",
177+
"Reason":"TestReason",
178+
"IsTaskComplete": {
179+
"Result": false,
180+
"Reason": "Test"
181+
},
182+
"IsTaskProgressing": {
183+
"Result": false,
184+
"Reason": "Test"
185+
},
186+
"IsTaskInLoop": {
187+
"Result": true,
188+
"Reason": "Test"
189+
}
190+
}
191+
""";
192+
Mock<IChatCompletionService> chatServiceMock = CreateMockChatCompletionService(jsonStatus);
193+
194+
FakePromptExecutionSettings settings = new();
195+
StandardMagenticManager manager = new(chatServiceMock.Object, settings)
196+
{
197+
MaximumResetCount = 1, // Fast failure for testing
198+
MaximumStallCount = 0, // No stalls allowed
199+
};
200+
201+
MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]);
202+
203+
// Act
204+
await runtime.StartAsync();
205+
206+
const string InitialInput = "123";
207+
OrchestrationResult<string> result = await orchestration.InvokeAsync(InitialInput, runtime);
208+
string response = await result.GetValueAsync(TimeSpan.FromSeconds(20));
209+
210+
// Assert
211+
Assert.NotNull(response);
212+
Assert.Contains("No partial result available.", response);
213+
}
214+
215+
[Fact]
216+
public async Task MagenticOrchestrationMaxResetCountReached_WithPartialResultAsync()
217+
{
218+
// Arrange
219+
await using InProcessRuntime runtime = new();
220+
221+
MockAgent mockAgent1 = CreateMockAgent(1, "abc");
222+
MockAgent mockAgent2 = CreateMockAgent(2, "xyz");
223+
MockAgent mockAgent3 = CreateMockAgent(3, "lmn");
224+
225+
string jsonStatus =
226+
$$"""
227+
{
228+
"Name": "{{mockAgent1.Name}}",
229+
"Instruction":"Proceed",
230+
"Reason":"TestReason",
231+
"IsTaskComplete": {
232+
"Result": false,
233+
"Reason": "Test"
234+
},
235+
"IsTaskProgressing": {
236+
"Result": false,
237+
"Reason": "Test"
238+
},
239+
"IsTaskInLoop": {
240+
"Result": true,
241+
"Reason": "Test"
242+
}
243+
}
244+
""";
245+
Mock<IChatCompletionService> chatServiceMock = CreateMockChatCompletionService(jsonStatus);
246+
247+
FakePromptExecutionSettings settings = new();
248+
StandardMagenticManager manager = new(chatServiceMock.Object, settings)
249+
{
250+
MaximumResetCount = 1, // Fast failure for testing but at least one response
251+
MaximumStallCount = 2, // Allow some stalls for at least one response
252+
};
253+
254+
MagenticOrchestration orchestration = new(manager, [mockAgent1, mockAgent2, mockAgent3]);
255+
256+
// Act
257+
await runtime.StartAsync();
258+
259+
const string InitialInput = "123";
260+
OrchestrationResult<string> result = await orchestration.InvokeAsync(InitialInput, runtime);
261+
string response = await result.GetValueAsync(TimeSpan.FromSeconds(20));
262+
263+
// Assert
264+
Assert.NotNull(response);
265+
Assert.Contains("abc", response);
266+
}
267+
58268
private async Task<string> ExecuteOrchestrationAsync(InProcessRuntime runtime, string answer, params Agent[] mockAgents)
59269
{
60270
// Act
@@ -81,6 +291,7 @@ private static MockAgent CreateMockAgent(int index, string response)
81291
{
82292
return new()
83293
{
294+
Name = $"MockAgent{index}",
84295
Description = $"test {index}",
85296
Response = [new(AuthorRole.Assistant, response)]
86297
};
@@ -130,4 +341,29 @@ MagenticProgressLedger CreateLedger(bool isTaskComplete, string name)
130341
}
131342
}
132343
}
344+
345+
private static Mock<IChatCompletionService> CreateMockChatCompletionService(string response)
346+
{
347+
Mock<IChatCompletionService> chatServiceMock = new(MockBehavior.Strict);
348+
349+
chatServiceMock.Setup(
350+
(service) => service.GetChatMessageContentsAsync(
351+
It.IsAny<ChatHistory>(),
352+
It.IsAny<PromptExecutionSettings>(),
353+
null,
354+
It.IsAny<CancellationToken>()))
355+
.ReturnsAsync([new ChatMessageContent(AuthorRole.Assistant, response)]);
356+
357+
return chatServiceMock;
358+
}
359+
360+
private sealed class FakePromptExecutionSettings : PromptExecutionSettings
361+
{
362+
public override PromptExecutionSettings Clone()
363+
{
364+
return this;
365+
}
366+
367+
public object? ResponseFormat { get; set; }
368+
}
133369
}

‎python/semantic_kernel/agents/orchestration/magentic.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -660,7 +660,7 @@ async def _check_within_limits(self) -> bool:
660660

661661
if hit_round_limit or hit_reset_limit:
662662
limit_type = "round" if hit_round_limit else "reset"
663-
logger.debug(f"Max {limit_type} count reached.")
663+
logger.error(f"Max {limit_type} count reached.")
664664

665665
# Retrieve the latest assistant content produced so far
666666
partial_result = next(

0 commit comments

Comments
 (0)