Skip to content

Commit c73bc15

Browse files
balag0balag0
committed
[Linux consumption] Notify health events to mesh service process
Notifies mesh service process of fatal errors during specialization. These events are used to monitor overall container health.
1 parent fa9ebb7 commit c73bc15

File tree

7 files changed

+99
-8
lines changed

7 files changed

+99
-8
lines changed

‎src/WebJobs.Script.WebHost/Management/IMeshServiceClient.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

4+
using System;
45
using System.Collections.Generic;
56
using System.Threading.Tasks;
67
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
@@ -16,5 +17,7 @@ public interface IMeshServiceClient
1617
Task MountFuse(string type, string filePath, string scriptPath);
1718

1819
Task PublishContainerActivity(IEnumerable<ContainerFunctionExecutionActivity> activities);
20+
21+
Task NotifyHealthEvent(ContainerHealthEventType healthEventType, Type source, string details);
1922
}
2023
}

‎src/WebJobs.Script.WebHost/Management/InstanceManager.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ public async Task<string> SpecializeMSISidecar(HostAssignmentContext context)
6363
if (context.MSIContext == null)
6464
{
6565
_logger.LogWarning("Skipping specialization of MSI sidecar since MSIContext was absent");
66+
await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, this.GetType(),
67+
"Could not specialize MSI sidecar");
6668
}
6769
else
6870
{
@@ -250,7 +252,8 @@ private async Task Assign(HostAssignmentContext assignmentContext)
250252
}
251253
catch (Exception ex)
252254
{
253-
_logger.LogError(ex, $"Assign failed");
255+
_logger.LogError(ex, "Assign failed");
256+
await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, GetType(), "Assign failed");
254257
throw;
255258
}
256259
finally

‎src/WebJobs.Script.WebHost/Management/MeshServiceClient.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,29 @@ await Utility.InvokeWithRetriesAsync(async () =>
7575
}
7676
}
7777

78+
public async Task NotifyHealthEvent(ContainerHealthEventType healthEventType, Type source, string details)
79+
{
80+
var healthEvent = new ContainerHealthEvent()
81+
{
82+
EventTime = DateTime.UtcNow,
83+
EventType = healthEventType,
84+
Details = details,
85+
Source = source.ToString()
86+
};
87+
88+
var healthEventString = JsonConvert.SerializeObject(healthEvent);
89+
90+
_logger.LogInformation($"Posting health event {healthEventString}");
91+
92+
var responseMessage = await SendAsync(new[]
93+
{
94+
new KeyValuePair<string, string>(Operation, "add-health-event"),
95+
new KeyValuePair<string, string>("healthEvent", healthEventString),
96+
});
97+
98+
_logger.LogInformation($"Posted health event status: {responseMessage.StatusCode}");
99+
}
100+
78101
private async Task PublishActivities(IEnumerable<ContainerFunctionExecutionActivity> activities)
79102
{
80103
// Log one of the activities being published for debugging.

‎src/WebJobs.Script.WebHost/Management/NullMeshServiceClient.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,10 @@ public Task PublishContainerActivity(IEnumerable<ContainerFunctionExecutionActiv
4242
{
4343
return Task.CompletedTask;
4444
}
45+
46+
public Task NotifyHealthEvent(ContainerHealthEventType healthEventType, Type source, string details)
47+
{
48+
return Task.CompletedTask;
49+
}
4550
}
4651
}

‎src/WebJobs.Script.WebHost/Models/ContainerHealthEvent.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
6+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Models
7+
{
8+
public class ContainerHealthEvent
9+
{
10+
public DateTime EventTime { get; set; }
11+
12+
public ContainerHealthEventType EventType { get; set; }
13+
14+
public string Source { get; set; }
15+
16+
public string Details { get; set; }
17+
}
18+
}

‎src/WebJobs.Script.WebHost/Models/ContainerHealthEventType.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System.Runtime.Serialization;
5+
6+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Models
7+
{
8+
// Important: Needs to be in sync with container init process
9+
public enum ContainerHealthEventType
10+
{
11+
[EnumMember]
12+
Informational,
13+
14+
[EnumMember]
15+
Warning,
16+
17+
[EnumMember]
18+
Fatal
19+
}
20+
}

‎test/WebJobs.Script.Tests.Integration/Management/InstanceManagerTests.cs‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public class InstanceManagerTests : IDisposable
3030
private readonly TestEnvironmentEx _environment;
3131
private readonly ScriptWebHostEnvironment _scriptWebEnvironment;
3232
private readonly InstanceManager _instanceManager;
33+
private readonly Mock<IMeshServiceClient> _meshServiceClientMock;
3334
private readonly HttpClient _httpClient;
3435
private readonly LoggerFactory _loggerFactory = new LoggerFactory();
3536
private readonly TestOptionsFactory<ScriptApplicationHostOptions> _optionsFactory = new TestOptionsFactory<ScriptApplicationHostOptions>(new ScriptApplicationHostOptions() { ScriptPath = Path.GetTempPath() });
@@ -43,8 +44,10 @@ public InstanceManagerTests()
4344

4445
_environment = new TestEnvironmentEx();
4546
_scriptWebEnvironment = new ScriptWebHostEnvironment(_environment);
47+
_meshServiceClientMock = new Mock<IMeshServiceClient>(MockBehavior.Strict);
4648

47-
_instanceManager = new InstanceManager(_optionsFactory, _httpClient, _scriptWebEnvironment, _environment, _loggerFactory.CreateLogger<InstanceManager>(), new TestMetricsLogger(), null);
49+
_instanceManager = new InstanceManager(_optionsFactory, _httpClient, _scriptWebEnvironment, _environment,
50+
_loggerFactory.CreateLogger<InstanceManager>(), new TestMetricsLogger(), _meshServiceClientMock.Object);
4851

4952
InstanceManager.Reset();
5053
}
@@ -126,6 +129,10 @@ public async Task StartAssignment_Failure_ExitsPlaceholderMode()
126129
},
127130
IsWarmupRequest = false
128131
};
132+
133+
_meshServiceClientMock.Setup(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
134+
It.Is<Type>(t => t == typeof(InstanceManager)), "Assign failed")).Returns(Task.CompletedTask);
135+
129136
bool result = _instanceManager.StartAssignment(context);
130137
Assert.True(result);
131138
Assert.True(_scriptWebEnvironment.InStandbyMode);
@@ -135,6 +142,9 @@ public async Task StartAssignment_Failure_ExitsPlaceholderMode()
135142
var error = _loggerProvider.GetAllLogMessages().First(p => p.Level == LogLevel.Error);
136143
Assert.Equal("Assign failed", error.FormattedMessage);
137144
Assert.Equal("Kaboom!", error.Exception.Message);
145+
146+
_meshServiceClientMock.Verify(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
147+
It.Is<Type>(t => t == typeof(InstanceManager)), "Assign failed"), Times.Once);
138148
}
139149

140150
[Fact]
@@ -504,7 +514,7 @@ public async Task SpecializeMSISidecar_Succeeds()
504514
MSIContext = new MSIContext()
505515
};
506516

507-
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.OK);
517+
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.OK, null);
508518

509519
string error = await instanceManager.SpecializeMSISidecar(assignmentContext);
510520
Assert.Null(error);
@@ -532,7 +542,7 @@ public async Task SpecializeMSISidecar_NoOp_ForWarmup_Request()
532542
IsWarmupRequest = true
533543
};
534544

535-
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.OK);
545+
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.OK, null);
536546

537547
string error = await instanceManager.SpecializeMSISidecar(assignmentContext);
538548
Assert.Null(error);
@@ -558,7 +568,7 @@ public async Task SpecializeMSISidecar_Fails()
558568
MSIContext = new MSIContext()
559569
};
560570

561-
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.BadRequest);
571+
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.BadRequest, null);
562572

563573
string error = await instanceManager.SpecializeMSISidecar(assignmentContext);
564574
Assert.NotNull(error);
@@ -588,7 +598,12 @@ public async Task DoesNotSpecializeMSISidecar_WhenMSIContextNull()
588598
MSIContext = null
589599
};
590600

591-
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.BadRequest);
601+
602+
var meshServiceClient = new Mock<IMeshServiceClient>(MockBehavior.Strict);
603+
meshServiceClient.Setup(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
604+
It.Is<Type>(t => t == typeof(InstanceManager)), "Could not specialize MSI sidecar")).Returns(Task.CompletedTask);
605+
606+
var instanceManager = GetInstanceManagerForMSISpecialization(assignmentContext, HttpStatusCode.BadRequest, meshServiceClient.Object);
592607

593608
string error = await instanceManager.SpecializeMSISidecar(assignmentContext);
594609
Assert.Null(error);
@@ -597,6 +612,9 @@ public async Task DoesNotSpecializeMSISidecar_WhenMSIContextNull()
597612
Assert.Collection(logs,
598613
p => Assert.StartsWith("MSI enabled status: True", p),
599614
p => Assert.StartsWith("Skipping specialization of MSI sidecar since MSIContext was absent", p));
615+
616+
meshServiceClient.Verify(c => c.NotifyHealthEvent(ContainerHealthEventType.Fatal,
617+
It.Is<Type>(t => t == typeof(InstanceManager)), "Could not specialize MSI sidecar"), Times.Once);
600618
}
601619

602620
[Fact]
@@ -710,7 +728,8 @@ public async Task Does_Not_Mount_Invalid_BYOS_Accounts()
710728
client => client.MountCifs(It.IsAny<string>(), It.IsAny<string>(), It.Is<string>(s => s != targetPath1)), Times.Never());
711729
}
712730

713-
private InstanceManager GetInstanceManagerForMSISpecialization(HostAssignmentContext hostAssignmentContext, HttpStatusCode httpStatusCode)
731+
private InstanceManager GetInstanceManagerForMSISpecialization(HostAssignmentContext hostAssignmentContext,
732+
HttpStatusCode httpStatusCode, IMeshServiceClient meshServiceClient)
714733
{
715734
var handlerMock = new Mock<HttpMessageHandler>(MockBehavior.Strict);
716735

@@ -728,7 +747,7 @@ private InstanceManager GetInstanceManagerForMSISpecialization(HostAssignmentCon
728747
InstanceManager.Reset();
729748

730749
return new InstanceManager(_optionsFactory, new HttpClient(handlerMock.Object), _scriptWebEnvironment, _environment,
731-
_loggerFactory.CreateLogger<InstanceManager>(), new TestMetricsLogger(), null);
750+
_loggerFactory.CreateLogger<InstanceManager>(), new TestMetricsLogger(), meshServiceClient);
732751
}
733752

734753
public void Dispose()

0 commit comments

Comments
 (0)