Skip to content
This repository was archived by the owner on Oct 11, 2023. It is now read-only.

Commit 8a31fcf

Browse files
ap-git-hubsaixiaohui
authored andcommitted
Integrate and merge replay feature to master (#338)
* Create replay thread (#325) * Create empty replay thread * Remove connection using * Remove references to connections in the replay task * Update name of replay task test * Move const to config file * Add device replay actors (#329) * Create empty replay thread * Remove connection using * Remove references to connections in the replay task * Update name of replay task test * Add device replay actors * Read replay file and stream telemetry (#334) * update * read replay file from storage * replay file settings * update * update * update validation * PR comments * fix test
1 parent 942b2d5 commit 8a31fcf

File tree

18 files changed

+666
-52
lines changed

18 files changed

+666
-52
lines changed

Services.Test/ReplayFileServiceTest.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ public ReplayFileServiceTest()
4848
this.target = new ReplayFileService(
4949
this.config.Object,
5050
this.enginesFactory.Object,
51-
this.replayFilesStorage.Object,
5251
this.log.Object);
5352
}
5453

Services/Concurrency/ConcurrencyConfig.cs

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public interface IAppConcurrencyConfig
1212
int MaxPendingTelemetry { get; }
1313
int MaxPendingTwinWrites { get; }
1414
int MinDeviceStateLoopDuration { get; }
15+
int MinDeviceReplayLoopDuration { get; }
1516
int MinDeviceConnectionLoopDuration { get; }
1617
int MinDeviceTelemetryLoopDuration { get; }
1718
int MinDevicePropertiesLoopDuration { get; }
@@ -26,6 +27,7 @@ public class AppConcurrencyConfig : IAppConcurrencyConfig
2627
private const int DEFAULT_MAX_PENDING_TELEMETRY = 1000;
2728
private const int DEFAULT_MAX_PENDING_TWIN_WRITES = 50;
2829
private const int DEFAULT_MIN_DEVICE_STATE_LOOP_DURATION = 1000;
30+
private const int DEFAULT_MIN_DEVICE_REPLAY_LOOP_DURATION = 1000;
2931
private const int DEFAULT_MIN_DEVICE_CONNECTION_LOOP_DURATION = 1000;
3032
private const int DEFAULT_MIN_DEVICE_TELEMETRY_LOOP_DURATION = 500;
3133
private const int DEFAULT_MIN_DEVICE_PROPERTIES_LOOP_DURATION = 2000;
@@ -42,6 +44,7 @@ public class AppConcurrencyConfig : IAppConcurrencyConfig
4244
private int maxPendingTelemetry;
4345
private int maxPendingTwinWrites;
4446
private int minDeviceStateLoopDuration;
47+
private int minDeviceReplayLoopDuration;
4548
private int minDeviceConnectionLoopDuration;
4649
private int minDeviceTelemetryLoopDuration;
4750
private int minDevicePropertiesLoopDuration;
@@ -55,6 +58,7 @@ public AppConcurrencyConfig()
5558
this.MaxPendingTelemetry = DEFAULT_MAX_PENDING_TELEMETRY;
5659
this.MaxPendingTwinWrites = DEFAULT_MAX_PENDING_TWIN_WRITES;
5760
this.MinDeviceStateLoopDuration = DEFAULT_MIN_DEVICE_STATE_LOOP_DURATION;
61+
this.MinDeviceReplayLoopDuration = DEFAULT_MIN_DEVICE_REPLAY_LOOP_DURATION;
5862
this.MinDeviceConnectionLoopDuration = DEFAULT_MIN_DEVICE_CONNECTION_LOOP_DURATION;
5963
this.MinDeviceTelemetryLoopDuration = DEFAULT_MIN_DEVICE_TELEMETRY_LOOP_DURATION;
6064
this.MinDevicePropertiesLoopDuration = DEFAULT_MIN_DEVICE_PROPERTIES_LOOP_DURATION;
@@ -207,6 +211,27 @@ public int MinDeviceTelemetryLoopDuration
207211
}
208212
}
209213

214+
/// <summary>
215+
/// When sending telemetry for all the devices in a thread, slow down if the loop through
216+
/// all the devices takes less than N msecs. This is also the minimum time between two
217+
/// messages from the same device.
218+
/// </summary>
219+
public int MinDeviceReplayLoopDuration
220+
{
221+
get => this.minDeviceReplayLoopDuration;
222+
set
223+
{
224+
if (value < 1 || value > MAX_LOOP_DURATION)
225+
{
226+
throw new InvalidConfigurationException(
227+
"The min duration of the device telemetry loop is not valid. " +
228+
"Use a value within the range of 1 and " + MAX_LOOP_DURATION);
229+
}
230+
231+
this.minDeviceReplayLoopDuration = value;
232+
}
233+
}
234+
210235
/// <summary>
211236
/// When writing device twins for all the devices in a thread, slow down if the loop through
212237
/// all the devices takes less than N msecs.

Services/Models/DeviceModel.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ public DeviceModel()
6363
this.Protocol = IoTHubProtocol.AMQP;
6464
this.Simulation = new StateSimulation();
6565
this.Properties = new Dictionary<string, object>();
66+
6667
this.Telemetry = new List<DeviceModelMessage>();
6768
this.CloudToDeviceMethods = new Dictionary<string, Script>();
6869
}

Services/Models/Simulation.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,12 @@ public DateTimeOffset? EndTime
136136
[JsonProperty(Order = 150)]
137137
public DateTimeOffset? ActualStartTime { get; set; }
138138

139-
// ReplayFileId is the id of the replay file in storage
139+
// ReplayFileId is the replay file data in storage
140140
[JsonProperty(Order = 160)]
141141
public string ReplayFileId { get; set; }
142142

143+
public bool ReplayFileRunIndefinitely { get; set; }
144+
143145
public Simulation()
144146
{
145147
// When unspecified, a simulation is enabled

Services/ReplayFileService.cs

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Runtime;
1010
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Storage;
1111
using Newtonsoft.Json;
12-
using Microsoft.VisualBasic.FileIO;
13-
using FieldType = Microsoft.VisualBasic.FileIO.FieldType;
1412

1513
namespace Microsoft.Azure.IoTSolutions.DeviceSimulation.Services
1614
{
@@ -39,13 +37,13 @@ public interface IReplayFileService
3937

4038
public class ReplayFileService : IReplayFileService
4139
{
40+
private const int NUM_CSV_COLS = 3;
4241
private readonly IEngine replayFilesStorage;
4342
private readonly ILogger log;
4443

4544
public ReplayFileService(
4645
IServicesConfig config,
4746
IEngines engines,
48-
IEngine storage,
4947
ILogger logger)
5048
{
5149
this.replayFilesStorage = engines.Build(config.ReplayFilesStorage);
@@ -149,25 +147,26 @@ public string ValidateFile(Stream stream)
149147
{
150148
var reader = new StreamReader(stream);
151149
var file = reader.ReadToEnd();
152-
153-
using (TextFieldParser parser = new TextFieldParser(file))
150+
151+
while (!reader.EndOfStream)
154152
{
155-
parser.TextFieldType = FieldType.Delimited;
156-
parser.SetDelimiters(",");
157-
while (!parser.EndOfData)
153+
try
158154
{
159-
try
160-
{
161-
string[] lines = parser.ReadFields();
162-
}
163-
catch (MalformedLineException ex)
155+
string line = reader.ReadLine();
156+
string[] fields = line.Split(',');
157+
if (fields.Length < NUM_CSV_COLS)
164158
{
165-
this.log.Error("Replay file has invalid csv format", () => new { ex });
166-
throw new InvalidInputException("Replay file has invalid csv format", ex);
159+
this.log.Error("Replay file has invalid csv format");
160+
throw new InvalidInputException("Replay file has invalid csv format");
167161
}
168162
}
163+
catch (Exception ex)
164+
{
165+
this.log.Error("Error parsing replay file", () => new { ex });
166+
throw new InvalidInputException("Error parsing replay file", ex);
167+
}
169168
}
170-
169+
171170
return file;
172171
}
173172
}

Services/Services.csproj

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
<RootNamespace>Microsoft.Azure.IoTSolutions.DeviceSimulation.Services</RootNamespace>
77
</PropertyGroup>
88
<ItemGroup>
9+
<Content Include="data\replayfile\replay.json">
10+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
11+
</Content>
12+
<Content Include="data\replayfile\simulationReplayTest.csv">
13+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
14+
</Content>
915
<Content Include="data\templates\multiple-simulations-template.json">
1016
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
1117
</Content>
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"SchemaVersion": "1.0.0",
3+
"Id": "replay",
4+
"Version": "0.0.1",
5+
"Name": "Replay",
6+
"Description": "Fake device model for csv file replay",
7+
"Protocol": "AMQP",
8+
"ReplayFile": "",
9+
"Simulation": {
10+
"InitialState": {
11+
"online": true
12+
},
13+
"Interval": "00:00:00",
14+
"Scripts": [
15+
]
16+
},
17+
"Properties": {
18+
},
19+
"Tags": {
20+
},
21+
"Telemetry": [
22+
{
23+
"MessageTemplate": "",
24+
"MessageSchema": {
25+
"Name": "replay-sensors;v1",
26+
"Format": "JSON",
27+
"Fields": {
28+
}
29+
}
30+
}
31+
],
32+
"CloudToDeviceMethods": {
33+
}
34+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
telemetry, 00:00:00,{"temperature": 51.32, "temperature_unit": "fahrenheit", "humidity": 69.59, "humidity_unit":"RH", "pressure": 440.20, "pressure_unit": "psi"}
2+
telemetry, 00:00:30,{"temperature": 71.32, "temperature_unit": "fahrenheit", "humidity": 79.59, "humidity_unit":"RH", "pressure": 240.20, "pressure_unit": "psi"}
3+
telemetry, 00:01:00,{"temperature": 30.00, "temperature_unit": "fahrenheit", "humidity": 59.59, "humidity_unit":"RH", "pressure": 340.20, "pressure_unit": "psi"}

SimulationAgent.Test/SimulationManagerTest.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent;
1616
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceConnection;
1717
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceProperties;
18+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceReplay;
1819
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceState;
1920
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceTelemetry;
2021
using Moq;
@@ -50,6 +51,7 @@ public class SimulationManagerTest
5051
private readonly ConcurrentDictionary<string, IDeviceConnectionActor> mockDeviceContext;
5152
private readonly ConcurrentDictionary<string, IDeviceTelemetryActor> deviceTelemetryActors;
5253
private readonly ConcurrentDictionary<string, IDevicePropertiesActor> devicePropertiesActors;
54+
private readonly ConcurrentDictionary<string, IDeviceReplayActor> deviceReplayActors;
5355

5456
private SimulationManager target;
5557

@@ -85,13 +87,15 @@ public SimulationManagerTest()
8587
this.mockDeviceContext = new ConcurrentDictionary<string, IDeviceConnectionActor>();
8688
this.deviceTelemetryActors = new ConcurrentDictionary<string, IDeviceTelemetryActor>();
8789
this.devicePropertiesActors = new ConcurrentDictionary<string, IDevicePropertiesActor>();
90+
this.deviceReplayActors = new ConcurrentDictionary<string, IDeviceReplayActor>();
8891

8992
this.target.InitAsync(
9093
simulation,
9194
this.deviceStateActors,
9295
this.mockDeviceContext,
9396
this.deviceTelemetryActors,
94-
this.devicePropertiesActors).Wait(Constants.TEST_TIMEOUT);
97+
this.devicePropertiesActors,
98+
this.deviceReplayActors).Wait(Constants.TEST_TIMEOUT);
9599
}
96100

97101
[Fact]
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System.Collections.Concurrent;
4+
using System.Threading;
5+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Concurrency;
6+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.Services.Diagnostics;
7+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent;
8+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.DeviceReplay;
9+
using Microsoft.Azure.IoTSolutions.DeviceSimulation.SimulationAgent.SimulationThreads;
10+
using Moq;
11+
using Xunit;
12+
using System.Threading.Tasks;
13+
14+
namespace SimulationAgent.Test.SimulationThreads
15+
{
16+
public class DeviceReplayTaskTest
17+
{
18+
private const int NUM_ACTORS = 9;
19+
private const int MAX_PENDING_TASKS = 5;
20+
21+
private readonly Mock<IAppConcurrencyConfig> mockAppConcurrencyConfig;
22+
private readonly Mock<ILogger> mockLogger;
23+
private readonly DeviceReplayTask target;
24+
private readonly ConcurrentDictionary<string, Mock<IDeviceReplayActor>> mockDeviceReplayActors;
25+
private readonly ConcurrentDictionary<string, IDeviceReplayActor> mockDeviceReplayActorObjects;
26+
private readonly ConcurrentDictionary<string, Mock<ISimulationManager>> mockSimulationManagers;
27+
private readonly ConcurrentDictionary<string, ISimulationManager> mockSimulationManagerObjects;
28+
29+
public DeviceReplayTaskTest()
30+
{
31+
this.mockDeviceReplayActors = new ConcurrentDictionary<string, Mock<IDeviceReplayActor>>();
32+
this.mockDeviceReplayActorObjects = new ConcurrentDictionary<string, IDeviceReplayActor>();
33+
this.mockSimulationManagers = new ConcurrentDictionary<string, Mock<ISimulationManager>>();
34+
this.mockSimulationManagerObjects = new ConcurrentDictionary<string, ISimulationManager>();
35+
36+
this.mockAppConcurrencyConfig = new Mock<IAppConcurrencyConfig>();
37+
this.mockAppConcurrencyConfig.SetupGet(x => x.MaxPendingTasks).Returns(MAX_PENDING_TASKS);
38+
this.mockLogger = new Mock<ILogger>();
39+
40+
this.target = new DeviceReplayTask(this.mockAppConcurrencyConfig.Object, this.mockLogger.Object);
41+
}
42+
43+
[Fact]
44+
public void ItCallsRunAsyncOnAllReplayActors()
45+
{
46+
// Arrange
47+
var cancellationToken = new CancellationTokenSource();
48+
49+
this.BuildMockDeviceReplayActors(
50+
this.mockDeviceReplayActors,
51+
this.mockDeviceReplayActorObjects,
52+
cancellationToken,
53+
NUM_ACTORS);
54+
55+
// Build a list of SimulationManagers
56+
this.BuildMockSimluationManagers(
57+
this.mockSimulationManagers,
58+
this.mockSimulationManagerObjects,
59+
cancellationToken,
60+
NUM_ACTORS);
61+
62+
// Act
63+
// Act on the target. The cancellation token will be cancelled through
64+
// a callback that will be triggered when each device-replay actor
65+
// is called.
66+
var targetTask = this.target.RunAsync(
67+
this.mockSimulationManagerObjects,
68+
this.mockDeviceReplayActorObjects,
69+
cancellationToken.Token);
70+
71+
// Assert
72+
// Verify that each SimulationManager was called at least once
73+
foreach (var actor in this.mockDeviceReplayActors)
74+
actor.Value.Verify(x => x.HasWorkToDo(), Times.Once);
75+
}
76+
77+
private void BuildMockDeviceReplayActors(
78+
ConcurrentDictionary<string, Mock<IDeviceReplayActor>> mockDictionary,
79+
ConcurrentDictionary<string, IDeviceReplayActor> objectDictionary,
80+
CancellationTokenSource cancellationToken,
81+
int count)
82+
{
83+
mockDictionary.Clear();
84+
objectDictionary.Clear();
85+
86+
for (int i = 0; i < count; i++)
87+
{
88+
var deviceName = $"device_{i}";
89+
var mockDeviceReplayActor = new Mock<IDeviceReplayActor>();
90+
91+
// Have each DeviceReplayActor report that it has work to do
92+
mockDeviceReplayActor.Setup(x => x.HasWorkToDo()).Returns(true);
93+
mockDeviceReplayActor.Setup(x => x.RunAsync()).Returns(Task.CompletedTask)
94+
.Callback(() => { cancellationToken.Cancel(); });
95+
96+
mockDictionary.TryAdd(deviceName, mockDeviceReplayActor);
97+
objectDictionary.TryAdd(deviceName, mockDeviceReplayActor.Object);
98+
}
99+
}
100+
101+
/*
102+
* Creating two collections: one for the mocks, and another to store the
103+
* mock objects. If we only created one collection and populated it with
104+
* the mock objects, we wouldn't have a reference to the backing mock for
105+
* each.
106+
*/
107+
private void BuildMockSimluationManagers(
108+
ConcurrentDictionary<string, Mock<ISimulationManager>> mockSimulationManagers,
109+
ConcurrentDictionary<string, ISimulationManager> mockSimulationManagerObjects,
110+
CancellationTokenSource cancellationToken,
111+
int count)
112+
{
113+
mockSimulationManagers.Clear();
114+
mockSimulationManagerObjects.Clear();
115+
116+
for (int i = 0; i < count; i++)
117+
{
118+
var deviceName = $"simulation_{i}";
119+
var mockSimulationManager = new Mock<ISimulationManager>();
120+
121+
// We only want the main loop in the target to run once, so here we'll
122+
// trigger a callback which will cancel the cancellation token that
123+
// the main loop uses.
124+
mockSimulationManager.Setup(x => x.NewConnectionLoop())
125+
.Callback(() => cancellationToken.Cancel());
126+
127+
mockSimulationManagers.TryAdd(deviceName, mockSimulationManager);
128+
mockSimulationManagerObjects.TryAdd(deviceName, mockSimulationManager.Object);
129+
}
130+
}
131+
}
132+
}

0 commit comments

Comments
 (0)