Skip to content

Commit c565f4c

Browse files
jhonabreuljhonabreul
authored
Refactor optimization statistics serialization (#8984)
* Refactor optimization result stats serialization * Handle custom optimization statistics serialization * Support custom statistics * Support newest Lean statistics Address peer review * Add more tests * Make indices reserved statistic names * Minor fixes and cleanup
1 parent f804b8f commit c565f4c

File tree

4 files changed

+196
-51
lines changed

4 files changed

+196
-51
lines changed

‎Algorithm/QCAlgorithm.Plotting.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,12 @@ public void SetRuntimeStatistic(string name, double value)
497497
[DocumentationAttribute(StatisticsTag)]
498498
public void SetSummaryStatistic(string name, string value)
499499
{
500+
if (int.TryParse(name, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intName) &&
501+
intName >= 0 && intName <= 100)
502+
{
503+
throw new ArgumentException($"'{name}' is a reserved statistic name.");
504+
}
505+
500506
_statisticsService.SetSummaryStatistic(name, value);
501507
}
502508

@@ -508,7 +514,7 @@ public void SetSummaryStatistic(string name, string value)
508514
[DocumentationAttribute(StatisticsTag)]
509515
public void SetSummaryStatistic(string name, int value)
510516
{
511-
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
517+
SetSummaryStatistic(name, value.ToStringInvariant());
512518
}
513519

514520
/// <summary>
@@ -519,7 +525,7 @@ public void SetSummaryStatistic(string name, int value)
519525
[DocumentationAttribute(StatisticsTag)]
520526
public void SetSummaryStatistic(string name, double value)
521527
{
522-
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
528+
SetSummaryStatistic(name, value.ToStringInvariant());
523529
}
524530

525531
/// <summary>
@@ -530,7 +536,7 @@ public void SetSummaryStatistic(string name, double value)
530536
[DocumentationAttribute(StatisticsTag)]
531537
public void SetSummaryStatistic(string name, decimal value)
532538
{
533-
_statisticsService.SetSummaryStatistic(name, value.ToStringInvariant());
539+
SetSummaryStatistic(name, value.ToStringInvariant());
534540
}
535541

536542
/// <summary>

‎Common/Api/OptimizationBacktestJsonConverter.cs‎

Lines changed: 73 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515

1616
using System;
1717
using System.Collections.Generic;
18-
using System.Globalization;
1918
using System.Linq;
19+
using System.Runtime.CompilerServices;
2020
using Newtonsoft.Json;
2121
using Newtonsoft.Json.Linq;
2222
using QuantConnect.Optimizer.Parameters;
@@ -30,6 +30,43 @@ namespace QuantConnect.Api
3030
/// </summary>
3131
public class OptimizationBacktestJsonConverter : JsonConverter
3232
{
33+
private static Dictionary<string, int> StatisticsIndices = new()
34+
{
35+
{ PerformanceMetrics.Alpha, 0 },
36+
{ PerformanceMetrics.AnnualStandardDeviation, 1 },
37+
{ PerformanceMetrics.AnnualVariance, 2 },
38+
{ PerformanceMetrics.AverageLoss, 3 },
39+
{ PerformanceMetrics.AverageWin, 4 },
40+
{ PerformanceMetrics.Beta, 5 },
41+
{ PerformanceMetrics.CompoundingAnnualReturn, 6 },
42+
{ PerformanceMetrics.Drawdown, 7 },
43+
{ PerformanceMetrics.EstimatedStrategyCapacity, 8 },
44+
{ PerformanceMetrics.Expectancy, 9 },
45+
{ PerformanceMetrics.InformationRatio, 10 },
46+
{ PerformanceMetrics.LossRate, 11 },
47+
{ PerformanceMetrics.NetProfit, 12 },
48+
{ PerformanceMetrics.ProbabilisticSharpeRatio, 13 },
49+
{ PerformanceMetrics.ProfitLossRatio, 14 },
50+
{ PerformanceMetrics.SharpeRatio, 15 },
51+
{ PerformanceMetrics.TotalFees, 16 },
52+
{ PerformanceMetrics.TotalOrders, 17 },
53+
{ PerformanceMetrics.TrackingError, 18 },
54+
{ PerformanceMetrics.TreynorRatio, 19 },
55+
{ PerformanceMetrics.WinRate, 20 },
56+
{ PerformanceMetrics.SortinoRatio, 21 },
57+
{ PerformanceMetrics.StartEquity, 22 },
58+
{ PerformanceMetrics.EndEquity, 23 },
59+
{ PerformanceMetrics.DrawdownRecovery, 24 },
60+
};
61+
62+
private static string[] StatisticNames { get; } = StatisticsIndices
63+
.OrderBy(kvp => kvp.Value)
64+
.Select(kvp => kvp.Key)
65+
.ToArray();
66+
67+
// Only 21 Lean statistics where supported when the serialized statistics where a json array
68+
private static int ArrayStatisticsCount = 21;
69+
3370
/// <summary>
3471
/// Determines whether this instance can convert the specified object type.
3572
/// </summary>
@@ -97,25 +134,23 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
97134
if (!optimizationBacktest.Statistics.IsNullOrEmpty())
98135
{
99136
writer.WritePropertyName("statistics");
100-
writer.WriteStartArray();
101-
foreach (var keyValuePair in optimizationBacktest.Statistics.OrderBy(pair => pair.Key))
137+
writer.WriteStartObject();
138+
139+
var customStatisticsNames = new HashSet<string>();
140+
141+
foreach (var (name, statisticValue, index) in optimizationBacktest.Statistics
142+
.Select(kvp => (Name: kvp.Key, kvp.Value, Index: StatisticsIndices.TryGetValue(kvp.Key, out var index) ? index : int.MaxValue))
143+
.OrderBy(t => t.Index)
144+
.ThenByDescending(t => t.Name))
102145
{
103-
switch (keyValuePair.Key)
104-
{
105-
case PerformanceMetrics.PortfolioTurnover:
106-
case PerformanceMetrics.SortinoRatio:
107-
case PerformanceMetrics.StartEquity:
108-
case PerformanceMetrics.EndEquity:
109-
case PerformanceMetrics.DrawdownRecovery:
110-
continue;
111-
}
112-
var statistic = keyValuePair.Value.Replace("%", string.Empty);
146+
var statistic = statisticValue.Replace("%", string.Empty, StringComparison.InvariantCulture);
113147
if (Currencies.TryParse(statistic, out var result))
114148
{
149+
writer.WritePropertyName(index < StatisticsIndices.Count ? index.ToStringInvariant() : name);
115150
writer.WriteValue(result);
116151
}
117152
}
118-
writer.WriteEndArray();
153+
writer.WriteEndObject();
119154
}
120155

121156
if (optimizationBacktest.ParameterSet != null)
@@ -164,34 +199,25 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
164199
Dictionary<string, string> statistics = default;
165200
if (jStatistics != null)
166201
{
167-
statistics = new Dictionary<string, string>
202+
if (jStatistics.Type == JTokenType.Array)
168203
{
169-
{ PerformanceMetrics.Alpha, jStatistics[0].Value<string>() },
170-
{ PerformanceMetrics.AnnualStandardDeviation, jStatistics[1].Value<string>() },
171-
{ PerformanceMetrics.AnnualVariance, jStatistics[2].Value<string>() },
172-
{ PerformanceMetrics.AverageLoss, jStatistics[3].Value<string>() },
173-
{ PerformanceMetrics.AverageWin, jStatistics[4].Value<string>() },
174-
{ PerformanceMetrics.Beta, jStatistics[5].Value<string>() },
175-
{ PerformanceMetrics.CompoundingAnnualReturn, jStatistics[6].Value<string>() },
176-
{ PerformanceMetrics.Drawdown, jStatistics[7].Value<string>() },
177-
{ PerformanceMetrics.EstimatedStrategyCapacity, jStatistics[8].Value<string>() },
178-
{ PerformanceMetrics.Expectancy, jStatistics[9].Value<string>() },
179-
{ PerformanceMetrics.InformationRatio, jStatistics[10].Value<string>() },
180-
{ PerformanceMetrics.LossRate, jStatistics[11].Value<string>() },
181-
{ PerformanceMetrics.NetProfit, jStatistics[12].Value<string>() },
182-
{ PerformanceMetrics.ProbabilisticSharpeRatio, jStatistics[13].Value<string>() },
183-
{ PerformanceMetrics.ProfitLossRatio, jStatistics[14].Value<string>() },
184-
{ PerformanceMetrics.SharpeRatio, jStatistics[15].Value<string>() },
185-
// TODO: Add SortinoRatio
186-
// TODO: Add StartingEquity
187-
// TODO: Add EndingEquity
188-
// TODO: Add DrawdownRecovery
189-
{ PerformanceMetrics.TotalFees, jStatistics[16].Value<string>() },
190-
{ PerformanceMetrics.TotalOrders, jStatistics[17].Value<string>() },
191-
{ PerformanceMetrics.TrackingError, jStatistics[18].Value<string>() },
192-
{ PerformanceMetrics.TreynorRatio, jStatistics[19].Value<string>() },
193-
{ PerformanceMetrics.WinRate, jStatistics[20].Value<string>() },
194-
};
204+
var statsCount = Math.Min(ArrayStatisticsCount, (jStatistics as JArray).Count);
205+
statistics = new Dictionary<string, string>(StatisticsIndices
206+
.Where(kvp => kvp.Value < statsCount)
207+
.Select(kvp => KeyValuePair.Create(kvp.Key, jStatistics[kvp.Value].Value<string>()))
208+
.Where(kvp => kvp.Value != null));
209+
}
210+
else
211+
{
212+
statistics = new();
213+
foreach (var statistic in jStatistics.Children<JProperty>())
214+
{
215+
var statisticName = TryConvertToLeanStatisticIndex(statistic.Name, out var index)
216+
? StatisticNames[index]
217+
: statistic.Name;
218+
statistics[statisticName] = statistic.Value.Value<string>();
219+
}
220+
}
195221
}
196222

197223
var parameterSet = serializer.Deserialize<ParameterSet>(jObject["parameterSet"].CreateReader());
@@ -220,5 +246,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
220246

221247
return optimizationBacktest;
222248
}
249+
250+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
251+
private static bool TryConvertToLeanStatisticIndex(string statistic, out int index)
252+
{
253+
return int.TryParse(statistic, out index) && index >= 0 && index < StatisticsIndices.Count;
254+
}
223255
}
224256
}

‎Tests/Algorithm/AlgorithmPlottingTests.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,5 +220,16 @@ public void PlotIndicatorPlotsBaseIndicator()
220220
Assert.AreEqual("PlotTest", chart.Name);
221221
Assert.AreEqual(sma1.Current.Value / sma2.Current.Value, chart.Series[ratio.Name].GetValues<ChartPoint>().First().y);
222222
}
223+
224+
private static string[] ReservedSummaryStatisticNames => Enumerable.Range(0, 101).Select(i => i.ToStringInvariant()).ToArray();
225+
226+
[TestCaseSource(nameof(ReservedSummaryStatisticNames))]
227+
public void ThrowsOnReservedSummaryStatisticName(string statisticName)
228+
{
229+
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 0.1m));
230+
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 0.1));
231+
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, 1));
232+
Assert.Throws<ArgumentException>(() => _algorithm.SetSummaryStatistic(statisticName, "0.1"));
233+
}
223234
}
224235
}

0 commit comments

Comments
 (0)