Skip to content

Commit 08afbc7

Browse files
tanderson-ldtanderson-ld
committed
chore: updates datasources to report recoverability in errors
1 parent ac41228 commit 08afbc7

15 files changed

+335
-93
lines changed

‎pkgs/sdk/server/contract-tests/test-supressions-fdv2.txt‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
1-
streaming/retry behavior/do not retry after unrecoverable HTTP error on initial connect/error 401
2-
streaming/retry behavior/do not retry after unrecoverable HTTP error on initial connect/error 403
3-
streaming/retry behavior/do not retry after unrecoverable HTTP error on initial connect/error 405
4-
streaming/retry behavior/do not retry after unrecoverable HTTP error on reconnect/error 401
5-
streaming/retry behavior/do not retry after unrecoverable HTTP error on reconnect/error 403
6-
streaming/retry behavior/do not retry after unrecoverable HTTP error on reconnect/error 405
71
streaming/fdv2/reconnection state management/initializes from 2 polling initializers
82
streaming/fdv2/disconnects on goodbye
93
streaming/fdv2/reconnection state management/initializes from polling initializer

‎pkgs/sdk/server/src/Interfaces/DataSourceStatus.cs‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ public struct ErrorInfo
6262
/// </summary>
6363
public ErrorKind Kind { get; set; }
6464

65+
/// <summary>
66+
/// Whether the error is recoverable. Recoverable errors are those that can be retried, such as network errors. Unrecoverable
67+
/// errors are those that cannot be retried, such as invalid SDK key errors.
68+
/// </summary>
69+
public bool Recoverable { get; set; }
70+
6571
/// <summary>
6672
/// The HTTP status code if the error was <see cref="ErrorKind.ErrorResponse"/>, or zero otherwise.
6773
/// </summary>
@@ -90,24 +96,28 @@ public struct ErrorInfo
9096
/// Constructs an instance based on an exception.
9197
/// </summary>
9298
/// <param name="e">the exception</param>
99+
/// <param name="recoverable">whether the error is recoverable</param>
93100
/// <returns>an ErrorInfo</returns>
94-
public static ErrorInfo FromException(Exception e) => new ErrorInfo
101+
public static ErrorInfo FromException(Exception e, bool recoverable) => new ErrorInfo
95102
{
96103
Kind = e is IOException ? ErrorKind.NetworkError : ErrorKind.Unknown,
97104
Message = e.Message,
98-
Time = DateTime.Now
105+
Time = DateTime.Now,
106+
Recoverable = recoverable
99107
};
100108

101109
/// <summary>
102110
/// Constructs an instance based on an HTTP error status.
103111
/// </summary>
104112
/// <param name="statusCode">the status code</param>
113+
/// <param name="recoverable">whether the error is recoverable</param>
105114
/// <returns>an ErrorInfo</returns>
106-
public static ErrorInfo FromHttpError(int statusCode) => new ErrorInfo
115+
public static ErrorInfo FromHttpError(int statusCode, bool recoverable) => new ErrorInfo
107116
{
108117
Kind = ErrorKind.ErrorResponse,
109118
StatusCode = statusCode,
110-
Time = DateTime.Now
119+
Time = DateTime.Now,
120+
Recoverable = recoverable
111121
};
112122

113123
/// <inheritdoc/>

‎pkgs/sdk/server/src/Internal/DataSources/PollingDataSource.cs‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -81,46 +81,48 @@ private async Task UpdateTaskAsync()
8181
}
8282
catch (UnsuccessfulResponseException ex)
8383
{
84-
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode);
84+
var recoverable = HttpErrors.IsRecoverable(ex.StatusCode);
85+
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode, recoverable);
8586

86-
if (HttpErrors.IsRecoverable(ex.StatusCode))
87+
if (errorInfo.Recoverable)
8788
{
8889
_log.Warn(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry"));
8990
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
9091
}
9192
else
9293
{
9394
_log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", ""));
94-
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
9595
try
9696
{
9797
// if client is initializing, make it stop waiting
98-
_initTask.SetResult(true);
98+
_initTask.SetResult(false);
9999
}
100100
catch (InvalidOperationException)
101101
{
102102
// the task was already set - nothing more to do
103103
}
104-
((IDisposable)this).Dispose();
104+
Shutdown(errorInfo);
105105
}
106106
}
107107
catch (JsonException ex)
108108
{
109109
_log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex));
110-
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
111-
new DataSourceStatus.ErrorInfo
112-
{
113-
Kind = DataSourceStatus.ErrorKind.InvalidData,
114-
Time = DateTime.Now
115-
});
110+
var errorInfo = new DataSourceStatus.ErrorInfo
111+
{
112+
Kind = DataSourceStatus.ErrorKind.InvalidData,
113+
Message = ex.Message,
114+
Time = DateTime.Now,
115+
Recoverable = true
116+
};
117+
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
116118
}
117119
catch (Exception ex)
118120
{
119121
Exception realEx = (ex is AggregateException ae) ? ae.Flatten() : ex;
120122
_log.Warn("Polling for feature flag updates failed: {0}", LogValues.ExceptionSummary(ex));
121123
_log.Debug(LogValues.ExceptionTrace(ex));
122-
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
123-
DataSourceStatus.ErrorInfo.FromException(realEx));
124+
var errorInfo = DataSourceStatus.ErrorInfo.FromException(realEx, true); // default to recoverable
125+
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
124126
}
125127
}
126128

@@ -134,11 +136,17 @@ private void Dispose(bool disposing)
134136
{
135137
if (disposing)
136138
{
137-
_canceller?.Cancel();
138-
_featureRequestor.Dispose();
139+
Shutdown(null);
139140
}
140141
}
141142

143+
private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo)
144+
{
145+
_canceller?.Cancel();
146+
_featureRequestor.Dispose();
147+
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
148+
}
149+
142150
private bool InitWithHeaders(DataStoreTypes.FullDataSet<DataStoreTypes.ItemDescriptor> allData,
143151
IEnumerable<KeyValuePair<string, IEnumerable<string>>> headers)
144152
{

‎pkgs/sdk/server/src/Internal/DataSources/StreamingDataSource.cs‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -112,14 +112,20 @@ private void Dispose(bool disposing)
112112
{
113113
if (disposing)
114114
{
115-
_es.Close();
116-
if (_storeStatusMonitoringEnabled)
117-
{
118-
_dataSourceUpdates.DataStoreStatusProvider.StatusChanged -= OnDataStoreStatusChanged;
119-
}
115+
Shutdown(null);
120116
}
121117
}
122118

119+
private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo)
120+
{
121+
_es.Close();
122+
if (_storeStatusMonitoringEnabled)
123+
{
124+
_dataSourceUpdates.DataStoreStatusProvider.StatusChanged -= OnDataStoreStatusChanged;
125+
}
126+
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
127+
}
128+
123129
#endregion
124130

125131
private IEventSource CreateEventSource(Uri uri, HttpConfiguration httpConfig)
@@ -175,7 +181,8 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e)
175181
{
176182
Kind = DataSourceStatus.ErrorKind.InvalidData,
177183
Message = ex.Message,
178-
Time = DateTime.Now
184+
Time = DateTime.Now,
185+
Recoverable = true
179186
};
180187
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
181188

@@ -187,7 +194,8 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e)
187194
{
188195
Kind = DataSourceStatus.ErrorKind.StoreError,
189196
Message = (ex.InnerException ?? ex).Message,
190-
Time = DateTime.Now
197+
Time = DateTime.Now,
198+
Recoverable = true
191199
};
192200
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
193201
if (!_storeStatusMonitoringEnabled)
@@ -210,17 +218,16 @@ private void OnMessage(object sender, EventSource.MessageReceivedEventArgs e)
210218
private void OnError(object sender, EventSource.ExceptionEventArgs e)
211219
{
212220
var ex = e.Exception;
213-
var recoverable = true;
214221
DataSourceStatus.ErrorInfo errorInfo;
215222

216223
if (ex is EventSourceServiceUnsuccessfulResponseException respEx)
217224
{
218225
int status = respEx.StatusCode;
219-
errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status);
226+
var recoverable = HttpErrors.IsRecoverable(status);
227+
errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(status, recoverable);
220228
RecordStreamInit(true);
221-
if (!HttpErrors.IsRecoverable(status))
229+
if (!recoverable)
222230
{
223-
recoverable = false;
224231
_log.Error(HttpErrors.ErrorMessage(status, "streaming connection", ""));
225232
}
226233
else
@@ -230,21 +237,23 @@ private void OnError(object sender, EventSource.ExceptionEventArgs e)
230237
}
231238
else
232239
{
233-
errorInfo = DataSourceStatus.ErrorInfo.FromException(ex);
240+
errorInfo = DataSourceStatus.ErrorInfo.FromException(ex, true); // default to recoverable
234241
_log.Warn("Encountered EventSource error: {0}", LogValues.ExceptionSummary(ex));
235242
_log.Debug(LogValues.ExceptionTrace(ex));
236243
}
237244

238-
_dataSourceUpdates.UpdateStatus(recoverable ? DataSourceState.Interrupted : DataSourceState.Off,
239-
errorInfo);
240-
241-
if (!recoverable)
245+
if (errorInfo.Recoverable)
246+
{
247+
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
248+
return;
249+
}
250+
else
242251
{
243252
// Make _initTask complete to tell the client to stop waiting for initialization. We use
244253
// TrySetResult rather than SetResult here because it might have already been completed
245254
// (if for instance the stream started successfully, then restarted and got a 401).
246255
_initTask.TrySetResult(false);
247-
((IDisposable)this).Dispose();
256+
Shutdown(errorInfo);
248257
}
249258
}
250259

‎pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2DataSource.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,11 @@ public void UpdateStatus(DataSourceState newState, DataSourceStatus.ErrorInfo? n
169169
// When a synchronizer reports it is off, fall back immediately
170170
if (newState == DataSourceState.Off)
171171
{
172+
if (newError != null && !newError.Value.Recoverable)
173+
{
174+
_actionable.BlacklistCurrent();
175+
}
176+
172177
_actionable.DisposeCurrent();
173178
_actionable.GoToNext();
174179
_actionable.StartCurrent();

‎pkgs/sdk/server/src/Internal/FDv2DataSources/FDv2PollingDataSource.cs‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ private async Task UpdateTaskAsync()
9595
}
9696
catch (UnsuccessfulResponseException ex)
9797
{
98-
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode);
98+
var recoverable = HttpErrors.IsRecoverable(ex.StatusCode);
99+
var errorInfo = DataSourceStatus.ErrorInfo.FromHttpError(ex.StatusCode, recoverable);
99100

100101
// Check for LD fallback header
101102
if (ex.Headers != null)
@@ -106,44 +107,45 @@ private async Task UpdateTaskAsync()
106107
.Any(v => string.Equals(v, "true", StringComparison.OrdinalIgnoreCase));
107108
}
108109

109-
if (HttpErrors.IsRecoverable(ex.StatusCode))
110+
if (errorInfo.Recoverable)
110111
{
111112
_log.Warn(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", "will retry"));
112113
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
113114
}
114115
else
115116
{
116117
_log.Error(HttpErrors.ErrorMessage(ex.StatusCode, "polling request", ""));
117-
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
118118
try
119119
{
120-
_initTask.SetResult(true);
120+
_initTask.SetResult(false);
121121
}
122122
catch (InvalidOperationException)
123123
{
124124
// the task was already set - nothing more to do
125125
}
126126

127-
((IDisposable)this).Dispose();
127+
Shutdown(errorInfo);
128128
}
129129
}
130130
catch (JsonException ex)
131131
{
132132
_log.Error("Polling request received malformed data: {0}", LogValues.ExceptionSummary(ex));
133-
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
134-
new DataSourceStatus.ErrorInfo
135-
{
136-
Kind = DataSourceStatus.ErrorKind.InvalidData,
137-
Time = DateTime.Now
138-
});
133+
var errorInfo = new DataSourceStatus.ErrorInfo
134+
{
135+
Kind = DataSourceStatus.ErrorKind.InvalidData,
136+
Message = ex.Message,
137+
Time = DateTime.Now,
138+
Recoverable = true
139+
};
140+
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
139141
}
140142
catch (Exception ex)
141143
{
142144
var realEx = (ex is AggregateException ae) ? ae.Flatten() : ex;
143145
_log.Warn("Polling for feature flag updates failed: {0}", LogValues.ExceptionSummary(ex));
144146
_log.Debug(LogValues.ExceptionTrace(ex));
145-
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted,
146-
DataSourceStatus.ErrorInfo.FromException(realEx));
147+
var errorInfo = DataSourceStatus.ErrorInfo.FromException(realEx, true); // default to recoverable
148+
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
147149
}
148150
}
149151

@@ -169,7 +171,8 @@ private void HandleJsonError(string message)
169171
{
170172
Kind = DataSourceStatus.ErrorKind.InvalidData,
171173
Message = message,
172-
Time = DateTime.Now
174+
Time = DateTime.Now,
175+
Recoverable = true
173176
};
174177
_dataSourceUpdates.UpdateStatus(DataSourceState.Interrupted, errorInfo);
175178
}
@@ -235,13 +238,14 @@ private void Dispose(bool disposing)
235238
{
236239
if (!disposing) return;
237240

238-
Shutdown();
241+
Shutdown(null);
239242
}
240243

241-
private void Shutdown()
244+
private void Shutdown(DataSourceStatus.ErrorInfo? errorInfo)
242245
{
243246
_canceler?.Cancel();
244247
_requestor.Dispose();
248+
_dataSourceUpdates.UpdateStatus(DataSourceState.Off, errorInfo);
245249
}
246250
}
247251
}

0 commit comments

Comments
 (0)