Skip to content

Commit c5122b7

Browse files
jagilberjagilber
authored
Merge pull request #158 from jagilber/jagilber
fix configuration class inconsistencies in base classes
2 parents 9c82929 + a765064 commit c5122b7

File tree

20 files changed

+192
-133
lines changed

20 files changed

+192
-133
lines changed

‎.vscode/launch.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
55
"version": "0.2.0",
66
"configurations": [
7+
78
{
89
"name": "Launch and Debug hosted blazor web assembly",
910
"request": "launch",
@@ -65,6 +66,21 @@
6566
"stopAtEntry": false,
6667
"console": "internalConsole",
6768
},
69+
{
70+
"name": ".NET Core 60 Launch with Args (console)",
71+
"type": "coreclr",
72+
"request": "launch",
73+
"preLaunchTask": "build",
74+
"program": "${workspaceFolder}/src/bin/Debug/net6.0/CollectSFData.dll",
75+
"args": [
76+
//"${input:args}"
77+
//"-config",
78+
//".\\local.json"
79+
],
80+
"cwd": "${workspaceFolder}/src/bin/Debug/net6.0",
81+
"stopAtEntry": false,
82+
"console": "internalConsole",
83+
},
6884
{
6985
"name": "NET 6 Launch (console)",
7086
"type": "coreclr",

‎.vscode/settings.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
{
2-
"dotnet-test-explorer.testProjectPath": "**/*Test.csproj"
2+
"dotnet-test-explorer.testProjectPath": "**/*Test.csproj",
3+
"audioCues.volume": 0
34
}

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
11
# Change log
22

3+
## 02/05/2023
4+
5+
- fix configuration class inconsistencies in base classes
6+
- remove System.Diagnostic.DiagnosticSource from GetManagedIdentity
7+
- add functionality to remove failed and succeeded kusto ingestion queue messages if over 1 day old to prevent having to continually enumerate and process all stale messages.
8+
- fix hang on trace retry for failed csv traces
9+
10+
## 12/21/2022
11+
12+
- add additional counters to no progress check / timeout
13+
- change noisy debug trace to trivial
14+
315
## 12/6/2022
416

517
- UploadCacheData() set file status from formatting to succeeded if not uploading to kusto or log analytics

‎src/CollectSFData/CollectSFData.csproj‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,6 @@
127127
<PackageReference Include="System.Data.SqlClient">
128128
<Version>4.8.2</Version>
129129
</PackageReference>
130-
<PackageReference Include="System.Diagnostics.DiagnosticSource">
131-
<Version>4.6.0</Version>
132-
</PackageReference>
133130
<PackageReference Include="System.Reactive">
134131
<Version>4.0.0</Version>
135132
</PackageReference>

‎src/CollectSFData/CollectSFData.nuspec‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
<dependency id="Newtonsoft.Json" version="13.0.1" exclude="Build,Analyzers" />
4040
<dependency id="System.CodeDom" version="4.7.0" exclude="Build,Analyzers" />
4141
<dependency id="System.Data.SqlClient" version="4.8.2" exclude="Build,Analyzers" />
42-
<dependency id="System.Diagnostics.DiagnosticSource" version="4.6.0" exclude="Build,Analyzers" />
4342
<dependency id="System.Reactive" version="4.0.0" exclude="Build,Analyzers" />
4443
<dependency id="System.Security.Cryptography.Cng" version="4.7.0" exclude="Build,Analyzers" />
4544
<dependency id="System.Security.Cryptography.ProtectedData" version="4.7.0" exclude="Build,Analyzers" />
@@ -64,7 +63,6 @@
6463
<dependency id="Newtonsoft.Json" version="13.0.1" exclude="Build,Analyzers" />
6564
<dependency id="System.CodeDom" version="4.7.0" exclude="Build,Analyzers" />
6665
<dependency id="System.Data.SqlClient" version="4.8.2" exclude="Build,Analyzers" />
67-
<dependency id="System.Diagnostics.DiagnosticSource" version="4.6.0" exclude="Build,Analyzers" />
6866
<dependency id="System.Reactive" version="4.0.0" exclude="Build,Analyzers" />
6967
<dependency id="System.Security.Cryptography.Cng" version="4.7.0" exclude="Build,Analyzers" />
7068
<dependency id="System.Security.Cryptography.ProtectedData" version="4.7.0" exclude="Build,Analyzers" />
@@ -89,7 +87,6 @@
8987
<dependency id="Newtonsoft.Json" version="13.0.1" exclude="Build,Analyzers" />
9088
<dependency id="System.CodeDom" version="4.7.0" exclude="Build,Analyzers" />
9189
<dependency id="System.Data.SqlClient" version="4.8.2" exclude="Build,Analyzers" />
92-
<dependency id="System.Diagnostics.DiagnosticSource" version="4.6.0" exclude="Build,Analyzers" />
9390
<dependency id="System.Reactive" version="4.0.0" exclude="Build,Analyzers" />
9491
<dependency id="System.Security.Cryptography.Cng" version="4.7.0" exclude="Build,Analyzers" />
9592
<dependency id="System.Security.Cryptography.ProtectedData" version="4.7.0" exclude="Build,Analyzers" />
@@ -113,7 +110,6 @@
113110
<dependency id="Newtonsoft.Json" version="13.0.1" exclude="Build,Analyzers" />
114111
<dependency id="System.CodeDom" version="4.7.0" exclude="Build,Analyzers" />
115112
<dependency id="System.Data.SqlClient" version="4.8.2" exclude="Build,Analyzers" />
116-
<dependency id="System.Diagnostics.DiagnosticSource" version="4.6.0" exclude="Build,Analyzers" />
117113
<dependency id="System.Reactive" version="4.0.0" exclude="Build,Analyzers" />
118114
<dependency id="System.Security.Cryptography.Cng" version="4.7.0" exclude="Build,Analyzers" />
119115
<dependency id="System.Security.Cryptography.ProtectedData" version="4.7.0" exclude="Build,Analyzers" />

‎src/CollectSFDataDll/Azure/ClientIdentity.cs‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -69,22 +69,7 @@ private bool GetManagedIdentity(string managedClientId)
6969
bool retval = false;
7070
try
7171
{
72-
ManagedIdentityCredential managedCredential = new ManagedIdentityCredential(managedClientId, new TokenCredentialOptions
73-
{
74-
Diagnostics = {
75-
ApplicationId = Constants.ApplicationName,
76-
IsDistributedTracingEnabled = true,
77-
IsLoggingContentEnabled = true,
78-
IsLoggingEnabled = true,
79-
LoggedHeaderNames = {
80-
"x-ms-request-id"
81-
},
82-
LoggedQueryParameters = {
83-
"api-version"
84-
}
85-
}
86-
});
87-
72+
ManagedIdentityCredential managedCredential = new ManagedIdentityCredential(managedClientId, new TokenCredentialOptions());
8873
ManagedIdentityToken = managedCredential.GetTokenAsync(new TokenRequestContext(new string[1] { $"{Constants.ManagementAzureCom}/.default" })).Result;
8974

9075
retval = true;

‎src/CollectSFDataDll/CollectSFDataDll.csproj‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,6 @@
8888
<PackageReference Include="System.Data.SqlClient">
8989
<Version>4.8.2</Version>
9090
</PackageReference>
91-
<PackageReference Include="System.Diagnostics.DiagnosticSource">
92-
<Version>4.6.0</Version>
93-
</PackageReference>
9491
<PackageReference Include="System.Reactive">
9592
<Version>4.0.0</Version>
9693
</PackageReference>

‎src/CollectSFDataDll/Collector.cs‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public class Collector
2323
private int _noProgressCounter = 0;
2424
private Timer _noProgressTimer;
2525
private ParallelOptions _parallelConfig;
26-
private Tuple<int, int, int, int, int, int, int> _progressTuple = new Tuple<int, int, int, int, int, int, int>(0, 0, 0, 0, 0, 0, 0);
26+
private Total _progressTotal = new Total();
2727

2828
public ConfigurationOptions Config { get => Instance.Config; }
2929

@@ -43,7 +43,7 @@ public void Close()
4343

4444
public int Collect()
4545
{
46-
return Collect(new ConfigurationOptions());
46+
return Collect(Config);
4747
}
4848

4949
public int Collect(ConfigurationOptions configurationOptions)
@@ -274,6 +274,11 @@ private void LogSummary()
274274
Log.Last($"timed out: {Instance.TimedOut}.");
275275
Log.Last($"{Instance.FileObjects.StatusString()}", ConsoleColor.Cyan);
276276

277+
if (Config.List)
278+
{
279+
Log.Last($"file list:", jsonSerializer: Instance.FileObjects.Select(x => x.FileUri));
280+
}
281+
277282
if (Instance.TotalFilesEnumerated > 0)
278283
{
279284
if (Config.FileType != FileTypesEnum.table)
@@ -326,16 +331,10 @@ private void NoProgressCallback(object state)
326331
return;
327332
}
328333

329-
Tuple<int, int, int, int, int, int, int> tuple = new Tuple<int, int, int, int, int, int, int>(
330-
Instance.TotalErrors,
331-
Instance.TotalFilesDownloaded,
332-
Instance.TotalFilesEnumerated,
333-
Instance.TotalFilesFormatted,
334-
Instance.TotalFilesMatched,
335-
Instance.TotalFilesSkipped,
336-
Instance.TotalRecords);
334+
Total total = Instance.Totals();
335+
Log.Highlight($"totals:", total);
337336

338-
if (tuple.Equals(_progressTuple))
337+
if (total.Equals(_progressTotal))
339338
{
340339
if (_noProgressCounter >= Config.NoProgressTimeoutMin)
341340
{
@@ -359,7 +358,7 @@ private void NoProgressCallback(object state)
359358
else
360359
{
361360
_noProgressCounter = 0;
362-
_progressTuple = tuple;
361+
_progressTotal = total;
363362
}
364363
}
365364

@@ -427,7 +426,16 @@ private void UploadCacheData()
427426
}
428427
else
429428
{
430-
Log.Error($"configuration set to upload cache files from 'fileUris' count:{Config.FileUris.Length} but no files found");
429+
string logString = $"configuration set to upload cache files from 'fileUris' count:{Config.FileUris.Length} but no files found";
430+
431+
if(Config.SasEndpointInfo.IsPopulated())
432+
{
433+
Log.Warning(logString);
434+
}
435+
else
436+
{
437+
Log.Error(logString);
438+
}
431439
}
432440
}
433441
else if (Config.IsCacheLocationPreConfigured())

‎src/CollectSFDataDll/Common/ConfigurationOptions.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,21 @@ public class ConfigurationOptions : ConfigurationProperties
2929
private static bool? _cacheLocationPreconfigured = null;
3030
private static string[] _commandlineArguments = new string[0];
3131
private static ConfigurationOptions _defaultConfig;
32+
private static ConfigurationOptions _singleton;// = new ConfigurationOptions();
3233
private readonly string _tempName = "csfd";
3334
private string _tempPath;
35+
private static object _singleLock = new Object();
36+
37+
public static ConfigurationOptions Singleton()
38+
{
39+
lock (_singleLock)
40+
{
41+
if(_singleton == null) {
42+
_singleton = new ConfigurationOptions();
43+
}
44+
return _singleton;
45+
}
46+
}
3447

3548
public X509Certificate2 ClientCertificate { get; set; }
3649

@@ -1157,6 +1170,7 @@ private bool ProcessArguments()
11571170
if (!_commandlineArguments[0].StartsWith("/?") && !_commandlineArguments[0].StartsWith("-") && _commandlineArguments[0].EndsWith(".json") && File.Exists(_commandlineArguments[0]))
11581171
{
11591172
ConfigurationFile = _commandlineArguments[0];
1173+
_commandlineArguments = new string[0];
11601174
MergeConfig(ConfigurationFile);
11611175
Log.Info($"setting options to {Constants.DefaultOptionsFile}", ConsoleColor.Yellow);
11621176
}

‎src/CollectSFDataDll/Common/CustomTaskManager.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ namespace CollectSFData.Common
1414
public class CustomTaskManager
1515
{
1616
private static SynchronizedList<CustomTaskManager> _allInstances = new SynchronizedList<CustomTaskManager>();
17-
private static ConfigurationOptions _config = new ConfigurationOptions();
17+
private static ConfigurationOptions _config = ConfigurationOptions.Singleton();
1818
private static CustomTaskScheduler _customScheduler;
1919
private static Instance _instance;
2020
private static Task _taskMonitor = new Task(TaskMonitor);
@@ -246,9 +246,10 @@ private Task AddToQueue(TaskObject taskObject, bool taskWait = false)
246246
Log.Info($"added new taskobject to queue:{CallerName} total queued:{QueuedTaskObjects.Count()} throttle ms:{count * 10}");
247247
TimeSpan delay = new TimeSpan();
248248

249-
if (taskWait)
249+
while (taskWait && !CancellationTokenSource.IsCancellationRequested)
250250
{
251-
taskObject.TaskScheduled.WaitOne();
251+
// do not log in this loop as logging will be disabled on close
252+
taskWait = !taskObject.TaskScheduled.WaitOne(Constants.ThreadSleepMs1000);
252253
}
253254

254255
Log.Debug($"added new taskobject to queue:{CallerName} total queued:{QueuedTaskObjects.Count()} delay:{delay.TotalMilliseconds}ms");

0 commit comments

Comments
 (0)