From 3d0d3559db0fa516d15340d4019f78cd116384d8 Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 18 Sep 2025 15:35:45 +1000 Subject: [PATCH 1/5] Increasing metrics days collected for Azure --- .../Commands/AzureServiceBusCommand.cs | 27 ++++++++++++++----- src/Query/AzureServiceBus/AzureClient.cs | 6 ++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/AppCommon/Commands/AzureServiceBusCommand.cs b/src/AppCommon/Commands/AzureServiceBusCommand.cs index ce37ae73..1994f557 100644 --- a/src/AppCommon/Commands/AzureServiceBusCommand.cs +++ b/src/AppCommon/Commands/AzureServiceBusCommand.cs @@ -66,8 +66,8 @@ protected override async Task GetData(CancellationToken cancellati { try { - var endTime = DateTime.UtcNow.Date.AddDays(1); - var startTime = endTime.AddDays(-30); + var endTime = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(-1); + var startTime = endTime.AddDays(-90); var results = new List(); azure.ResetConnectionQueue(); @@ -85,10 +85,25 @@ protected override async Task GetData(CancellationToken cancellati { var maxThroughput = metricValues.Select(timeEntry => timeEntry.Total).Max(); - // Since we get 30 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint + // Since we get 90 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput is not null and not 0) { - results.Add(new QueueThroughput { QueueName = queueName, Throughput = (long?)maxThroughput }); + DateOnly currentDate = startTime; + var data = new Dictionary(); + while (currentDate <= endTime) + { + data.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate }); + + currentDate = currentDate.AddDays(1); + } + + foreach (var metricValue in metricValues) + { + currentDate = DateOnly.FromDateTime(metricValue.TimeStamp.UtcDateTime); + data[currentDate] = new DailyThroughput { MessageCount = (long)(metricValue.Total ?? 0), DateUTC = currentDate }; + } + + results.Add(new QueueThroughput { QueueName = queueName, Throughput = (long?)maxThroughput, DailyThroughputFromBroker = [.. data.Values] }); } else { @@ -99,8 +114,8 @@ protected override async Task GetData(CancellationToken cancellati return new QueueDetails { - StartTime = new DateTimeOffset(startTime, TimeSpan.Zero), - EndTime = new DateTimeOffset(endTime, TimeSpan.Zero), + StartTime = new DateTimeOffset(startTime, TimeOnly.MinValue, TimeSpan.Zero), + EndTime = new DateTimeOffset(endTime, TimeOnly.MaxValue, TimeSpan.Zero), Queues = results.OrderBy(q => q.QueueName).ToArray(), TimeOfObservation = TimeSpan.FromDays(1) }; diff --git a/src/Query/AzureServiceBus/AzureClient.cs b/src/Query/AzureServiceBus/AzureClient.cs index d313269c..1f9fce4e 100644 --- a/src/Query/AzureServiceBus/AzureClient.cs +++ b/src/Query/AzureServiceBus/AzureClient.cs @@ -116,18 +116,18 @@ bool NextCredentials() } } - public Task> GetMetrics(string queueName, DateTime startTime, DateTime endTime, CancellationToken cancellationToken = default) + public Task> GetMetrics(string queueName, DateOnly startTime, DateOnly endTime, CancellationToken cancellationToken = default) { return GetDataWithCurrentCredentials(async token => { try { var response = await currentClients.Metrics.QueryResourceAsync(resourceId, - new[] { "CompleteMessage" }, + ["CompleteMessage"], new MetricsQueryOptions { Filter = $"EntityName eq '{queueName}'", - TimeRange = new QueryTimeRange(startTime, endTime), + TimeRange = new QueryTimeRange(startTime.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), endTime.ToDateTime(TimeOnly.MaxValue, DateTimeKind.Utc)), Granularity = TimeSpan.FromDays(1) }, token).ConfigureAwait(false); From 4cb57c9eb4f7888b9e46b162b631bed404f630db Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 18 Sep 2025 16:51:43 +1000 Subject: [PATCH 2/5] Fixed message with new 90 days --- src/AppCommon/Commands/AzureServiceBusCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/AppCommon/Commands/AzureServiceBusCommand.cs b/src/AppCommon/Commands/AzureServiceBusCommand.cs index 1994f557..ecdf556f 100644 --- a/src/AppCommon/Commands/AzureServiceBusCommand.cs +++ b/src/AppCommon/Commands/AzureServiceBusCommand.cs @@ -107,7 +107,7 @@ protected override async Task GetData(CancellationToken cancellati } else { - Out.WriteLine(" - No throughput detected in 30 days, ignoring"); + Out.WriteLine(" - No throughput detected in 90 days, ignoring"); } } } From 8245503a92bf49fd99023c9c94773bba8396292d Mon Sep 17 00:00:00 2001 From: John Simons Date: Thu, 18 Sep 2025 16:52:02 +1000 Subject: [PATCH 3/5] Update SQS to collect 365 days of metrics --- src/AppCommon/Commands/SqsCommand.cs | 30 +++++++++++++++++++++++++--- src/Query/AmazonSQS/AwsQuery.cs | 7 +++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/AppCommon/Commands/SqsCommand.cs b/src/AppCommon/Commands/SqsCommand.cs index 118ed765..13f7d323 100644 --- a/src/AppCommon/Commands/SqsCommand.cs +++ b/src/AppCommon/Commands/SqsCommand.cs @@ -74,15 +74,39 @@ protected override async Task GetData(CancellationToken cancellati var tasks = queueNames.Select(async queueName => { - var maxThroughput = await aws.GetMaxThroughput(queueName, cancellationToken).ConfigureAwait(false); + var datapoints = await aws.GetMMetricsData(queueName, cancellationToken).ConfigureAwait(false); - // Since we get 30 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint + var maxThroughput = datapoints is { Count: > 0 } ? + (long)datapoints.Select(d => d.Sum.GetValueOrDefault(0)).Max() : 0L; + // Since we get 365 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput > 0) { + DateOnly currentDate = aws.StartDate; + var dailyData = new Dictionary(); + while (currentDate <= aws.EndDate) + { + dailyData.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate }); + + currentDate = currentDate.AddDays(1); + } + + foreach (var datapoint in datapoints) + { + // There is a bug in the AWS SDK. The timestamp is actually UTC time, eventhough the DateTime returned type says Local + // See https://github.com/aws/aws-sdk-net/issues/167 + // So do not convert the timestamp to UTC time! + if (datapoint.Timestamp.HasValue) + { + currentDate = DateOnly.FromDateTime(datapoint.Timestamp.Value); + dailyData[currentDate] = new DailyThroughput { MessageCount = (long)datapoint.Sum.GetValueOrDefault(0), DateUTC = currentDate }; + } + } + data.Add(new QueueThroughput { QueueName = queueName, - Throughput = maxThroughput + Throughput = maxThroughput, + DailyThroughputFromBroker = [.. dailyData.Values] }); } diff --git a/src/Query/AmazonSQS/AwsQuery.cs b/src/Query/AmazonSQS/AwsQuery.cs index d478c1f9..4bfaded8 100644 --- a/src/Query/AmazonSQS/AwsQuery.cs +++ b/src/Query/AmazonSQS/AwsQuery.cs @@ -35,7 +35,7 @@ public AwsQuery() QueueLimit = int.MaxValue }); EndDate = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(1); - StartDate = EndDate.AddDays(-30); + StartDate = EndDate.AddDays(-365); sqs = new AmazonSQSClient(); cloudWatch = new AmazonCloudWatchClient(); @@ -83,7 +83,7 @@ public async Task> GetQueueNames(Action onProgress, Cancellati } } - public async Task GetMaxThroughput(string queueName, CancellationToken cancellationToken = default) + public async Task> GetMMetricsData(string queueName, CancellationToken cancellationToken = default) { var req = new GetMetricStatisticsRequest { @@ -100,8 +100,7 @@ public async Task GetMaxThroughput(string queueName, CancellationToken can using var lease = await rateLimiter.AcquireAsync(cancellationToken: cancellationToken).ConfigureAwait(false); var resp = await cloudWatch.GetMetricStatisticsAsync(req, cancellationToken).ConfigureAwait(false); - return resp.Datapoints is { Count: > 0 } ? - (long)resp.Datapoints.Select(d => d.Sum.GetValueOrDefault(0)).Max() : 0L; + return resp.Datapoints ?? []; } } } From c62172ba0418956f5394924ad8d9c2a087fe0d88 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 1 Oct 2025 12:47:42 +1000 Subject: [PATCH 4/5] Include report duration --- src/AppCommon/Commands/AzureServiceBusCommand.cs | 8 +++++--- src/AppCommon/Commands/BaseCommand.cs | 2 +- src/AppCommon/Commands/SqsCommand.cs | 8 +++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/AppCommon/Commands/AzureServiceBusCommand.cs b/src/AppCommon/Commands/AzureServiceBusCommand.cs index ecdf556f..69ef46a3 100644 --- a/src/AppCommon/Commands/AzureServiceBusCommand.cs +++ b/src/AppCommon/Commands/AzureServiceBusCommand.cs @@ -112,12 +112,14 @@ protected override async Task GetData(CancellationToken cancellati } } + var s = new DateTimeOffset(startTime, TimeOnly.MinValue, TimeSpan.Zero); + var e = new DateTimeOffset(endTime, TimeOnly.MaxValue, TimeSpan.Zero); return new QueueDetails { - StartTime = new DateTimeOffset(startTime, TimeOnly.MinValue, TimeSpan.Zero), - EndTime = new DateTimeOffset(endTime, TimeOnly.MaxValue, TimeSpan.Zero), + StartTime = s, + EndTime = e, Queues = results.OrderBy(q => q.QueueName).ToArray(), - TimeOfObservation = TimeSpan.FromDays(1) + TimeOfObservation = e - s }; } catch (QueryException x) diff --git a/src/AppCommon/Commands/BaseCommand.cs b/src/AppCommon/Commands/BaseCommand.cs index 49f13cba..a283c141 100644 --- a/src/AppCommon/Commands/BaseCommand.cs +++ b/src/AppCommon/Commands/BaseCommand.cs @@ -224,7 +224,7 @@ async Task RunInternal(CancellationToken cancellationToken) ScopeType = data.ScopeType, StartTime = data.StartTime, EndTime = data.EndTime, - ReportDuration = data.TimeOfObservation ?? data.EndTime - data.StartTime, + ReportDuration = data.TimeOfObservation ?? (data.EndTime - data.StartTime), Queues = data.Queues, TotalThroughput = data.Queues.Sum(q => q.Throughput ?? 0), TotalQueues = data.Queues.Length, diff --git a/src/AppCommon/Commands/SqsCommand.cs b/src/AppCommon/Commands/SqsCommand.cs index 13f7d323..840a48d5 100644 --- a/src/AppCommon/Commands/SqsCommand.cs +++ b/src/AppCommon/Commands/SqsCommand.cs @@ -118,12 +118,14 @@ protected override async Task GetData(CancellationToken cancellati Out.EndProgress(); + var s = new DateTimeOffset(aws.StartDate.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + var e = new DateTimeOffset(aws.EndDate.ToDateTime(TimeOnly.MaxValue), TimeSpan.Zero); return new QueueDetails { - StartTime = new DateTimeOffset(aws.StartDate.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), - EndTime = new DateTimeOffset(aws.EndDate.ToDateTime(TimeOnly.MaxValue), TimeSpan.Zero), + StartTime = s, + EndTime = e, Queues = [.. data.OrderBy(q => q.QueueName)], - TimeOfObservation = TimeSpan.FromDays(1) + TimeOfObservation = e - s }; } From bcb04fb0250026943465021540c900ee1ad3112b Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 3 Oct 2025 11:54:06 +1000 Subject: [PATCH 5/5] Reduce the size of the data being sent per queue --- src/AppCommon/Commands/AzureServiceBusCommand.cs | 8 +++++--- src/AppCommon/Commands/SqsCommand.cs | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/AppCommon/Commands/AzureServiceBusCommand.cs b/src/AppCommon/Commands/AzureServiceBusCommand.cs index 69ef46a3..19a87865 100644 --- a/src/AppCommon/Commands/AzureServiceBusCommand.cs +++ b/src/AppCommon/Commands/AzureServiceBusCommand.cs @@ -79,7 +79,7 @@ protected override async Task GetData(CancellationToken cancellati Out.WriteLine($"Gathering metrics for queue {i + 1}/{queueNames.Length}: {queueName}"); - var metricValues = await azure.GetMetrics(queueName, startTime, endTime, cancellationToken); + var metricValues = (await azure.GetMetrics(queueName, startTime, endTime, cancellationToken)).OrderBy(m => m.TimeStamp).ToArray(); if (metricValues is not null) { @@ -88,9 +88,11 @@ protected override async Task GetData(CancellationToken cancellati // Since we get 90 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput is not null and not 0) { - DateOnly currentDate = startTime; + var start = DateOnly.FromDateTime(metricValues.First().TimeStamp.UtcDateTime); + var end = DateOnly.FromDateTime(metricValues.Last().TimeStamp.UtcDateTime); + var currentDate = start; var data = new Dictionary(); - while (currentDate <= endTime) + while (currentDate <= end) { data.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate }); diff --git a/src/AppCommon/Commands/SqsCommand.cs b/src/AppCommon/Commands/SqsCommand.cs index 840a48d5..9bd76a9e 100644 --- a/src/AppCommon/Commands/SqsCommand.cs +++ b/src/AppCommon/Commands/SqsCommand.cs @@ -74,16 +74,18 @@ protected override async Task GetData(CancellationToken cancellati var tasks = queueNames.Select(async queueName => { - var datapoints = await aws.GetMMetricsData(queueName, cancellationToken).ConfigureAwait(false); + var datapoints = (await aws.GetMMetricsData(queueName, cancellationToken)).OrderBy(d => d.Timestamp).ToArray(); - var maxThroughput = datapoints is { Count: > 0 } ? + var maxThroughput = datapoints is { Length: > 0 } ? (long)datapoints.Select(d => d.Sum.GetValueOrDefault(0)).Max() : 0L; // Since we get 365 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput > 0) { - DateOnly currentDate = aws.StartDate; + var startTime = DateOnly.FromDateTime(datapoints.First().Timestamp.Value); + var endTime = DateOnly.FromDateTime(datapoints.Last().Timestamp.Value); + DateOnly currentDate = startTime; var dailyData = new Dictionary(); - while (currentDate <= aws.EndDate) + while (currentDate <= endTime) { dailyData.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate });