diff --git a/src/AppCommon/Commands/AzureServiceBusCommand.cs b/src/AppCommon/Commands/AzureServiceBusCommand.cs index ce37ae73..19a87865 100644 --- a/src/AppCommon/Commands/AzureServiceBusCommand.cs +++ b/src/AppCommon/Commands/AzureServiceBusCommand.cs @@ -66,8 +66,8 @@ protected override async Task GetData(CancellationToken cancellati { try { - var endTime = DateTime.UtcNow.Date.AddDays(1); - var startTime = endTime.AddDays(-30); + var endTime = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(-1); + var startTime = endTime.AddDays(-90); var results = new List(); azure.ResetConnectionQueue(); @@ -79,30 +79,49 @@ protected override async Task GetData(CancellationToken cancellati Out.WriteLine($"Gathering metrics for queue {i + 1}/{queueNames.Length}: {queueName}"); - var metricValues = await azure.GetMetrics(queueName, startTime, endTime, cancellationToken); + var metricValues = (await azure.GetMetrics(queueName, startTime, endTime, cancellationToken)).OrderBy(m => m.TimeStamp).ToArray(); if (metricValues is not null) { var maxThroughput = metricValues.Select(timeEntry => timeEntry.Total).Max(); - // Since we get 30 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint + // Since we get 90 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput is not null and not 0) { - results.Add(new QueueThroughput { QueueName = queueName, Throughput = (long?)maxThroughput }); + var start = DateOnly.FromDateTime(metricValues.First().TimeStamp.UtcDateTime); + var end = DateOnly.FromDateTime(metricValues.Last().TimeStamp.UtcDateTime); + var currentDate = start; + var data = new Dictionary(); + while (currentDate <= end) + { + data.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate }); + + currentDate = currentDate.AddDays(1); + } + + foreach (var metricValue in metricValues) + { + currentDate = DateOnly.FromDateTime(metricValue.TimeStamp.UtcDateTime); + data[currentDate] = new DailyThroughput { MessageCount = (long)(metricValue.Total ?? 0), DateUTC = currentDate }; + } + + results.Add(new QueueThroughput { QueueName = queueName, Throughput = (long?)maxThroughput, DailyThroughputFromBroker = [.. data.Values] }); } else { - Out.WriteLine(" - No throughput detected in 30 days, ignoring"); + Out.WriteLine(" - No throughput detected in 90 days, ignoring"); } } } + var s = new DateTimeOffset(startTime, TimeOnly.MinValue, TimeSpan.Zero); + var e = new DateTimeOffset(endTime, TimeOnly.MaxValue, TimeSpan.Zero); return new QueueDetails { - StartTime = new DateTimeOffset(startTime, TimeSpan.Zero), - EndTime = new DateTimeOffset(endTime, TimeSpan.Zero), + StartTime = s, + EndTime = e, Queues = results.OrderBy(q => q.QueueName).ToArray(), - TimeOfObservation = TimeSpan.FromDays(1) + TimeOfObservation = e - s }; } catch (QueryException x) diff --git a/src/AppCommon/Commands/BaseCommand.cs b/src/AppCommon/Commands/BaseCommand.cs index 49f13cba..a283c141 100644 --- a/src/AppCommon/Commands/BaseCommand.cs +++ b/src/AppCommon/Commands/BaseCommand.cs @@ -224,7 +224,7 @@ async Task RunInternal(CancellationToken cancellationToken) ScopeType = data.ScopeType, StartTime = data.StartTime, EndTime = data.EndTime, - ReportDuration = data.TimeOfObservation ?? data.EndTime - data.StartTime, + ReportDuration = data.TimeOfObservation ?? (data.EndTime - data.StartTime), Queues = data.Queues, TotalThroughput = data.Queues.Sum(q => q.Throughput ?? 0), TotalQueues = data.Queues.Length, diff --git a/src/AppCommon/Commands/SqsCommand.cs b/src/AppCommon/Commands/SqsCommand.cs index 118ed765..9bd76a9e 100644 --- a/src/AppCommon/Commands/SqsCommand.cs +++ b/src/AppCommon/Commands/SqsCommand.cs @@ -74,15 +74,41 @@ protected override async Task GetData(CancellationToken cancellati var tasks = queueNames.Select(async queueName => { - var maxThroughput = await aws.GetMaxThroughput(queueName, cancellationToken).ConfigureAwait(false); + var datapoints = (await aws.GetMMetricsData(queueName, cancellationToken)).OrderBy(d => d.Timestamp).ToArray(); - // Since we get 30 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint + var maxThroughput = datapoints is { Length: > 0 } ? + (long)datapoints.Select(d => d.Sum.GetValueOrDefault(0)).Max() : 0L; + // Since we get 365 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint if (maxThroughput > 0) { + var startTime = DateOnly.FromDateTime(datapoints.First().Timestamp.Value); + var endTime = DateOnly.FromDateTime(datapoints.Last().Timestamp.Value); + DateOnly currentDate = startTime; + var dailyData = new Dictionary(); + while (currentDate <= endTime) + { + dailyData.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate }); + + currentDate = currentDate.AddDays(1); + } + + foreach (var datapoint in datapoints) + { + // There is a bug in the AWS SDK. The timestamp is actually UTC time, eventhough the DateTime returned type says Local + // See https://github.com/aws/aws-sdk-net/issues/167 + // So do not convert the timestamp to UTC time! + if (datapoint.Timestamp.HasValue) + { + currentDate = DateOnly.FromDateTime(datapoint.Timestamp.Value); + dailyData[currentDate] = new DailyThroughput { MessageCount = (long)datapoint.Sum.GetValueOrDefault(0), DateUTC = currentDate }; + } + } + data.Add(new QueueThroughput { QueueName = queueName, - Throughput = maxThroughput + Throughput = maxThroughput, + DailyThroughputFromBroker = [.. dailyData.Values] }); } @@ -94,12 +120,14 @@ protected override async Task GetData(CancellationToken cancellati Out.EndProgress(); + var s = new DateTimeOffset(aws.StartDate.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + var e = new DateTimeOffset(aws.EndDate.ToDateTime(TimeOnly.MaxValue), TimeSpan.Zero); return new QueueDetails { - StartTime = new DateTimeOffset(aws.StartDate.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), - EndTime = new DateTimeOffset(aws.EndDate.ToDateTime(TimeOnly.MaxValue), TimeSpan.Zero), + StartTime = s, + EndTime = e, Queues = [.. data.OrderBy(q => q.QueueName)], - TimeOfObservation = TimeSpan.FromDays(1) + TimeOfObservation = e - s }; } diff --git a/src/Query/AmazonSQS/AwsQuery.cs b/src/Query/AmazonSQS/AwsQuery.cs index d478c1f9..4bfaded8 100644 --- a/src/Query/AmazonSQS/AwsQuery.cs +++ b/src/Query/AmazonSQS/AwsQuery.cs @@ -35,7 +35,7 @@ public AwsQuery() QueueLimit = int.MaxValue }); EndDate = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(1); - StartDate = EndDate.AddDays(-30); + StartDate = EndDate.AddDays(-365); sqs = new AmazonSQSClient(); cloudWatch = new AmazonCloudWatchClient(); @@ -83,7 +83,7 @@ public async Task> GetQueueNames(Action onProgress, Cancellati } } - public async Task GetMaxThroughput(string queueName, CancellationToken cancellationToken = default) + public async Task> GetMMetricsData(string queueName, CancellationToken cancellationToken = default) { var req = new GetMetricStatisticsRequest { @@ -100,8 +100,7 @@ public async Task GetMaxThroughput(string queueName, CancellationToken can using var lease = await rateLimiter.AcquireAsync(cancellationToken: cancellationToken).ConfigureAwait(false); var resp = await cloudWatch.GetMetricStatisticsAsync(req, cancellationToken).ConfigureAwait(false); - return resp.Datapoints is { Count: > 0 } ? - (long)resp.Datapoints.Select(d => d.Sum.GetValueOrDefault(0)).Max() : 0L; + return resp.Datapoints ?? []; } } } diff --git a/src/Query/AzureServiceBus/AzureClient.cs b/src/Query/AzureServiceBus/AzureClient.cs index d313269c..1f9fce4e 100644 --- a/src/Query/AzureServiceBus/AzureClient.cs +++ b/src/Query/AzureServiceBus/AzureClient.cs @@ -116,18 +116,18 @@ bool NextCredentials() } } - public Task> GetMetrics(string queueName, DateTime startTime, DateTime endTime, CancellationToken cancellationToken = default) + public Task> GetMetrics(string queueName, DateOnly startTime, DateOnly endTime, CancellationToken cancellationToken = default) { return GetDataWithCurrentCredentials(async token => { try { var response = await currentClients.Metrics.QueryResourceAsync(resourceId, - new[] { "CompleteMessage" }, + ["CompleteMessage"], new MetricsQueryOptions { Filter = $"EntityName eq '{queueName}'", - TimeRange = new QueryTimeRange(startTime, endTime), + TimeRange = new QueryTimeRange(startTime.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc), endTime.ToDateTime(TimeOnly.MaxValue, DateTimeKind.Utc)), Granularity = TimeSpan.FromDays(1) }, token).ConfigureAwait(false);