Update SQS query logic to improve time handling and CloudWatch metric collection

- Adjusted `Initialize` to use the most eastern timezone UTC offset for accurate time handling to validate internally using TUTC
- Fixed Start/End timestamps to correctly be Utc
- Upgraded `AWSSDK.CloudWatch` package to version 4.0.6.1 for latest fixes and improvements.
- Replaced manual cancellation handling by utilizing `TestContext.CurrentContext.CancellationToken` to improve test cancellation flow.

Author: ramonsmits

src/Directory.Packages.props

@@ -5,7 +5,7 @@
   </PropertyGroup>
   <ItemGroup Label="Versions for direct package references">
     <PackageVersion Include="Autofac" Version="8.2.0" />
-    <PackageVersion Include="AWSSDK.CloudWatch" Version="4.0.1" />
+    <PackageVersion Include="AWSSDK.CloudWatch" Version="4.0.6.1" />
     <PackageVersion Include="AWSSDK.SecurityToken" Version="4.0.0.4" />
     <PackageVersion Include="Azure.Identity" Version="1.13.2" />
     <PackageVersion Include="Azure.Monitor.Query.Metrics" Version="1.0.0" />

src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs

@@ -25,12 +25,13 @@ class AmazonSQSQueryTests : TransportTestFixture
     public void Initialise()
     {
         provider = new();
-        provider.SetUtcNow(DateTimeOffset.UtcNow);
+
+        var kiribati = TimeZoneInfo.FindSystemTimeZoneById("Pacific/Kiritimati");
+        var furthestAhead = TimeZoneInfo.ConvertTime(DateTimeOffset.UtcNow, kiribati);
+        provider.SetUtcNow(furthestAhead);
         transportSettings = new TransportSettings
         {
-            ConnectionString = configuration.ConnectionString,
-            MaxConcurrency = 1,
-            EndpointName = Guid.NewGuid().ToString("N")
+            ConnectionString = configuration.ConnectionString, MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N")
         };
         query = new AmazonSQSQuery(NullLogger<AmazonSQSQuery>.Instance, provider, transportSettings);
     }
@@ -94,11 +95,9 @@ public async Task TestConnectionWithValidSettings()
     }
 
     [Test]
+    [CancelAfter(6 * 60 * 1000)]
     public async Task RunScenario()
     {
-        // We need to wait a bit of time, to ensure AWS metrics are retrievable
-        using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(6));
-        CancellationToken token = cancellationTokenSource.Token;
         const int numMessagesToIngest = 15;
 
         await CreateTestQueue(transportSettings.EndpointName);
@@ -120,12 +119,13 @@ public async Task RunScenario()
             dictionary.Add(AmazonSQSQuery.AmazonSQSSettings.Region, connectionString.Region);
         }
 
-        query.Initialize(new ReadOnlyDictionary<string, string>(dictionary));
+        query.Initialize(dictionary.AsReadOnly());
 
-        await Task.Delay(TimeSpan.FromMinutes(2), token);
+        // Wait for metrics to become visible, usually takes 20-30 seconds
+        await Task.Delay(TimeSpan.FromMinutes(1), TestContext.CurrentContext.CancellationToken);
 
         var queueNames = new List<IBrokerQueue>();
-        await foreach (IBrokerQueue queueName in query.GetQueueNames(token))
+        await foreach (IBrokerQueue queueName in query.GetQueueNames(TestContext.CurrentContext.CancellationToken))
         {
             queueNames.Add(queueName);
         }
@@ -137,7 +137,7 @@ public async Task RunScenario()
 
         DateTime startDate = provider.GetUtcNow().DateTime;
         provider.Advance(TimeSpan.FromDays(1));
-        await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), token))
+        await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), TestContext.CurrentContext.CancellationToken))
         {
             total += queueThroughput.TotalThroughput;
         }

src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs

@@ -197,24 +197,30 @@ public override async IAsyncEnumerable<QueueThroughput> GetThroughputPerDay(IBro
         DateOnly startDate,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        var endDate = DateOnly.FromDateTime(timeProvider.GetUtcNow().DateTime).AddDays(-1);
+        var utcNow = timeProvider.GetUtcNow();
+        var endDate = DateOnly.FromDateTime(utcNow.DateTime).AddDays(-1); // Query date up to but not including today
 
         if (endDate < startDate)
         {
             yield break;
         }
 
+        var startUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc);
+        var endUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc);
+
+        const int SecondsInDay = 24 * 60 * 60;;
+
         var req = new GetMetricStatisticsRequest
         {
             Namespace = "AWS/SQS",
             MetricName = "NumberOfMessagesDeleted",
-            StartTime = startDate.ToDateTime(TimeOnly.MinValue),
-            EndTime = endDate.ToDateTime(TimeOnly.MaxValue),
-            Period = 24 * 60 * 60, // 1 day
+            StartTime = startUtc,
+            EndTime = endUtc,  // exclusive
+            Period = SecondsInDay,
             Statistics = ["Sum"],
             Dimensions = [
                 new Dimension { Name = "QueueName", Value = brokerQueue.QueueName }
-            ]
+            ],
         };
 
         var resp = await cloudWatch!.GetMetricStatisticsAsync(req, cancellationToken);
@@ -228,11 +234,10 @@ public override async IAsyncEnumerable<QueueThroughput> GetThroughputPerDay(IBro
             currentDate = currentDate.AddDays(1);
         }
 
+        // Cloudwatch returns data points per 5 minutes in UTC
         foreach (var datapoint in resp.Datapoints ?? [])
         {
-            // There is a bug in the AWS SDK. The timestamp is actually UTC time, eventhough the DateTime returned type says Local
-            // See https://github.com/aws/aws-sdk-net/issues/167
-            // So do not convert the timestamp to UTC time!
+            logger.LogInformation("\tDatapoint {Timestamp:O} {Sum} {Unit}", datapoint.Timestamp, datapoint.Sum, datapoint.Unit);
             if (datapoint.Timestamp.HasValue)
             {
                 data[DateOnly.FromDateTime(datapoint.Timestamp.Value)].TotalThroughput = (long)datapoint.Sum.GetValueOrDefault(0);