Merge pull request #41 from DEFRA/feature/sonarqube

Sonar qube support

Author: krisddefra

.github/dependabot.yml

@@ -0,0 +1,74 @@
+version: 2
+
+updates:
+  - package-ecosystem: nuget
+    directory: /
+    schedule:
+      interval: weekly
+      day: monday
+      time: "09:00"
+      timezone: Europe/London
+    open-pull-requests-limit: 10
+    
+    # Only update direct dependencies (not transitive)
+    allow:
+      - dependency-type: direct
+
+    # Group related packages to reduce PR noise
+    groups:
+      # AWS SDK packages - update together for compatibility
+      aws-sdk:
+        patterns:
+          - "AWSSDK*"
+          - "Amazon.*"
+      
+      # Microsoft packages (excluding ASP.NET Core which should match runtime)
+      microsoft:
+        patterns:
+          - "Microsoft.Extensions.*"
+        exclude-patterns:
+          - "Microsoft.AspNetCore.*"
+          - "Microsoft.NET.Test.Sdk"
+      
+      # OpenTelemetry packages - update together
+      opentelemetry:
+        patterns:
+          - "OpenTelemetry*"
+      
+      # Serilog logging packages
+      serilog:
+        patterns:
+          - "Serilog*"
+          - "Elastic.CommonSchema.Serilog"
+      
+      # Test framework packages
+      testing:
+        patterns:
+          - "xunit*"
+          - "Moq"
+          - "FluentAssertions"
+          - "Bogus"
+          - "coverlet.*"
+          - "Microsoft.NET.Test.Sdk"
+          - "MartinCostello.Logging.XUnit"
+          - "GitHubActionsTestLogger"
+      
+      # Testcontainers packages
+      testcontainers:
+        patterns:
+          - "Testcontainers*"
+
+    # Ignore packages that should stay pinned to runtime version
+    ignore:
+      # ASP.NET Core packages should match the .NET 8 runtime
+      - dependency-name: "Microsoft.AspNetCore.*"
+        update-types:
+          - "version-update:semver-major"
+
+    commit-message:
+      prefix: "deps"
+      include: "scope"
+
+    labels:
+      - "dependencies"
+      - "nuget"

.github/example.dependabot.yml

@@ -88,12 +88,9 @@ jobs:
       - name: Run integration tests
         run: dotnet test ./working-dir/KeeperData.Bridge.sln --filter "Dependence=docker&testtype!=performance"
 
-    ## SonarCloud
-    ## Uncomment to unable SonarCloud scan
-    ## Requires project to be set up in SonarCloud
-    ## and the SonarCloud token to be set in the repository secrets
-#   sonarcloud-scan:
-#     name: CDP SonarCloud Scan
-#     uses: ./.github/workflows/sonarcloud.yml
-#     needs: pr-validator
-#     secrets: inherit
+  sonarcloud-scan:
+    name: CDP SonarCloud Scan
+    uses: ./.github/workflows/sonarcloud.yml
+    needs: pr-validator
+    secrets:
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/check-pull-request.yml

@@ -2,6 +2,10 @@ name: Run SonarCloud coverage scan
 
 on:
   workflow_call:
+    secrets:
+      SONAR_TOKEN:
+        description: "SonarCloud token used for authentication"
+        required: true
 
 permissions:
   id-token: write
@@ -64,7 +68,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          ./.sonar/scanner/dotnet-sonarscanner begin /k:"DEFRA_cdp-dotnet-backend-template" /o:"defra" /d:sonar.token="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.vscoveragexml.reportsPaths=coverage.xml
-          dotnet build --no-incremental
-          ./.sonar/coverage/dotnet-coverage collect "dotnet test" -f xml -o "coverage.xml"
-          ./.sonar/scanner/dotnet-sonarscanner end /d:sonar.token="${{ secrets.SONAR_TOKEN }}"
+          ./.sonar/scanner/dotnet-sonarscanner begin /k:"DEFRA_ls-keeper-data-bridge-backend" /o:"defra" /d:sonar.token="$SONAR_TOKEN" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.vscoveragexml.reportsPaths=coverage.xml
+          dotnet build KeeperData.Bridge.sln --no-incremental
+          ./.sonar/coverage/dotnet-coverage collect "dotnet test KeeperData.Bridge.sln --filter \"Dependence!=docker&testtype!=performance\"" -f xml -o "coverage.xml"
+          ./.sonar/scanner/dotnet-sonarscanner end /d:sonar.token="$SONAR_TOKEN"

.github/workflows/sonarcloud.yml

@@ -77,7 +77,6 @@ public void GetCurrentStatus_AfterUpdateProgress_ShouldReturnNonZeroProgress()
         // Assert
         status.RowNumber.Should().Be(500);
         status.PercentageCompleted.Should().BeGreaterThan(0);
-        status.RowsPerMinute.Should().NotBeNull();
     }
 
     [Fact]

tests/KeeperData.Core.Tests.Unit/ETL/Utils/IngestionProgressTrackerTests.cs

@@ -373,65 +373,7 @@ await _cryptoTransform.EncryptStreamAsync(
         progressReports.Should().Contain(r => r.percentage == 0 && r.status.Contains("processed") && !r.status.Contains("%"));
     }
 
-    [Fact(Skip = "This is too fuzzy for remote ci-cd pipelines")]
-    public async Task LargeFileStreaming_Over500MB_ShouldProcessWithMinimalMemoryUsage()
-    {
-        // Arrange
-        const long fileSizeBytes = 500L * 1024 * 1024 + 1024; // 500MB + 1KB
-        var inputFile = CreateLargeTempFile(fileSizeBytes);
-        var encryptedFile = GetTempFilePath();
-        var decryptedFile = GetTempFilePath();
-
-        var encryptProgressReports = new List<(int percentage, string status)>();
-        var decryptProgressReports = new List<(int percentage, string status)>();
-
-        // Monitor memory usage
-        var initialMemory = GC.GetTotalMemory(true);
-
-        // Act - Encrypt
-        await _cryptoTransform.EncryptFileAsync(
-            inputFile,
-            encryptedFile,
-            TestPassword,
-            TestSaltBytes,
-            (percentage, status) => encryptProgressReports.Add((percentage, status)));
-
-        var memoryAfterEncrypt = GC.GetTotalMemory(false);
-
-        // Act - Decrypt
-        await _cryptoTransform.DecryptFileAsync(
-            encryptedFile,
-            decryptedFile,
-            TestPassword,
-            TestSaltBytes,
-            (percentage, status) => decryptProgressReports.Add((percentage, status)));
-
-        var memoryAfterDecrypt = GC.GetTotalMemory(false);
-
-        // Assert file sizes
-        var originalFileInfo = new FileInfo(inputFile);
-        var decryptedFileInfo = new FileInfo(decryptedFile);
-
-        originalFileInfo.Length.Should().Be(fileSizeBytes);
-        decryptedFileInfo.Length.Should().Be(fileSizeBytes);
-
-        // Assert memory usage - should not buffer entire file in memory
-        // Memory increase should be much less than file size (allowing some overhead for test framework)
-        var maxMemoryIncrease = Math.Max(memoryAfterEncrypt - initialMemory, memoryAfterDecrypt - initialMemory);
-        maxMemoryIncrease.Should().BeLessThan(10 * 1024 * 1024); // Less than 10MB increase
-
-        // Assert progress reporting
-        encryptProgressReports.Should().NotBeEmpty();
-        encryptProgressReports.Should().Contain(r => r.percentage == 100);
-        encryptProgressReports.Where(r => r.status.Contains("%")).Should().NotBeEmpty();
-
-        decryptProgressReports.Should().NotBeEmpty();
-        decryptProgressReports.Should().Contain(r => r.percentage == 100);
-        decryptProgressReports.Where(r => r.status.Contains("%")).Should().NotBeEmpty();
-
-        // Verify data integrity by comparing checksums of first and last chunks
-        await VerifyFileIntegrity(inputFile, decryptedFile);
-    }
+    
 
     [Fact]
     public async Task StreamDecryption_LargeFile_ShouldMaintainConstantMemoryUsage()