Added special per-test only mode for code coverage (#143)

added special per-test only mode for code coverage

Author: ypopovych

DatadogSDKTesting.xcodeproj/project.pbxproj

@@ -36,7 +36,7 @@
 		A76174BD2CB3E3780098CE99 /* DDXCTestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = A76174BC2CB3E3780098CE99 /* DDXCTestCase.swift */; };
 		A76174BF2CB45F9B0098CE99 /* XCTestExtensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = A76174BE2CB45F9B0098CE99 /* XCTestExtensions.swift */; };
 		A76174C22CB685390098CE99 /* SettingsService.swift in Sources */ = {isa = PBXBuildFile; fileRef = A76174C02CB685250098CE99 /* SettingsService.swift */; };
-		A7A98C202C8F44BC00C93E59 /* CoveragePriority.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7A98C1F2C8F44BC00C93E59 /* CoveragePriority.swift */; };
+		A7A98C202C8F44BC00C93E59 /* CoverageSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7A98C1F2C8F44BC00C93E59 /* CoverageSettings.swift */; };
 		A7B39C6C2BCFC799007F98B2 /* CDatadogSDKTesting.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A7E232DC2BC6E68B0087D8F8 /* CDatadogSDKTesting.framework */; };
 		A7B39C702BCFCD77007F98B2 /* SubprocessWait.c in Sources */ = {isa = PBXBuildFile; fileRef = A7B39C6F2BCFCD77007F98B2 /* SubprocessWait.c */; };
 		A7B39C722BCFD234007F98B2 /* SubprocessWait.h in Headers */ = {isa = PBXBuildFile; fileRef = A7B39C712BCFD234007F98B2 /* SubprocessWait.h */; settings = {ATTRIBUTES = (Public, ); }; };
@@ -329,7 +329,7 @@
 		A76174BC2CB3E3780098CE99 /* DDXCTestCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DDXCTestCase.swift; sourceTree = "<group>"; };
 		A76174BE2CB45F9B0098CE99 /* XCTestExtensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = XCTestExtensions.swift; sourceTree = "<group>"; };
 		A76174C02CB685250098CE99 /* SettingsService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SettingsService.swift; sourceTree = "<group>"; };
-		A7A98C1F2C8F44BC00C93E59 /* CoveragePriority.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoveragePriority.swift; sourceTree = "<group>"; };
+		A7A98C1F2C8F44BC00C93E59 /* CoverageSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoverageSettings.swift; sourceTree = "<group>"; };
 		A7B39C6F2BCFCD77007F98B2 /* SubprocessWait.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = SubprocessWait.c; sourceTree = "<group>"; };
 		A7B39C712BCFD234007F98B2 /* SubprocessWait.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SubprocessWait.h; sourceTree = "<group>"; };
 		A7B4D4C02C21DC510059F39B /* TagsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TagsTests.swift; sourceTree = "<group>"; };
@@ -586,7 +586,7 @@
 			children = (
 				81EDADB2292698A200279027 /* LLVMTotalsCoverageFormat.swift */,
 				E1C8DEF4268363EC005B5411 /* DDCoverageHelper.swift */,
-				A7A98C1F2C8F44BC00C93E59 /* CoveragePriority.swift */,
+				A7A98C1F2C8F44BC00C93E59 /* CoverageSettings.swift */,
 			);
 			path = Coverage;
 			sourceTree = "<group>";
@@ -1539,7 +1539,7 @@
 				A7FC25C22BA1EAF500067E26 /* IntelligentTestRunner.swift in Sources */,
 				A7CC6D862C07D624003C13BC /* Tags.swift in Sources */,
 				A74246002BFF94FA00F7EC64 /* ITRSkippabble.swift in Sources */,
-				A7A98C202C8F44BC00C93E59 /* CoveragePriority.swift in Sources */,
+				A7A98C202C8F44BC00C93E59 /* CoverageSettings.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

Sources/DatadogSDKTesting/Config.swift

@@ -44,8 +44,8 @@ final class Config {
 
     /// Intelligent test runner related environment
     var gitUploadEnabled: Bool = true
-    var coverageEnabled: Bool = true
     var itrEnabled: Bool = true
+    var coverageMode: CodeCoverageMode = .total
     var excludedBranches: Set<String> = []
     var codeCoveragePriority: CodeCoveragePriority = .utility
 
@@ -124,8 +124,11 @@ final class Config {
         /// Intelligent test runner related configuration
         gitUploadEnabled = env[.enableCiVisibilityGitUpload] ?? gitUploadEnabled
         itrEnabled = env[.enableCiVisibilityITR] ?? itrEnabled
-        coverageEnabled = env[.enableCiVisibilityCodeCoverage] ?? itrEnabled
         excludedBranches = env[.ciVisibilityExcludedBranches] ?? excludedBranches
+
+        let coverageEnabled = env[.enableCiVisibilityCodeCoverage] ?? itrEnabled
+        let coveragePerTestOnly = env[.ciVisibilityCodeCoverageOnlyPerTest] ?? false
+        coverageMode = coverageEnabled ? (coveragePerTestOnly ? .perTest : .total) : .disabled
 
         /// Automatic Test Retries
         testRetriesEnabled = env[.enableCiVisibilityFlakyRetries] ?? testRetriesEnabled
@@ -215,7 +218,7 @@ extension Config: CustomDebugStringConvertible {
         Disable NTP Clock: \(disableNTPClock)
         Disable Git Information: \(disableGitInformation)
         Git Upload Enabled: \(gitUploadEnabled)
-        Coverage Enabled: \(coverageEnabled)
+        Coverage: \(coverageMode)
         ITR Enabled: \(itrEnabled)
         Excluded Branches: \(excludedBranches)
         Test Retries Enabled: \(testRetriesEnabled)

…KTesting/Coverage/CoveragePriority.swift …KTesting/Coverage/CoverageSettings.swiftSources/DatadogSDKTesting/Coverage/CoveragePriority.swift renamed to Sources/DatadogSDKTesting/Coverage/CoverageSettings.swift Sources/DatadogSDKTesting/Coverage/CoveragePriority.swift renamed to Sources/DatadogSDKTesting/Coverage/CoverageSettings.swift

@@ -40,3 +40,38 @@ enum CodeCoveragePriority: Int, EnvironmentValue, CustomDebugStringConvertible {
         }
     }
 }
+
+enum CodeCoverageMode {
+    case disabled
+    case perTest
+    case total
+    
+    var isTotal: Bool {
+        switch self {
+        case .total: return true
+        default: return false
+        }
+    }
+    
+    var isPerTest: Bool {
+        switch self {
+        case .perTest: return true
+        default: return false
+        }
+    }
+    
+    var isEnabled: Bool {
+        switch self {
+        case .disabled: return false
+        default: return true
+        }
+    }
+    
+    var debugDescription: String {
+        switch self {
+        case .total: return "Total"
+        case .perTest: return "Per-Test Only"
+        case .disabled: return "Disabled"
+        }
+    }
+}

Sources/DatadogSDKTesting/Coverage/DDCoverageHelper.swift

@@ -14,9 +14,10 @@ class DDCoverageHelper {
     var llvmProfileURL: URL
     var storagePath: Directory
     var initialCoverageSaved: Bool
+    let isTotal: Bool
     let coverageWorkQueue: OperationQueue
 
-    init?(storagePath: Directory, priority: CodeCoveragePriority) {
+    init?(storagePath: Directory, total: Bool, priority: CodeCoveragePriority) {
         guard let profilePath = Self.profileGetFileName(), BinaryImages.profileImages.count > 0 else {
             Log.print("Coverage not properly enabled in project, check documentation")
             Log.debug("LLVM_PROFILE_FILE: \(Self.profileGetFileName() ?? "NIL")")
@@ -30,6 +31,7 @@ class DDCoverageHelper {
         }
 
         llvmProfileURL = URL(fileURLWithPath: profilePath)
+        isTotal = total
         self.storagePath = path
         Log.debug("LLVM Coverage location: \(llvmProfileURL.path)")
         Log.debug("DDCoverageHelper location: \(path.url.path)")
@@ -93,10 +95,12 @@ class DDCoverageHelper {
     func writeTestProfile() {
         // Write first to our test file
         Self.internalWriteProfile()
-
-        // Write to llvm original destination
-        profileSetFilename(url: llvmProfileURL)
-        Self.internalWriteProfile()
+        if isTotal {
+            // Switch profile to llvm original destination
+            profileSetFilename(url: llvmProfileURL)
+            // Write to llvm original destination
+            Self.internalWriteProfile()
+        }
     }
 
     func removeStoragePath() {
@@ -140,7 +144,6 @@ class DDCoverageHelper {
         guard let profDataURL = DDCoverageHelper.generateProfData(profrawFile: profrawFile) else {
             return nil
         }
-        Thread.sleep(forTimeInterval: 0.1)
         let covJsonURL = profDataURL.deletingLastPathComponent().appendingPathComponent("coverageFile").appendingPathExtension("json")
         let binariesPath = binaryImagePaths.map { #""\#($0)""# }.joined(separator: " -object ")
         let commandToRun = #"xcrun llvm-cov export -instr-profile "\#(profDataURL.path)" \#(binariesPath) > "\#(covJsonURL.path)""#

Sources/DatadogSDKTesting/DDTestModule.swift

@@ -148,7 +148,9 @@ public class DDTestModule: NSObject, Encodable {
         if itrSkipped == 0 {
             meta[DDItrTags.itrSkippedTests] = "false"
             meta[DDTestSessionTags.testItrSkipped] = "false"
-            metrics[DDTestSessionTags.testCoverageLines] = DDCoverageHelper.getLineCodeCoverage()
+            if !DDTestMonitor.config.coverageMode.isPerTest {
+                metrics[DDTestSessionTags.testCoverageLines] = DDCoverageHelper.getLineCodeCoverage()
+            }
         } else {
             meta[DDItrTags.itrSkippedTests] = "true"
             meta[DDTestSessionTags.testItrSkipped] = "true"

Sources/DatadogSDKTesting/DDTestMonitor.swift

@@ -317,7 +317,7 @@ internal class DDTestMonitor {
             }
         }
 
-        if DDTestMonitor.config.coverageEnabled {
+        if DDTestMonitor.config.coverageMode.isEnabled {
             let coverageSetup = BlockOperation { [self] in
                 guard let settings = tracerBackendConfig?.itr,
                       settings.itrEnabled && settings.codeCoverage
@@ -333,6 +333,7 @@ internal class DDTestMonitor {
                     return
                 }
                 coverageHelper = DDCoverageHelper(storagePath: temp,
+                                                  total: DDTestMonitor.config.coverageMode.isTotal,
                                                   priority: DDTestMonitor.config.codeCoveragePriority)
             }
             coverageSetup.addDependency(updateTracerConfig)

Sources/DatadogSDKTesting/Environment/EnvironmentKeys.swift

@@ -41,6 +41,7 @@ internal enum EnvironmentKey: String, CaseIterable {
     case enableCiVisibilityGitUpload = "DD_CIVISIBILITY_GIT_UPLOAD_ENABLED"
     case enableCiVisibilityCodeCoverage = "DD_CIVISIBILITY_CODE_COVERAGE_ENABLED"
     case ciVisibilityCodeCoveragePriority = "DD_CIVISIBILITY_CODE_COVERAGE_PRIORITY"
+    case ciVisibilityCodeCoverageOnlyPerTest = "DD_CIVISIBILITY_CODE_COVERAGE_ONLY_PER_TEST"
     case enableCiVisibilityITR = "DD_CIVISIBILITY_ITR_ENABLED"
     case ciVisibilityExcludedBranches = "DD_CIVISIBILITY_EXCLUDED_BRANCHES"
     case ciVisibilityReportHostname = "DD_CIVISIBILITY_REPORT_HOSTNAME"

Sources/DatadogSDKTesting/FrameworkLoadHandler.swift

@@ -30,7 +30,7 @@ enum FrameworkLoadHandler {
         if config.isInTestMode {
             // When code coverage is enabled modify profile name so it disables countinuous profiling
             // or we cannot recover coverage manually
-            if config.coverageEnabled || config.itrEnabled {
+            if config.coverageMode.isEnabled || config.itrEnabled {
                 if DDCoverageHelper.load() {
                     DDTestMonitor.envReader = ProcessEnvironmentReader()
                 }