Fix binary metrics so they properly return NaN when appropriate (#113)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: ericphanson

Project.toml

@@ -1,7 +1,7 @@
 name = "Lighthouse"
 uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.17.1"
+version = "0.17.2"
 
 [deps]
 ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

src/metrics.jl

@@ -73,20 +73,11 @@ function binary_statistics(confusion::AbstractMatrix, class_index::Integer)
     false_positives = predicted_positives - true_positives
     false_negatives = actual_positives - true_positives
     true_negatives = actual_negatives - false_positives
-    true_positive_rate = (true_positives == 0 && actual_positives == 0) ?
-                         (one(true_positives) / one(actual_positives)) :
-                         (true_positives / actual_positives)
-    true_negative_rate = (true_negatives == 0 && actual_negatives == 0) ?
-                         (one(true_negatives) / one(actual_negatives)) :
-                         (true_negatives / actual_negatives)
-    false_positive_rate = (false_positives == 0 && actual_negatives == 0) ?
-                          (zero(false_positives) / one(actual_negatives)) :
-                          (false_positives / actual_negatives)
-    false_negative_rate = (false_negatives == 0 && actual_positives == 0) ?
-                          (zero(false_negatives) / one(actual_positives)) :
-                          (false_negatives / actual_positives)
-    precision = (true_positives == 0 && predicted_positives == 0) ? NaN :
-                (true_positives / predicted_positives)
+    true_positive_rate = true_positives / actual_positives
+    true_negative_rate = true_negatives / actual_negatives
+    false_positive_rate = false_positives / actual_negatives
+    false_negative_rate = false_negatives / actual_positives
+    precision = true_positives / predicted_positives
     f1 = true_positives / (true_positives + 0.5 * (false_positives + false_negatives))
     return (; predicted_positives, predicted_negatives, actual_positives, actual_negatives,
             true_positives, true_negatives, false_positives, false_negatives,

src/utilities.jl

@@ -15,11 +15,15 @@ end
     area_under_curve(x, y)
 
 Calculates the area under the curve specified by the `x` vector and `y` vector
-using the trapezoidal rule. If inputs are empty, return `missing`.
+using the trapezoidal rule. If inputs are empty, return `NaN`. Excludes NaN entries.
 """
 function area_under_curve(x, y)
     length(x) == length(y) || throw(ArgumentError("Length of inputs must match."))
-    length(x) == 0 && return missing
+    isempty(x) && return NaN
+    non_nan = (!).(isnan.(x) .| isnan.(y))
+    x = x[non_nan]
+    y = y[non_nan]
+    isempty(x) && return NaN
     auc = zero(middle(one(eltype(x)), one(eltype(y))))
     perms = sortperm(x)
     sorted_x = view(x, perms)

test/metrics.jl

@@ -85,10 +85,10 @@
     @test stats.true_negatives == 0
     @test stats.false_positives == 0
     @test stats.false_negatives == 0
-    @test stats.true_positive_rate == 1
-    @test stats.true_negative_rate == 1
-    @test stats.false_positive_rate == 0
-    @test stats.false_negative_rate == 0
+    @test isnan(stats.true_positive_rate)
+    @test isnan(stats.true_negative_rate)
+    @test isnan(stats.false_positive_rate)
+    @test isnan(stats.false_negative_rate)
     @test isnan(stats.precision)
     @test isnan(stats.f1)
 
@@ -100,6 +100,8 @@
     @test stats.false_positives == 0
     @test stats.false_negatives == 0
     @test isnan(stats.f1)
+    @test isnan(stats.true_positive_rate)
+    @test isnan(stats.false_negative_rate)
 
     c = [0 2
          0 6]
@@ -109,6 +111,8 @@
     @test stats.false_positives == 2
     @test stats.false_negatives == 0
     @test stats.f1 == 0
+    @test isnan(stats.true_positive_rate)
+    @test isnan(stats.false_negative_rate)
 
     c = [0 0
          2 6]
@@ -118,7 +122,6 @@
     @test stats.false_positives == 0
     @test stats.false_negatives == 2
     @test stats.f1 == 0
-
     for p in 0:0.1:1
         @test Lighthouse._cohens_kappa(p, p) == 0
         if p > 0

test/utilities.jl

@@ -8,7 +8,8 @@ end
 
 @testset "`Lighthouse.area_under_curve`" begin
     @test_throws ArgumentError Lighthouse.area_under_curve([0, 1, 2], [0, 1])
-    @test ismissing(Lighthouse.area_under_curve([], []))
+    @test isnan(Lighthouse.area_under_curve([], []))
+    @test isnan(Lighthouse.area_under_curve([NaN], [NaN]))
     @test isapprox(Lighthouse.area_under_curve(collect(0:0.01:1), collect(0:0.01:1)), 0.5;
                    atol=0.01)
     @test isapprox(Lighthouse.area_under_curve(collect(0:0.01:(2π)), sin.(0:0.01:(2π))),
@@ -17,7 +18,8 @@ end
 
 @testset "`Lighthouse.area_under_curve_unit_square`" begin
     @test_throws ArgumentError Lighthouse.area_under_curve_unit_square([0, 1, 2], [0, 1])
-    @test ismissing(Lighthouse.area_under_curve_unit_square([], []))
+    @test isnan(Lighthouse.area_under_curve_unit_square([], []))
+    @test isnan(Lighthouse.area_under_curve_unit_square([NaN], [NaN]))
     @test isapprox(Lighthouse.area_under_curve_unit_square(collect(0:0.01:1),
                                                            collect(0:0.01:1)), 0.5;
                    atol=0.01)