don't error if only one voter (#94)

Author: ericphanson

Project.toml

@@ -1,7 +1,7 @@
 name = "Lighthouse"
 uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.14.14"
+version = "0.14.15"
 
 [deps]
 ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

src/metrics.jl

@@ -483,7 +483,7 @@ function evaluation_metrics_row(predicted_hard_labels::AbstractVector,
     # Step 4: Calculate all metrics derived directly from labels (does not depend on
     # predictions)
     labels_metrics_table = LabelMetricsRow[]
-    if has_value(votes)
+    if has_value(votes) && size(votes, 2) > 1
         labels_metrics_table = map(c -> get_label_metrics_multirater(votes, c),
                                    class_indices)
         labels_metrics_table = vcat(labels_metrics_table,

test/row.jl

@@ -52,62 +52,70 @@ end
     @test isequal(metrics_from_inputs, metrics_from_table)
 
     # Multiple labelers: round-trip `ObservationRow``...
-    num_voters = 5
-    possible_vote_labels = collect(0:length(classes)) # vote 0 == "no vote"
-    vote_rng = StableRNG(22)
-    votes = [rand(vote_rng, possible_vote_labels)
-             for sample in 1:num_observations, voter in 1:num_voters]
-    votes[:, 3] .= votes[:, 4] # Voter 4 voted identically to voter 3 (force non-zero agreement)
-    elected_hard_multilabeller = map(row -> majority(vote_rng, row, 1:length(classes)),
-                                     eachrow(votes))
-    table = test_roundtrip_observation_table(; predicted_soft_labels, predicted_hard_labels,
-                                             elected_hard_labels=elected_hard_multilabeller,
-                                             votes)
-
-    # ...is there parity in evaluation_metrics calculations?
-    metrics_from_inputs = Lighthouse.evaluation_metrics_row(predicted_hard_labels,
-                                                            predicted_soft_labels,
-                                                            elected_hard_multilabeller,
-                                                            classes; votes)
-    metrics_from_table = Lighthouse.evaluation_metrics_row(table, classes)
-    @test isequal(metrics_from_inputs, metrics_from_table)
-
-    r_table = Lighthouse._inputs_to_observation_table(; predicted_soft_labels,
-                                                      predicted_hard_labels,
-                                                      elected_hard_labels=elected_hard_multilabeller,
-                                                      votes)
-    @test isnothing(Legolas.validate(r_table, Lighthouse.OBSERVATION_ROW_SCHEMA))
-
-    # ...can we handle both dataframe input and more generic row iterators?
-    df_table = DataFrame(r_table)
-    output_r = Lighthouse._observation_table_to_inputs(r_table)
-    output_df = Lighthouse._observation_table_to_inputs(df_table)
-    @test isequal(output_r, output_df)
-
-    # Safety last!
-    transform!(df_table, :votes => ByRow(v -> isodd(sum(v)) ? missing : v);
-               renamecols=false)
-    @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table)
-
-    transform!(df_table, :votes => ByRow(v -> ismissing(v) ? [1, 2, 3] : v);
-               renamecols=false)
-    @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table)
-
-    @test_throws DimensionMismatch Lighthouse._inputs_to_observation_table(;
-                                                                           predicted_soft_labels,
-                                                                           predicted_hard_labels=predicted_hard_labels[1:4],
-                                                                           elected_hard_labels=elected_hard_multilabeller,
-                                                                           votes)
+    for num_voters in (1, 5)
+        possible_vote_labels = collect(0:length(classes)) # vote 0 == "no vote"
+        vote_rng = StableRNG(22)
+        votes = [rand(vote_rng, possible_vote_labels)
+                 for sample in 1:num_observations, voter in 1:num_voters]
+        if num_voters >= 4
+            votes[:, 3] .= votes[:, 4] # Voter 4 voted identically to voter 3 (force non-zero agreement)
+        end
+        elected_hard_multilabeller = map(row -> majority(vote_rng, row, 1:length(classes)),
+                                         eachrow(votes))
+        table = test_roundtrip_observation_table(; predicted_soft_labels,
+                                                 predicted_hard_labels,
+                                                 elected_hard_labels=elected_hard_multilabeller,
+                                                 votes)
+
+        # ...is there parity in evaluation_metrics calculations?
+        metrics_from_inputs = Lighthouse.evaluation_metrics_row(predicted_hard_labels,
+                                                                predicted_soft_labels,
+                                                                elected_hard_multilabeller,
+                                                                classes; votes)
+        metrics_from_table = Lighthouse.evaluation_metrics_row(table, classes)
+        @test isequal(metrics_from_inputs, metrics_from_table)
+
+        r_table = Lighthouse._inputs_to_observation_table(; predicted_soft_labels,
+                                                          predicted_hard_labels,
+                                                          elected_hard_labels=elected_hard_multilabeller,
+                                                          votes)
+        @test isnothing(Legolas.validate(r_table, Lighthouse.OBSERVATION_ROW_SCHEMA))
+
+        # ...can we handle both dataframe input and more generic row iterators?
+        df_table = DataFrame(r_table)
+        output_r = Lighthouse._observation_table_to_inputs(r_table)
+        output_df = Lighthouse._observation_table_to_inputs(df_table)
+        @test isequal(output_r, output_df)
+
+        # Safety last!
+        transform!(df_table, :votes => ByRow(v -> isodd(sum(v)) ? missing : v);
+                   renamecols=false)
+        @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table)
+
+        transform!(df_table, :votes => ByRow(v -> ismissing(v) ? [1, 2, 3] : v);
+                   renamecols=false)
+        @test_throws ArgumentError Lighthouse._observation_table_to_inputs(df_table)
+
+        @test_throws DimensionMismatch Lighthouse._inputs_to_observation_table(;
+                                                                               predicted_soft_labels,
+                                                                               predicted_hard_labels=predicted_hard_labels[1:4],
+                                                                               elected_hard_labels=elected_hard_multilabeller,
+                                                                               votes)
+    end
 end
 
 @testset "`ClassRow" begin
     @test isa(Lighthouse.ClassRow(; class_index=3, class_labels=missing).class_index, Int64)
-    @test isa(Lighthouse.ClassRow(; class_index=Int8(3), class_labels=missing).class_index, Int64)
+    @test isa(Lighthouse.ClassRow(; class_index=Int8(3), class_labels=missing).class_index,
+              Int64)
     @test Lighthouse.ClassRow(; class_index=:multiclass).class_index == :multiclass
-    @test Lighthouse.ClassRow(; class_index=:multiclass, class_labels=["a", "b"]).class_labels == ["a", "b"]
+    @test Lighthouse.ClassRow(; class_index=:multiclass,
+                              class_labels=["a", "b"]).class_labels == ["a", "b"]
 
-    @test_throws ArgumentError Lighthouse.ClassRow(; class_index=3.0f0, class_labels=missing)
-    @test_throws ArgumentError Lighthouse.ClassRow(; class_index=:mUlTiClAsS, class_labels=missing)
+    @test_throws ArgumentError Lighthouse.ClassRow(; class_index=3.0f0,
+                                                   class_labels=missing)
+    @test_throws ArgumentError Lighthouse.ClassRow(; class_index=:mUlTiClAsS,
+                                                   class_labels=missing)
 end
 
 @testset "class_labels" begin
@@ -117,7 +125,7 @@ end
                              0.9 0.1
                              0.0 1.0]
     elected_hard_labels = [1, 2, 2, 2, 1]
-    predicted_hard_labels = [1,2,2,1,2]
+    predicted_hard_labels = [1, 2, 2, 1, 2]
     thresholds = [0.25, 0.5, 0.75]
     i_class = 2
     class_labels = ["a", "b"]
@@ -127,7 +135,6 @@ end
     @test default_metrics.class_labels == class_labels
 end
 
-
 @testset "`Curve`" begin
     c = ([1, 2, 3], [4, 5, 6])
     @test Curve(c...) isa Curve