Allow missing expert curves in discrimination calibration curve plot (#95)

* fix plotting when no expert discrimination curves

* fix plot setup

* undo excess lint

* Update src/plotting.jl

Co-authored-by: Eric Hanson <5846501+ericphanson@users.noreply.github.com>

* plot discrimination plots in isolation too

* flip test order for unique plots

Co-authored-by: Simon <sdanisch@protonmail.com>
Co-authored-by: Eric Hanson <5846501+ericphanson@users.noreply.github.com>

Author: 3 people

Project.toml

@@ -1,7 +1,7 @@
 name = "Lighthouse"
 uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.14.15"
+version = "0.14.16"
 
 [deps]
 ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

src/plotting.jl

@@ -141,10 +141,14 @@ function set_from_kw!(theme, key, kw, default)
 end
 
 function plot_binary_discrimination_calibration_curves!(subfig::FigurePosition,
-                                                        calibration_curve::SeriesCurves, calibration_score,
-                                                        per_expert_calibration_curves::SeriesCurves,
-                                                        per_expert_calibration_scores, optimal_threshold,
-                                                        discrimination_class::AbstractString; kw...)
+                                                        calibration_curve::SeriesCurves,
+                                                        calibration_score,
+                                                        per_expert_calibration_curves::Union{SeriesCurves,
+                                                                                             Missing},
+                                                        per_expert_calibration_scores,
+                                                        optimal_threshold,
+                                                        discrimination_class::AbstractString;
+                                                        kw...)
     kw = values(kw)
     scatter_theme = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :Scatter; strokewidth=0)
     # Hayaah, this theme merging is getting out of hand
@@ -154,12 +158,18 @@ function plot_binary_discrimination_calibration_curves!(subfig::FigurePosition,
     set_from_kw!(scatter_theme, :makersize, kw, 5)
     set_from_kw!(scatter_theme, :marker, kw, :rect)
 
-    per_expert = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :PerExpert; solid_color=:darkgrey,
-                           color=nothing)
-    set_from_kw!(per_expert, :linewidth, kw, 2)
-    ax = series_plot!(subfig, per_expert_calibration_curves, nothing; legend=nothing,
-                      title="Detection calibration", xlabel="Expert agreement rate",
-                      ylabel="Predicted positive probability", scatter=scatter_theme, per_expert...)
+    if ismissing(per_expert_calibration_curves)
+        ax = Axis(subfig; title="Detection calibration", xlabel="Expert agreement rate",
+                  ylabel="Predicted positive probability")
+    else
+        per_expert = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :PerExpert;
+                               solid_color=:darkgrey, color=nothing)
+        set_from_kw!(per_expert, :linewidth, kw, 2)
+        ax = series_plot!(subfig, per_expert_calibration_curves, nothing; legend=nothing,
+                          title="Detection calibration", xlabel="Expert agreement rate",
+                          ylabel="Predicted positive probability", scatter=scatter_theme,
+                          per_expert...)
+    end
 
     calibration = get_theme(subfig, :BinaryDiscriminationCalibrationCurves, :CalibrationCurve;
                             solid_color=:navyblue, markerstrokewidth=0)

test/learn.jl

@@ -296,11 +296,34 @@ end
         @test !isequal(thresh_from_roc, thresh_from_calibration)
         @test isequal(thresh_from_calibration, plot_data_2["optimal_threshold"])
 
+        # Also, let's make sure we get an isolated discrimination plots
+        discrimination_cal = Lighthouse.plot_binary_discrimination_calibration_curves(plot_data_1["discrimination_calibration_curve"],
+                                                                                      plot_data_1["discrimination_calibration_score"],
+                                                                                      plot_data_1["per_expert_discrimination_calibration_curves"],
+                                                                                      plot_data_1["per_expert_discrimination_calibration_scores"],
+                                                                                      plot_data_1["optimal_threshold"],
+                                                                                      plot_data_1["class_labels"][plot_data_1["optimal_threshold_class"]])
+        @testplot discrimination_cal
+
+        discrimination_cal_no_experts = Lighthouse.plot_binary_discrimination_calibration_curves(plot_data_1["discrimination_calibration_curve"],
+                                                                                                 plot_data_1["discrimination_calibration_score"],
+                                                                                                 missing,
+                                                                                                 missing,
+                                                                                                 plot_data_1["optimal_threshold"],
+                                                                                                 plot_data_1["class_labels"][plot_data_1["optimal_threshold_class"]])
+
+        # Test binary discrimination with no multiclass votes
+        plot_data_1["per_expert_discrimination_calibration_curves"] = missing
+        no_expert_calibration = evaluation_metrics_plot(EvaluationRow(plot_data_1))
+        @testplot no_expert_calibration
+
         # Test that plotting succeeds (no specialization relative to the multi-class tests)
         plot_data = last(logger.logged["validation_set_evaluation/metrics_per_epoch"])
         all_together = evaluation_metrics_plot(plot_data)
         #savefig(all_together, "/tmp/binary.png")
         @testplot all_together
+
+        @testplot discrimination_cal_no_experts
     end
 end