Merge pull request #11 from JuliaAI/dev

For a 01.1 release

Author: ablaom

Project.toml

@@ -1,7 +1,7 @@
 name = "StatisticalMeasuresBase"
 uuid = "c062fc1d-0d66-479b-b6ac-8b44719de4cc"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.1.0"
+version = "0.1.1"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"

README.md

@@ -7,15 +7,15 @@ A Julia package for building production-ready measures (metrics) for statistics
 [![Docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliaai.github.io/StatisticalMeasuresBase.jl/dev/)
 
 Related:
-[StatisticalMeasures.jl](https://github.com/JuliaAI/StatisticalMeasures.jl).
+[StatisticalMeasures.jl](https://juliaai.github.io/StatisticalMeasures.jl/dev/)
 
 
-## The main idea 
+## The main idea
 
 Here's an example of a simple statistical measure that can be applied to a pair of scalars:
 
 ```julia
-l1(ŷ, y) = abs(ŷ - y) 
+l1(ŷ, y) = abs(ŷ - y)
 y = 5 # ground truth
 ŷ = 2 # prediction
 
@@ -55,7 +55,7 @@ julia> multitarget_L1(t̂, t, weights)
 39
 ```
 
-Access per-observation measurements with the `measurement` method:
+Generate measurements *for each observation* with the `measurement` method:
 
 ```julia
 julia> measurements(multitarget_L1, t̂, t, weights)

docs/src/index.md

@@ -1,5 +1,13 @@
 ```@raw html
 <script async defer src="https://buttons.github.io/buttons.js"></script>
+
+<div style="font-size:1.4em;font-weight:bold;">
+  <a href="tutorial"
+    style="color: #389826;">Tutorial</a>           &nbsp;|&nbsp;
+  <a href="https://juliaai.github.io/StatisticalMeasuresBase.jl/dev/implementing_new_measures/#definitions"
+    style="color: #9558B2;">What is a measure?</a>
+</div>
+
 <span style="color: #9558B2;font-size:4.5em;">
 StatisticalMeasuresBase.jl</span>
 <br>
@@ -45,18 +53,12 @@ t̂ = ŷ' |> Tables.table |> Tables.rowtable
 multitarget_L1(t̂, t, weights)
 ```
 
-Access per-observation measurements with the `measurement` method:
+Generate measurements *for each observation* with the `measurement` method:
 
 ```@example 01
 measurements(multitarget_L1, t̂, t, weights)
 ```
 
-### Quick links
-
-- [Tutorial](@ref), to jump right into building new measures
-- [What is a measure?](@ref definitions)
-
-
 # Overview
 
 This package specifies [an interface](@ref definitions) for statistical measures (metrics)
@@ -74,9 +76,9 @@ Specically, this package provides:
   function) that consumes single observations to measures consuming vectors, arrays or
   tables (multi-target measures).
 
-- Other [wrappers](@ref wrappers) to add missing value support, argument checks,
-  direct callability, or to silently treat unsupported weights as uniform (good for
-  application of a batch of measures with mixed degrees of weight support)
+- Other [wrappers](@ref wrappers) to add missing value support, argument checks, or to
+  silently treat unsupported weights as uniform (good for application of a batch of
+  measures with mixed degrees of weight support)
 
 - [`measurements`](@ref), a method to return *unaggregated* measurements
 

src/api.jl

@@ -11,8 +11,9 @@ on data.
 
 # New implementations
 
-Overloading this trait is optional. A fallback returns the aggregated measure, repeated
-`n` times, where `n = MLUtils.numobs(y)`.  It is not typically necessary to overload
+Overloading this function for new measure types is optional. A fallback returns the
+aggregated measure, repeated `n` times, where `n = MLUtils.numobs(y)` (which falls back to
+`length(y)` if `numobs` is not implemented).  It is not typically necessary to overload
 `measurements` for wrapped measures.  All [`multimeasure`](@ref)s provide the obvious
 fallback and other wrappers simply forward the `measurements` method of the atomic
 measure. If overloading, use the following signatures:
@@ -24,7 +25,6 @@ measure. If overloading, use the following signatures:
 
 """
 function measurements(measure, yhat, y, args...)
-    consumes_multiple_observations(measure) || return measure(yhat, y, args...)
     m = measure(yhat, y, args...)
     fill(m, MLUtils.numobs(y))
 end

src/traits.jl

@@ -91,8 +91,9 @@ with the same number of observations as `y`.
 
 # New implementations
 
-Overloading the trait is optional and it is typically not overloaded. The general fallback
-returns `false` but it is `true` for any [`multimeasure`](@ref), and the value is
+Overload this trait for a new measure type that consumes multiple observations, unless it
+has been constructed using `multimeaure` or is an $API.jl wrap thereof. The general
+fallback returns `false` but it is `true` for any [`multimeasure`](@ref), and the value is
 propagated by other wrappers.
 
 """

test/api.jl

@@ -9,6 +9,12 @@ w = 10:10:10N
     measurements(measure, 2.3, 3.4) == LPLossOnScalars()(2.3, 3.4)
     measure = MeanAbsoluteError()
     measurements(measure, yhat, y) == fill(measure(yhat, y), N)
+
+    # if a measure does not overload `consumes_multiple_observations` but really does, then
+    # `meausurements` should nevertheless have the expected behavior:
+    badly_implemented_rms(yhat, y) = (yhat - y).^2 |> mean |> sqrt
+    μ = badly_implemented_rms([4, 5], [1, 1])
+    @test measurements(badly_implemented_rms, [4, 5], [1, 1]) ≈ [μ, μ]
 end
 
 true

test/measure.jl

@@ -9,7 +9,7 @@ API.is_measure(::Measure{typeof(min)}) = true
 @testset "calling" begin
     measure = Measure(min)
     @test measure(2, -3) == 3
-    @test measurements(measure, 2, -3) == 3
+    @test measurements(measure, 2, -3) == [3,]
 end
 
 true