Skip to content

Fix: Set default average='micro' consistently across classification metrics (some of the subclasses of MulticlassStatScores, MultilabelStatScores) #2882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
Draft

Fix: Set default average='micro' consistently across classification metrics (some of the subclasses of MulticlassStatScores, MultilabelStatScores) #2882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fb80698
fix: Ensure consistent default values('micro') for average argument i…
rittik9 Dec 24, 2024
5c1db37
fix modular doctests
rittik9 Dec 24, 2024
954be86
Update accuracy.py
rittik9 Dec 24, 2024
d50999f
fix default vals for functional forms
rittik9 Dec 24, 2024
64c10c8
Merge branch 'master' into rittik/avg
rittik9 Jan 13, 2025
9489cad
Merge branch 'master' into rittik/avg
rittik9 Jan 25, 2025
d19a1eb
Merge branch 'master' into rittik/avg
rittik9 Jan 31, 2025
fcb51ee
Merge branch 'master' into rittik/avg
rittik9 Feb 26, 2025
66d22c8
Merge branch 'master' into rittik/avg
rittik9 Apr 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/source/pages/overview.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ of metrics e.g. computation of confidence intervals by resampling of input data.
.. testoutput::
:options: +NORMALIZE_WHITESPACE

{'mean': tensor(0.1333), 'std': tensor(0.1554)}
{'mean': tensor(0.1069), 'std': tensor(0.1180)}

You can see all implemented wrappers under the wrapper section of the API docs.

Expand Down
6 changes: 3 additions & 3 deletions src/torchmetrics/classification/accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ class MulticlassAccuracy(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassAccuracy(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mca = MulticlassAccuracy(num_classes=3, average=None)
>>> mca(preds, target)
tensor([0.5000, 1.0000, 1.0000])
Expand All @@ -228,7 +228,7 @@ class MulticlassAccuracy(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassAccuracy(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mca = MulticlassAccuracy(num_classes=3, average=None)
>>> mca(preds, target)
tensor([0.5000, 1.0000, 1.0000])
Expand All @@ -239,7 +239,7 @@ class MulticlassAccuracy(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassAccuracy(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.5000, 0.2778])
tensor([0.5000, 0.3333])
>>> mca = MulticlassAccuracy(num_classes=3, multidim_average='samplewise', average=None)
>>> mca(preds, target)
tensor([[1.0000, 0.0000, 0.5000],
Expand Down
6 changes: 3 additions & 3 deletions src/torchmetrics/classification/hamming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ class MulticlassHammingDistance(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassHammingDistance(num_classes=3)
>>> metric(preds, target)
tensor(0.1667)
tensor(0.2500)
>>> mchd = MulticlassHammingDistance(num_classes=3, average=None)
>>> mchd(preds, target)
tensor([0.5000, 0.0000, 0.0000])
Expand All @@ -238,7 +238,7 @@ class MulticlassHammingDistance(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassHammingDistance(num_classes=3)
>>> metric(preds, target)
tensor(0.1667)
tensor(0.2500)
>>> mchd = MulticlassHammingDistance(num_classes=3, average=None)
>>> mchd(preds, target)
tensor([0.5000, 0.0000, 0.0000])
Expand All @@ -249,7 +249,7 @@ class MulticlassHammingDistance(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassHammingDistance(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.5000, 0.7222])
tensor([0.5000, 0.6667])
>>> mchd = MulticlassHammingDistance(num_classes=3, multidim_average='samplewise', average=None)
>>> mchd(preds, target)
tensor([[0.0000, 1.0000, 0.5000],
Expand Down
12 changes: 6 additions & 6 deletions src/torchmetrics/classification/negative_predictive_value.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ class MulticlassNegativePredictiveValue(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassNegativePredictiveValue(num_classes=3)
>>> metric(preds, target)
tensor(0.8889)
tensor(0.8750)
>>> metric = MulticlassNegativePredictiveValue(num_classes=3, average=None)
>>> metric(preds, target)
tensor([0.6667, 1.0000, 1.0000])
Expand All @@ -234,7 +234,7 @@ class MulticlassNegativePredictiveValue(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassNegativePredictiveValue(num_classes=3)
>>> metric(preds, target)
tensor(0.8889)
tensor(0.8750)
>>> metric = MulticlassNegativePredictiveValue(num_classes=3, average=None)
>>> metric(preds, target)
tensor([0.6667, 1.0000, 1.0000])
Expand All @@ -245,7 +245,7 @@ class MulticlassNegativePredictiveValue(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassNegativePredictiveValue(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.7833, 0.6556])
tensor([0.7500, 0.6667])
>>> metric = MulticlassNegativePredictiveValue(num_classes=3, multidim_average='samplewise', average=None)
>>> metric(preds, target)
tensor([[1.0000, 0.6000, 0.7500],
Expand Down Expand Up @@ -371,7 +371,7 @@ class MultilabelNegativePredictiveValue(MultilabelStatScores):
>>> preds = tensor([[0, 0, 1], [1, 0, 1]])
>>> metric = MultilabelNegativePredictiveValue(num_labels=3)
>>> metric(preds, target)
tensor(0.5000)
tensor(0.6667)
>>> mls = MultilabelNegativePredictiveValue(num_labels=3, average=None)
>>> mls(preds, target)
tensor([1.0000, 0.5000, 0.0000])
Expand All @@ -382,7 +382,7 @@ class MultilabelNegativePredictiveValue(MultilabelStatScores):
>>> preds = tensor([[0.11, 0.22, 0.84], [0.73, 0.33, 0.92]])
>>> metric = MultilabelNegativePredictiveValue(num_labels=3)
>>> metric(preds, target)
tensor(0.5000)
tensor(0.6667)
>>> mls = MultilabelNegativePredictiveValue(num_labels=3, average=None)
>>> mls(preds, target)
tensor([1.0000, 0.5000, 0.0000])
Expand All @@ -394,7 +394,7 @@ class MultilabelNegativePredictiveValue(MultilabelStatScores):
... [[0.38, 0.04], [0.86, 0.780], [0.45, 0.37]]])
>>> metric = MultilabelNegativePredictiveValue(num_labels=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.0000, 0.1667])
tensor([0.0000, 0.2500])
>>> mls = MultilabelNegativePredictiveValue(num_labels=3, multidim_average='samplewise', average=None)
>>> mls(preds, target)
tensor([[0.0000, 0.0000, 0.0000],
Expand Down
18 changes: 9 additions & 9 deletions src/torchmetrics/classification/precision_recall.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ class MulticlassPrecision(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassPrecision(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mcp = MulticlassPrecision(num_classes=3, average=None)
>>> mcp(preds, target)
tensor([1.0000, 0.5000, 1.0000])
Expand All @@ -251,7 +251,7 @@ class MulticlassPrecision(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassPrecision(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mcp = MulticlassPrecision(num_classes=3, average=None)
>>> mcp(preds, target)
tensor([1.0000, 0.5000, 1.0000])
Expand All @@ -262,7 +262,7 @@ class MulticlassPrecision(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassPrecision(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.3889, 0.2778])
tensor([0.5000, 0.3333])
>>> mcp = MulticlassPrecision(num_classes=3, multidim_average='samplewise', average=None)
>>> mcp(preds, target)
tensor([[0.6667, 0.0000, 0.5000],
Expand Down Expand Up @@ -402,7 +402,7 @@ class MultilabelPrecision(MultilabelStatScores):
>>> preds = tensor([[0, 0, 1], [1, 0, 1]])
>>> metric = MultilabelPrecision(num_labels=3)
>>> metric(preds, target)
tensor(0.5000)
tensor(0.6667)
>>> mlp = MultilabelPrecision(num_labels=3, average=None)
>>> mlp(preds, target)
tensor([1.0000, 0.0000, 0.5000])
Expand All @@ -413,7 +413,7 @@ class MultilabelPrecision(MultilabelStatScores):
>>> preds = tensor([[0.11, 0.22, 0.84], [0.73, 0.33, 0.92]])
>>> metric = MultilabelPrecision(num_labels=3)
>>> metric(preds, target)
tensor(0.5000)
tensor(0.6667)
>>> mlp = MultilabelPrecision(num_labels=3, average=None)
>>> mlp(preds, target)
tensor([1.0000, 0.0000, 0.5000])
Expand All @@ -425,7 +425,7 @@ class MultilabelPrecision(MultilabelStatScores):
... [[0.38, 0.04], [0.86, 0.780], [0.45, 0.37]]])
>>> metric = MultilabelPrecision(num_labels=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.3333, 0.0000])
tensor([0.4000, 0.0000])
>>> mlp = MultilabelPrecision(num_labels=3, multidim_average='samplewise', average=None)
>>> mlp(preds, target)
tensor([[0.5000, 0.5000, 0.0000],
Expand Down Expand Up @@ -696,7 +696,7 @@ class MulticlassRecall(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassRecall(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mcr = MulticlassRecall(num_classes=3, average=None)
>>> mcr(preds, target)
tensor([0.5000, 1.0000, 1.0000])
Expand All @@ -710,7 +710,7 @@ class MulticlassRecall(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassRecall(num_classes=3)
>>> metric(preds, target)
tensor(0.8333)
tensor(0.7500)
>>> mcr = MulticlassRecall(num_classes=3, average=None)
>>> mcr(preds, target)
tensor([0.5000, 1.0000, 1.0000])
Expand All @@ -721,7 +721,7 @@ class MulticlassRecall(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassRecall(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.5000, 0.2778])
tensor([0.5000, 0.3333])
>>> mcr = MulticlassRecall(num_classes=3, multidim_average='samplewise', average=None)
>>> mcr(preds, target)
tensor([[1.0000, 0.0000, 0.5000],
Expand Down
6 changes: 3 additions & 3 deletions src/torchmetrics/classification/specificity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ class MulticlassSpecificity(MulticlassStatScores):
>>> preds = tensor([2, 1, 0, 1])
>>> metric = MulticlassSpecificity(num_classes=3)
>>> metric(preds, target)
tensor(0.8889)
tensor(0.8750)
>>> mcs = MulticlassSpecificity(num_classes=3, average=None)
>>> mcs(preds, target)
tensor([1.0000, 0.6667, 1.0000])
Expand All @@ -228,7 +228,7 @@ class MulticlassSpecificity(MulticlassStatScores):
... [0.05, 0.82, 0.13]])
>>> metric = MulticlassSpecificity(num_classes=3)
>>> metric(preds, target)
tensor(0.8889)
tensor(0.8750)
>>> mcs = MulticlassSpecificity(num_classes=3, average=None)
>>> mcs(preds, target)
tensor([1.0000, 0.6667, 1.0000])
Expand All @@ -239,7 +239,7 @@ class MulticlassSpecificity(MulticlassStatScores):
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> metric = MulticlassSpecificity(num_classes=3, multidim_average='samplewise')
>>> metric(preds, target)
tensor([0.7500, 0.6556])
tensor([0.7500, 0.6667])
>>> mcs = MulticlassSpecificity(num_classes=3, multidim_average='samplewise', average=None)
>>> mcs(preds, target)
tensor([[0.7500, 0.7500, 0.7500],
Expand Down
4 changes: 2 additions & 2 deletions src/torchmetrics/classification/stat_scores.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ def __init__(
self,
num_classes: Optional[int] = None,
top_k: int = 1,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
validate_args: bool = True,
Expand Down Expand Up @@ -462,7 +462,7 @@ def __init__(
self,
num_labels: int,
threshold: float = 0.5,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
validate_args: bool = True,
Expand Down
10 changes: 5 additions & 5 deletions src/torchmetrics/functional/classification/accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ def multiclass_accuracy(
preds: Tensor,
target: Tensor,
num_classes: Optional[int] = None,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
top_k: int = 1,
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
Expand Down Expand Up @@ -234,7 +234,7 @@ def multiclass_accuracy(
>>> target = tensor([2, 1, 0, 0])
>>> preds = tensor([2, 1, 0, 1])
>>> multiclass_accuracy(preds, target, num_classes=3)
tensor(0.8333)
tensor(0.7500)
>>> multiclass_accuracy(preds, target, num_classes=3, average=None)
tensor([0.5000, 1.0000, 1.0000])

Expand All @@ -246,7 +246,7 @@ def multiclass_accuracy(
... [0.71, 0.09, 0.20],
... [0.05, 0.82, 0.13]])
>>> multiclass_accuracy(preds, target, num_classes=3)
tensor(0.8333)
tensor(0.7500)
>>> multiclass_accuracy(preds, target, num_classes=3, average=None)
tensor([0.5000, 1.0000, 1.0000])

Expand All @@ -255,7 +255,7 @@ def multiclass_accuracy(
>>> target = tensor([[[0, 1], [2, 1], [0, 2]], [[1, 1], [2, 0], [1, 2]]])
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> multiclass_accuracy(preds, target, num_classes=3, multidim_average='samplewise')
tensor([0.5000, 0.2778])
tensor([0.5000, 0.3333])
>>> multiclass_accuracy(preds, target, num_classes=3, multidim_average='samplewise', average=None)
tensor([[1.0000, 0.0000, 0.5000],
[0.0000, 0.3333, 0.5000]])
Expand All @@ -276,7 +276,7 @@ def multilabel_accuracy(
target: Tensor,
num_labels: int,
threshold: float = 0.5,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
validate_args: bool = True,
Expand Down
10 changes: 5 additions & 5 deletions src/torchmetrics/functional/classification/hamming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ def multiclass_hamming_distance(
preds: Tensor,
target: Tensor,
num_classes: int,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
top_k: int = 1,
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
Expand Down Expand Up @@ -231,7 +231,7 @@ def multiclass_hamming_distance(
>>> target = tensor([2, 1, 0, 0])
>>> preds = tensor([2, 1, 0, 1])
>>> multiclass_hamming_distance(preds, target, num_classes=3)
tensor(0.1667)
tensor(0.2500)
>>> multiclass_hamming_distance(preds, target, num_classes=3, average=None)
tensor([0.5000, 0.0000, 0.0000])

Expand All @@ -243,7 +243,7 @@ def multiclass_hamming_distance(
... [0.71, 0.09, 0.20],
... [0.05, 0.82, 0.13]])
>>> multiclass_hamming_distance(preds, target, num_classes=3)
tensor(0.1667)
tensor(0.2500)
>>> multiclass_hamming_distance(preds, target, num_classes=3, average=None)
tensor([0.5000, 0.0000, 0.0000])

Expand All @@ -252,7 +252,7 @@ def multiclass_hamming_distance(
>>> target = tensor([[[0, 1], [2, 1], [0, 2]], [[1, 1], [2, 0], [1, 2]]])
>>> preds = tensor([[[0, 2], [2, 0], [0, 1]], [[2, 2], [2, 1], [1, 0]]])
>>> multiclass_hamming_distance(preds, target, num_classes=3, multidim_average='samplewise')
tensor([0.5000, 0.7222])
tensor([0.5000, 0.6667])
>>> multiclass_hamming_distance(preds, target, num_classes=3, multidim_average='samplewise', average=None)
tensor([[0.0000, 1.0000, 0.5000],
[1.0000, 0.6667, 0.5000]])
Expand All @@ -273,7 +273,7 @@ def multilabel_hamming_distance(
target: Tensor,
num_labels: int,
threshold: float = 0.5,
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "macro",
average: Optional[Literal["micro", "macro", "weighted", "none"]] = "micro",
multidim_average: Literal["global", "samplewise"] = "global",
ignore_index: Optional[int] = None,
validate_args: bool = True,
Expand Down
Loading
Loading