Add MRR metric (#5097)

* add MRR to metrics

* add MRR to metrics

* add MRR to metrics

* add MRR to metrics

* add MRR to metrics

Author: wuxin-555

paddlenlp/metrics/__init__.py

@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .perplexity import Perplexity
-from .chunk import ChunkEvaluator
 from .bleu import BLEU, BLEUForDuReader
-from .rouge import RougeL, RougeLForDuReader, RougeN, Rouge1, Rouge2
-from .glue import AccuracyAndF1, Mcc, PearsonAndSpearman, MultiLabelsMetric
+from .chunk import ChunkEvaluator
 from .distinct import Distinct
-from .sighan import DetectionF1, CorrectionF1
+from .glue import AccuracyAndF1, Mcc, MultiLabelsMetric, PearsonAndSpearman
+from .mrr import MRR
+from .perplexity import Perplexity
+from .rouge import Rouge1, Rouge2, RougeL, RougeLForDuReader, RougeN
+from .sighan import CorrectionF1, DetectionF1
 from .span import SpanEvaluator

paddlenlp/metrics/mrr.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+from sklearn.metrics import pairwise_distances
+
+__all__ = ["MRR"]
+
+
+class MRR:
+    """
+    MRR - Mean Reciprocal Rank, is a popular metric for recommend system
+    and other retrival task. The higher mrr is, the better performance of
+    model in retrival task.
+
+    Args:
+        distance: which algorithm to use to get distance of embeddings, for example: "cosine", "euclidean"
+
+    """
+
+    def __init__(self, distance="cosine"):
+        super().__init__()
+        self.distance = distance
+
+    def reset_distance(self, distance):
+        """
+        change the algorithm of calculating distance, need to be supported of sklearn.metrics.pairwise_distance
+        """
+        self.distance = distance
+
+    def compute_matrix_mrr(self, labels, embeddings):
+        """
+        A function which can calculate the distance of one embedding to other embeddings
+        in the matrix, and then it can find the most similar embedding's index to calculate
+        the mrr metric for this one embedding. After getting all the embeddings' mrr metric,
+        a mean pool is used to get the final mrr metric for input matrix.
+
+        Param:
+          - labels(np.array): label matrix, shape=[size, ]
+          - embeddings(np.array): embedding matrix, shape=[size, emb_dim]
+
+        Return:
+            mrr metric for input embedding matrix.
+        """
+        matrix_size = labels.shape[0]
+        if labels.shape[0] != embeddings.shape[0]:
+            raise Exception("label and embedding matrix must have same size at dim=0 !")
+        row_mrr = []  # mrr metric for each embedding of matrix
+        for i in range(0, matrix_size):
+            emb, label = embeddings[i, :], labels[i]
+            dists = pairwise_distances(emb.reshape(1, -1), embeddings, metric=self.distance).reshape(-1)
+            ranks_ids = np.argsort(dists)[1:]
+            ranks = (labels[ranks_ids] == label).astype(int)
+            ranks_nonzero_ids = ranks.nonzero()[0]
+            row_mrr.append(1.0 / (1 + ranks_nonzero_ids[0]) if ranks_nonzero_ids.size else 0.0)
+        mrr = np.mean(row_mrr)  # user mean value as final mrr metric for the matrix.
+        return mrr

tests/metrics/test_mrr.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import unittest
+
+import numpy as np
+
+from paddlenlp.metrics import MRR
+from tests.common_test import CommonTest
+
+
+class TestMRR(CommonTest):
+    def setUp(self):
+        self.distance = "cosine"
+        self.mrr = MRR(distance=self.distance)
+        self.label_num = 10
+        self.label_shape = (20,)
+        self.embedding_shape = (20, 128)
+
+    def get_random_case(self):
+        labels = np.random.randint(0, self.label_num, size=self.label_shape).astype("int64")
+        embeddings = np.random.uniform(0.1, 1.0, self.embedding_shape).astype("float64")
+        all_distance = ["cityblock", "cosine", "euclidean", "l1", "l2", "manhattan"]
+        distance = random.choice(all_distance)
+        return labels, embeddings, distance, all_distance
+
+    def get_true_mrr_case(self):
+        labels = np.array([1, 2, 1]).astype("int64")
+        embeddings = np.array(
+            [
+                # cosine similarity: 1,2 => 0.991; 1,3=>0.851; 2,3=>0.912
+                [1.0, 2.0, 3.0],
+                [1.0, 2.0, 4.0],
+                [1.0, 100.0, 1000.0],
+            ]
+        )
+        distance = "cosine"
+        true_mrr = (1.0 / 2 + 0 + 1.0 / 2) / 3
+        return labels, embeddings, distance, true_mrr
+
+    def test_reset_distance(self):
+        _, _, distance, _ = self.get_random_case()
+        self.mrr.reset_distance(distance)
+        self.check_output_equal(self.mrr.distance, distance)
+
+    def test_compute_matrix_mrr(self):
+        step = 100
+        for i in range(step):
+            labels, embeddings, distance, _ = self.get_random_case()
+            self.mrr.reset_distance(distance)
+            self.mrr.compute_matrix_mrr(labels, embeddings)
+
+    def test_compute_true_mrr(self):
+        labels, embeddings, distance, true_mrr = self.get_true_mrr_case()
+        self.mrr.reset_distance(distance)
+        mrr = self.mrr.compute_matrix_mrr(labels, embeddings)
+        self.check_output_equal(mrr, true_mrr)
+
+
+if __name__ == "__main__":
+    unittest.main()