Merge pull request #93 from sbintuitions/feat/jmteb_v2

[Feature] JMTEB v2リリースに伴うリーダーボード更新&対応モデル追加

Author: lsz05

docs/results/BAAI/bge-m3/summary.json

@@ -0,0 +1,96 @@
+{
+    "Classification": {
+        "amazon_counterfactual_classification": {
+            "macro_f1": 0.718621425743256
+        },
+        "amazon_review_classification": {
+            "macro_f1": 0.5664555524508175
+        },
+        "japanese_sentiment_classification": {
+            "macro_f1": 0.9441075327867781
+        },
+        "massive_intent_classification": {
+            "macro_f1": 0.7868184551588373
+        },
+        "massive_scenario_classification": {
+            "macro_f1": 0.8970320222457714
+        },
+        "sib200_japanese_classification": {
+            "macro_f1": 0.8424907003170607
+        },
+        "wrime_classification": {
+            "macro_f1": 0.4316630478439933
+        }
+    },
+    "Reranking": {
+        "esci": {
+            "ndcg@10": 0.9327323748768209
+        },
+        "jacwir_reranking": {
+            "ndcg@10": 0.8955144849023412
+        },
+        "jqara": {
+            "ndcg@10": 0.5391637817603238
+        },
+        "miracl_reranking": {
+            "ndcg@10": 0.8596271423829606
+        },
+        "mldr_reranking": {
+            "ndcg@10": 0.9778261029468881
+        }
+    },
+    "Retrieval": {
+        "jacwir_retrieval": {
+            "ndcg@10": 0.851348898788452
+        },
+        "jagovfaqs_22k": {
+            "ndcg@10": 0.6906829361885021
+        },
+        "jaqket": {
+            "ndcg@10": 0.5659460589444328
+        },
+        "mintaka_retrieval": {
+            "ndcg@10": 0.32175483024897333
+        },
+        "miracl_retrieval": {
+            "ndcg@10": 0.734809783755516
+        },
+        "mldr_retrieval": {
+            "ndcg@10": 0.5126063501865914
+        },
+        "mrtydi": {
+            "ndcg@10": 0.45179452203971654
+        },
+        "nlp_journal_abs_article": {
+            "ndcg@10": 0.9521915103722084
+        },
+        "nlp_journal_abs_intro": {
+            "ndcg@10": 0.9752948774973371
+        },
+        "nlp_journal_title_abs": {
+            "ndcg@10": 0.9602075886902439
+        },
+        "nlp_journal_title_intro": {
+            "ndcg@10": 0.9197525363243463
+        }
+    },
+    "STS": {
+        "jsick": {
+            "spearman": 0.7926524802982091
+        },
+        "jsts": {
+            "spearman": 0.8020865982595183
+        }
+    },
+    "Clustering": {
+        "livedoor_news": {
+            "v_measure_score": 0.5475619174246511
+        },
+        "mewsc16": {
+            "v_measure_score": 0.4200457612686986
+        },
+        "sib200_japanese_clustering": {
+            "v_measure_score": 0.3991288954568376
+        }
+    }
+}

docs/results/MU-Kindai/Japanese-DiffCSE-BERT-base/summary.json

@@ -1,62 +1,96 @@
 {
     "Classification": {
         "amazon_counterfactual_classification": {
-            "macro_f1": 0.7809527709426081
+            "macro_f1": 0.7769528027441275
         },
         "amazon_review_classification": {
-            "macro_f1": 0.5155899232320224
+            "macro_f1": 0.5146406875677701
+        },
+        "japanese_sentiment_classification": {
+            "macro_f1": 0.8844781754440035
         },
         "massive_intent_classification": {
-            "macro_f1": 0.7879373479249787
+            "macro_f1": 0.7872353730798753
         },
         "massive_scenario_classification": {
-            "macro_f1": 0.8662625888023707
+            "macro_f1": 0.8639715373498098
+        },
+        "sib200_japanese_classification": {
+            "macro_f1": 0.8350488266987821
+        },
+        "wrime_classification": {
+            "macro_f1": 0.3815230965003785
         }
     },
     "Reranking": {
         "esci": {
-            "ndcg@10": 0.9095168116460639
+            "ndcg@10": 0.909518320556229
+        },
+        "jacwir_reranking": {
+            "ndcg@10": 0.5981293078380808
+        },
+        "jqara": {
+            "ndcg@10": 0.3719557553111225
+        },
+        "miracl_reranking": {
+            "ndcg@10": 0.6789908587925922
+        },
+        "mldr_reranking": {
+            "ndcg@10": 0.8281088898171538
         }
     },
     "Retrieval": {
+        "jacwir_retrieval": {
+            "ndcg@10": 0.4085978545476503
+        },
         "jagovfaqs_22k": {
-            "ndcg@10": 0.42314124780036416
+            "ndcg@10": 0.43879890119990833
         },
         "jaqket": {
-            "ndcg@10": 0.36199154051747723
+            "ndcg@10": 0.3555985699236658
+        },
+        "mintaka_retrieval": {
+            "ndcg@10": 0.1997740482697841
+        },
+        "miracl_retrieval": {
+            "ndcg@10": 0.16521386136598404
+        },
+        "mldr_retrieval": {
+            "ndcg@10": 0.12060735418211223
         },
         "mrtydi": {
-            "ndcg@10": 0.07810683176415421
+            "ndcg@10": 0.07107405961190999
+        },
+        "nlp_journal_abs_article": {
+            "ndcg@10": 0.5430415601583998
         },
         "nlp_journal_abs_intro": {
-            "ndcg@10": 0.6077212544951452
+            "ndcg@10": 0.5585881454407594
         },
         "nlp_journal_title_abs": {
-            "ndcg@10": 0.6433890489201118
+            "ndcg@10": 0.629620778788499
         },
         "nlp_journal_title_intro": {
-            "ndcg@10": 0.39317174536190913
+            "ndcg@10": 0.3517328767423871
         }
     },
     "STS": {
         "jsick": {
-            "spearman": 0.754165277432144
+            "spearman": 0.7775668305928584
         },
         "jsts": {
-            "spearman": 0.7558202366183716
+            "spearman": 0.7563460117163054
         }
     },
     "Clustering": {
         "livedoor_news": {
-            "v_measure_score": 0.4966545453348478
+            "v_measure_score": 0.4601335671191492
         },
         "mewsc16": {
-            "v_measure_score": 0.3877356318022785
-        }
-    },
-    "PairClassification": {
-        "paws_x_ja": {
-            "binary_f1": 0.6237623762376237
+            "v_measure_score": 0.3922006290468797
+        },
+        "sib200_japanese_clustering": {
+            "v_measure_score": 0.3456006554316726
         }
     }
 }

docs/results/MU-Kindai/Japanese-MixCSE-BERT-base/summary.json

@@ -1,62 +1,96 @@
 {
     "Classification": {
         "amazon_counterfactual_classification": {
-            "macro_f1": 0.776174162517931
+            "macro_f1": 0.7779156199278396
         },
         "amazon_review_classification": {
-            "macro_f1": 0.5085781180553806
+            "macro_f1": 0.5111451768867725
+        },
+        "japanese_sentiment_classification": {
+            "macro_f1": 0.8782111274457993
         },
         "massive_intent_classification": {
-            "macro_f1": 0.7718541530739129
+            "macro_f1": 0.7796973463634825
         },
         "massive_scenario_classification": {
-            "macro_f1": 0.8592571786794985
+            "macro_f1": 0.8634142669499835
+        },
+        "sib200_japanese_classification": {
+            "macro_f1": 0.8506408877596591
+        },
+        "wrime_classification": {
+            "macro_f1": 0.3656175961601361
         }
     },
     "Reranking": {
         "esci": {
-            "ndcg@10": 0.9100551950168166
+            "ndcg@10": 0.9092446252246911
+        },
+        "jacwir_reranking": {
+            "ndcg@10": 0.605113846464576
+        },
+        "jqara": {
+            "ndcg@10": 0.36840730960684165
+        },
+        "miracl_reranking": {
+            "ndcg@10": 0.693114284522583
+        },
+        "mldr_reranking": {
+            "ndcg@10": 0.8530771666734125
         }
     },
     "Retrieval": {
+        "jacwir_retrieval": {
+            "ndcg@10": 0.42431895793525753
+        },
         "jagovfaqs_22k": {
-            "ndcg@10": 0.42368135774043536
+            "ndcg@10": 0.43601956332213093
         },
         "jaqket": {
-            "ndcg@10": 0.37721850397542034
+            "ndcg@10": 0.37354035206874886
+        },
+        "mintaka_retrieval": {
+            "ndcg@10": 0.2518443007449429
+        },
+        "miracl_retrieval": {
+            "ndcg@10": 0.14756204576714857
+        },
+        "mldr_retrieval": {
+            "ndcg@10": 0.16862391555076126
         },
         "mrtydi": {
-            "ndcg@10": 0.07878085186566607
+            "ndcg@10": 0.07770347901718931
+        },
+        "nlp_journal_abs_article": {
+            "ndcg@10": 0.5689006657309228
         },
         "nlp_journal_abs_intro": {
-            "ndcg@10": 0.636999375405723
+            "ndcg@10": 0.5911474254499767
         },
         "nlp_journal_title_abs": {
-            "ndcg@10": 0.6413498649875696
+            "ndcg@10": 0.618101892252404
         },
         "nlp_journal_title_intro": {
-            "ndcg@10": 0.397250919496823
+            "ndcg@10": 0.3287673013916751
         }
     },
     "STS": {
         "jsick": {
-            "spearman": 0.7756925231422259
+            "spearman": 0.7893346270810556
         },
         "jsts": {
-            "spearman": 0.7652968548841591
+            "spearman": 0.7657111966582518
         }
     },
     "Clustering": {
         "livedoor_news": {
-            "v_measure_score": 0.5262387436934941
+            "v_measure_score": 0.48558605187442483
         },
         "mewsc16": {
-            "v_measure_score": 0.37277574537292835
-        }
-    },
-    "PairClassification": {
-        "paws_x_ja": {
-            "binary_f1": 0.623321554770318
+            "v_measure_score": 0.4319848997472401
+        },
+        "sib200_japanese_clustering": {
+            "v_measure_score": 0.3860004176729398
         }
     }
 }