diff --git a/evals/registry/data/icelandic-qa-news/samples.jsonl b/evals/registry/data/icelandic-qa-news/samples.jsonl new file mode 100644 index 0000000000..070ed7ddcb --- /dev/null +++ b/evals/registry/data/icelandic-qa-news/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8957305229c180fe99c10df6463c754500e63f87739e8aee11d556b2fac8965 +size 29436 diff --git a/evals/registry/evals/icelandic-qa-news.yaml b/evals/registry/evals/icelandic-qa-news.yaml new file mode 100644 index 0000000000..8433283189 --- /dev/null +++ b/evals/registry/evals/icelandic-qa-news.yaml @@ -0,0 +1,9 @@ +icelandic-qa-news: + id: icelandic-qa-news.dev.v0 + metrics: [accuracy] + description: Test the model's ability to answer Icelandic questions on Icelandic culture and history, created from Icelandic online news. + +icelandic-qa-news.dev.v0: + class: evals.elsuite.basic.fuzzy_match:FuzzyMatch + args: + samples_jsonl: icelandic-qa-news/samples.jsonl \ No newline at end of file