diff --git a/evals/registry/data/icelandic-qa-wiki/samples.jsonl b/evals/registry/data/icelandic-qa-wiki/samples.jsonl new file mode 100644 index 0000000000..996057df4e --- /dev/null +++ b/evals/registry/data/icelandic-qa-wiki/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e91ae91f162e33782627e78d316b34b4ae71ed672f73e91e8f5879bd38150102 +size 532374 diff --git a/evals/registry/evals/icelandic-qa-wiki.yaml b/evals/registry/evals/icelandic-qa-wiki.yaml new file mode 100644 index 0000000000..9247039c8d --- /dev/null +++ b/evals/registry/evals/icelandic-qa-wiki.yaml @@ -0,0 +1,9 @@ +icelandic-qa-wiki: + id: icelandic-qa-wiki.dev.v0 + metrics: [accuracy] + description: Test the model's ability to answer Icelandic questions on Icelandic culture and history, created from the Icelandic Wikipedia. + +icelandic-qa-wiki.dev.v0: + class: evals.elsuite.basic.fuzzy_match:FuzzyMatch + args: + samples_jsonl: icelandic-qa-wiki/samples.jsonl \ No newline at end of file