diff --git a/evals/registry/data/ice_linguistic_benchmark/.gitattributes b/evals/registry/data/ice_linguistic_benchmark/.gitattributes new file mode 100644 index 0000000000..8c9c4f1c78 --- /dev/null +++ b/evals/registry/data/ice_linguistic_benchmark/.gitattributes @@ -0,0 +1 @@ +*.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/evals/registry/data/ice_linguistic_benchmark/samples.jsonl b/evals/registry/data/ice_linguistic_benchmark/samples.jsonl new file mode 100644 index 0000000000..4bf76ebcda --- /dev/null +++ b/evals/registry/data/ice_linguistic_benchmark/samples.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3fb67121942524c2945b1957ba47870c62059089065368979ec1ffc1445102 +size 276367 diff --git a/evals/registry/evals/ice_linguistic_benchmark.yaml b/evals/registry/evals/ice_linguistic_benchmark.yaml new file mode 100644 index 0000000000..1c6a7f5222 --- /dev/null +++ b/evals/registry/evals/ice_linguistic_benchmark.yaml @@ -0,0 +1,8 @@ +ice_linguistic_benchmark: + id: ice_linguistic_benchmark.dev.v0 + metrics: [match] + description: A benchmarking data set to evaluate LLMs’ grammatical knowledge and linguistic ability for Icelandic. +ice_linguistic_benchmark.dev.v0: + class: evals.elsuite.basic.match:Match + args: + samples_jsonl: ice_linguistic_benchmark/samples.jsonl