Skip to content

Commit 80df874

Browse files
committed
repoqa: update new gemini 1.5 pro and deepseek chat v2
1 parent 8fa9e66 commit 80df874

File tree

3 files changed

+4867
-258
lines changed

3 files changed

+4867
-258
lines changed

repoqa.html

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ <h3 id="limit" class="text-nowrap mt-5">Known limitations</h3>
349349
"codellama_slash_CodeLlama-7b-Instruct-hf-SCORES.json",
350350
"deepseek-ai_slash_deepseek-coder-33b-instruct-SCORES.json",
351351
"deepseek-ai_slash_deepseek-coder-6.7b-instruct-SCORES.json",
352+
"deepseek-ai_slash_DeepSeek-V2-Chat-SCORES.json",
352353
"google_slash_codegemma-7b-it-SCORES.json",
353354
"meta-llama_slash_Meta-Llama-3-70B-Instruct-SCORES.json",
354355
"meta-llama_slash_Meta-Llama-3-8B-Instruct-SCORES.json",
@@ -484,11 +485,21 @@ <h3 id="limit" class="text-nowrap mt-5">Known limitations</h3>
484485
var tbody = document.createElement("tbody");
485486
// add rank
486487
var rank = 0;
488+
var last_best = null;
489+
var n_last_best = 1;
487490
data.forEach((row) => {
488491
var dataRow = document.createElement("tr");
489492
// rank
490493
var rankCell = document.createElement("td");
491-
rankCell.textContent = ++rank;
494+
var cur_score = row["scores"]["all"][scoreThresh]["pass@1"];
495+
if (last_best != cur_score) {
496+
rank += n_last_best;
497+
last_best = cur_score;
498+
n_last_best = 1;
499+
} else {
500+
n_last_best += 1;
501+
}
502+
rankCell.textContent = rank;
492503
dataRow.appendChild(rankCell);
493504

494505
// model name

0 commit comments

Comments
 (0)