Merge branch 'main' into main

Author: NathanHB

.github/workflows/doc-pr-upload.yml

@@ -13,4 +13,5 @@ jobs:
       package_name: lighteval
     secrets:
       hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
-      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}
+      comment_bot_app_id: ${{ secrets.COMMENT_BOT_APP_ID }}
+      comment_bot_secret_pem: ${{ secrets.COMMENT_BOT_SECRET_PEM }}

.github/workflows/pr_style_bot.yaml

@@ -5,105 +5,12 @@ on:
     types: [created]
 
 permissions:
-  contents: write
   pull-requests: write
 
 jobs:
-  run-style-bot:
-    if: >
-      contains(github.event.comment.body, '@bot /style') &&
-      github.event.issue.pull_request != null
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Extract PR details
-        id: pr_info
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const prNumber = context.payload.issue.number;
-            const { data: pr } = await github.rest.pulls.get({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: prNumber
-            });
-
-            // We capture both the branch ref and the "full_name" of the head repo
-            // so that we can check out the correct repository & branch (including forks).
-            core.setOutput("prNumber", prNumber);
-            core.setOutput("headRef", pr.head.ref);
-            core.setOutput("headRepoFullName", pr.head.repo.full_name);
-      - name: Check out PR branch
-        uses: actions/checkout@v3
-        env:
-          HEADREPOFULLNAME: ${{ steps.pr_info.outputs.headRepoFullName }}
-          HEADREF: ${{ steps.pr_info.outputs.headRef }}
-        with:
-          # Instead of checking out the base repo, use the contributor's repo name
-          repository: ${{ env.HEADREPOFULLNAME }}
-          ref: ${{ env.HEADREF }}
-          # You may need fetch-depth: 0 for being able to push
-          fetch-depth: 0
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Debug
-        env:
-          HEADREPOFULLNAME: ${{ steps.pr_info.outputs.headRepoFullName }}
-          HEADREF: ${{ steps.pr_info.outputs.headRef }}
-          PRNUMBER: ${{ steps.pr_info.outputs.prNumber }}
-        run: |
-          echo "PR number: ${{ env.PRNUMBER }}"
-          echo "Head Ref: ${{ env.HEADREF }}"
-          echo "Head Repo Full Name: ${{ env.HEADREPOFULLNAME }}"
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-      - name: Install dependencies
-        run: |
-          pip install .[quality]
-
-      - name: Run ruff format --check
-        run: |
-          ruff format .
-          ruff check --fix .
-      - name: Commit and push changes
-        id: commit_and_push
-        env:
-          HEADREPOFULLNAME: ${{ steps.pr_info.outputs.headRepoFullName }}
-          HEADREF: ${{ steps.pr_info.outputs.headRef }}
-          PRNUMBER: ${{ steps.pr_info.outputs.prNumber }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          echo "HEADREPOFULLNAME: ${{ env.HEADREPOFULLNAME }}, HEADREF: ${{ env.HEADREF }}"
-          # Configure git with the Actions bot user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          # Make sure your 'origin' remote is set to the contributor's fork
-          git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${{ env.HEADREPOFULLNAME }}.git"
-          # If there are changes after running style/quality, commit them
-          if [ -n "$(git status --porcelain)" ]; then
-            git add .
-            git commit -m "Apply style fixes"
-            # Push to the original contributor's forked branch
-            git push origin HEAD:${{ env.HEADREF }}
-            echo "changes_pushed=true" >> $GITHUB_OUTPUT
-          else
-            echo "No changes to commit."
-            echo "changes_pushed=false" >> $GITHUB_OUTPUT
-          fi
-      - name: Comment on PR with workflow run link
-        if: steps.commit_and_push.outputs.changes_pushed == 'true'
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const prNumber = parseInt(process.env.prNumber, 10);
-            const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              body: `Style fixes have been applied. [View the workflow run here](${runUrl}).`
-            });
-        env:
-          prNumber: ${{ steps.pr_info.outputs.prNumber }}
+  style:
+    uses: huggingface/huggingface_hub/.github/workflows/style-bot-action.yml@main
+    with:
+      python_quality_dependencies: "[quality]"
+    secrets:
+      bot_token: ${{ secrets.HF_STYLE_BOT_ACTION }}

README.md

@@ -77,6 +77,7 @@ you need, or, here's an overview of some *popular benchmarks*:
   - **Turkic**: TUMLU (9 Turkic languages)
   - **Chinese**: CMMLU, CEval, AGIEval
   - **Russian**: RUMMLU, Russian SQuAD
+  - **Kyrgyz**: Kyrgyz LLM Benchmark
   - **And many more...**
 
 ### 🧠 **Core Language Understanding**
@@ -124,7 +125,7 @@ Lighteval offers the following entry points for model evaluation:
 Did not find what you need ? You can always make your custom model API by following [this guide](https://huggingface.co/docs/lighteval/main/en/evaluating-a-custom-model)
 - `lighteval custom`: Evaluate custom models (can be anything)
 
-Here's a **quick command** to evaluate using the *Accelerate backend*:
+Here's a **quick command** to evaluate using a remote inference service:
 
 ```shell
 lighteval eval "hf-inference-providers/openai/gpt-oss-20b" gpqa:diamond
@@ -186,7 +187,7 @@ If you're adding a **new feature**, please *open an issue first*.
 If you open a PR, don't forget to **run the styling**!
 
 ```bash
-pip install -e .[dev]
+pip install -e ".[dev]"
 pre-commit install
 pre-commit run --all-files
 ```

docs/source/available-tasks.mdx

@@ -2,7 +2,7 @@
 
 Browse and inspect tasks available in LightEval.
 <iframe
-	src="https://openevals-benchmark-finder.hf.space"
+	src="https://openevals-open-benchmark-index.hf.space"
 	frameborder="0"
 	width="850"
 	height="450"
@@ -16,6 +16,18 @@ List all tasks:
 lighteval tasks list
 ```
 
+Extract tasks details:
+
+```bash
+lighteval tasks dump
+```
+
+Store the tasks details in a JSON file:
+
+```bash
+lighteval tasks dump > tasks.json
+```
+
 ### Inspect specific tasks
 
 Inspect a task to view its config, metrics, and requirements:

docs/source/index.mdx

@@ -9,7 +9,7 @@ and see how your models stack up.
 
 ### 🚀 **Multi-Backend Support**
 Evaluate your models using the most popular and efficient inference backends:
-- `eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models ! (prefered way)
+- `eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models! (prefered way)
 - `transformers`: Evaluate models on CPU or one or more GPUs using [🤗
   Accelerate](https://github.com/huggingface/transformers)
 - `nanotron`: Evaluate models in distributed settings using [⚡️

src/lighteval/main_inspect.py

@@ -565,4 +565,4 @@ def bundle(log_dir: str, output_dir: str, overwrite: bool = True, repo_id: str |
         "tiny_benchmarks",
     ]
     model = "hf-inference-providers/meta-llama/Llama-3.1-8B-Instruct:nebius"
-    eval(models=[model], tasks=task)
+    eval(models=[model], tasks=tasks[0])