[Refactor] Code refactoarization

.pre-commit-config-zh-cn.yaml

@@ -1,6 +1,7 @@
 exclude: |
     (?x)^(
       tests/data/|
+      tests/dataset/|
       opencompass/models/internal/|
       opencompass/utils/internal/|
       opencompass/openicl/icl_evaluator/hf_metrics/|
@@ -10,12 +11,9 @@ exclude: |
       opencompass/datasets/teval/|
       opencompass/datasets/NPHardEval/|
       opencompass/datasets/TheoremQA|
+      opencompass/datasets/subjective/mtbench101.py|
       docs/zh_cn/advanced_guides/compassbench_intro.md |
       docs/zh_cn/advanced_guides/compassbench_v2_0.md |
-      opencompass/configs/datasets/ |
-      opencompass/configs/models/|
-      opencompass/configs/summarizers/|
-      opencompass/configs/dataset_collections/ |
       opencompass/utils/datasets.py |
       opencompass/utils/datasets_info.py
     )
@@ -26,26 +24,26 @@ repos:
       - id: flake8
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://gitee.com/openmmlab/mirrors-isort
     rev: 5.11.5
     hooks:
       - id: isort
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://gitee.com/openmmlab/mirrors-yapf
     rev: v0.32.0
     hooks:
       - id: yapf
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://gitee.com/openmmlab/mirrors-codespell
     rev: v2.2.1
@@ -55,9 +53,8 @@ repos:
             (?x)^(
                 .*\.jsonl|
                 .*\.md.template|
-                configs/ |
                 opencompass/configs/ |
-                example_scripts/
+                examples/
             )
   - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks
     rev: v4.3.0
@@ -67,15 +64,13 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
         exclude: |
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
@@ -107,7 +102,7 @@ repos:
         language: script
         pass_filenames: true
         require_serial: true
-        files: ^configs/datasets
+        files: ^opencompass/configs/datasets
   - repo: local
     hooks:
     -   id: update-dataset-suffix-pacakge
@@ -120,41 +115,6 @@ repos:
         args:
           - --root_folder
           - opencompass/configs/datasets
-  - repo: local
-    hooks:
-    -   id: compare-configs-datasets
-        name: compare configs datasets
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/datasets
-          - opencompass/configs/datasets
-  - repo: local
-    hooks:
-    -   id: compare-configs-models
-        name: compare configs models
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/models
-          - opencompass/configs/models
-          - --ignore
-          - llama
-  - repo: local
-    hooks:
-    -   id: compare-configs-summarizers
-        name: compare configs summarizers
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/summarizers
-          - opencompass/configs/summarizers
   # - repo: https://github.com/open-mmlab/pre-commit-hooks
   #   rev: v0.2.0  # Use the ref you want to point at
   #   hooks:

.pre-commit-config.yaml

@@ -14,10 +14,6 @@ exclude: |
       opencompass/datasets/subjective/mtbench101.py|
       docs/zh_cn/advanced_guides/compassbench_intro.md |
       docs/zh_cn/advanced_guides/compassbench_v2_0.md |
-      opencompass/configs/datasets/ |
-      opencompass/configs/models/|
-      opencompass/configs/summarizers/ |
-      opencompass/configs/dataset_collections/ |
       opencompass/utils/datasets.py |
       opencompass/utils/datasets_info.py
     )
@@ -28,26 +24,26 @@ repos:
       - id: flake8
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://github.com/PyCQA/isort
     rev: 5.11.5
     hooks:
       - id: isort
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://github.com/pre-commit/mirrors-yapf
     rev: v0.32.0
     hooks:
       - id: yapf
         exclude: |
             (?x)^(
-                configs/ |
-                example_scripts/
+                opencompass/configs/|
+                examples/
             )
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.1
@@ -57,9 +53,8 @@ repos:
             (?x)^(
                 .*\.jsonl|
                 .*\.md.template|
-                configs/ |
                 opencompass/configs/ |
-                example_scripts/
+                examples/
             )
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
@@ -69,15 +64,13 @@ repos:
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/.*?/.*\.txt
             )
       - id: check-yaml
       - id: end-of-file-fixer
         exclude: |
             (?x)^(
               dicts/|
               projects/.*?/dicts/|
-              configs/.*?/.*\.txt
             )
       - id: requirements-txt-fixer
       - id: double-quote-string-fixer
@@ -109,7 +102,7 @@ repos:
         language: script
         pass_filenames: true
         require_serial: true
-        files: ^configs/datasets
+        files: ^opencompass/configs/datasets
   - repo: local
     hooks:
     -   id: update-dataset-suffix-pacakge
@@ -122,42 +115,6 @@ repos:
         args:
           - --root_folder
           - opencompass/configs/datasets
-  - repo: local
-    hooks:
-    -   id: compare-configs-datasets
-        name: compare configs datasets
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/datasets
-          - opencompass/configs/datasets
-  - repo: local
-    hooks:
-    -   id: compare-configs-models
-        name: compare configs models
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/models
-          - opencompass/configs/models
-          - --ignore
-          - llama
-  - repo: local
-    hooks:
-    -   id: compare-configs-summarizers
-        name: compare configs summarizers
-        entry: ./tools/compare_configs.py
-        language: script
-        pass_filenames: false
-        # require_serial: true
-        args:
-          - configs/summarizers
-          - opencompass/configs/summarizers
-
   # - repo: https://github.com/open-mmlab/pre-commit-hooks
   #   rev: v0.2.0  # Use the ref you want to point at
   #   hooks:

README.md

@@ -182,10 +182,10 @@ After ensuring that OpenCompass is installed correctly according to the above st
   opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
 
   # Python scripts
-  opencompass ./configs/eval_chat_demo.py
+  opencompass examples/eval_chat_demo.py
   ```
 
-  You can find more script examples under [configs](./configs) folder.
+  You can find more script examples under [examples](./examples) folder.
 
 - API evaluation
 
@@ -197,7 +197,7 @@ After ensuring that OpenCompass is installed correctly according to the above st
   opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
 
   # Python scripts
-  opencompass ./configs/eval_api_demo.py
+  opencompass examples/eval_api_demo.py
 
   # You can use o1_mini_2024_09_12/o1_preview_2024_09_12  for o1 models, we set max_completion_tokens=8192 as default.
   ```
@@ -211,7 +211,7 @@ After ensuring that OpenCompass is installed correctly according to the above st
   opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen -a lmdeploy
 
   # Python scripts
-  opencompass ./configs/eval_lmdeploy_demo.py
+  opencompass examples/eval_lmdeploy_demo.py
   ```
 
 - Supported Models

README_zh-CN.md

@@ -169,17 +169,17 @@ humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ce
 
 - ### 首次评测
 
-  OpenCompass 支持通过命令行界面 (CLI) 或 Python 脚本来设置配置。对于简单的评估设置，我们推荐使用 CLI；而对于更复杂的评估，则建议使用脚本方式。你可以在configs文件夹下找到更多脚本示例。
+  OpenCompass 支持通过命令行界面 (CLI) 或 Python 脚本来设置配置。对于简单的评估设置，我们推荐使用 CLI；而对于更复杂的评估，则建议使用脚本方式。你可以在examples文件夹下找到更多脚本示例。
 
   ```bash
   # 命令行界面 (CLI)
   opencompass --models hf_internlm2_5_1_8b_chat --datasets demo_gsm8k_chat_gen
 
   # Python 脚本
-  opencompass ./configs/eval_chat_demo.py
+  opencompass examples/eval_chat_demo.py
   ```
 
-  你可以在[configs](./configs) 文件夹下找到更多的脚本示例。
+  你可以在[examples](./examples) 文件夹下找到更多的脚本示例。
 
 - ### API评测
 
@@ -191,7 +191,7 @@ humaneval, triviaqa, commonsenseqa, tydiqa, strategyqa, cmmlu, lambada, piqa, ce
   opencompass --models gpt_4o_2024_05_13 --datasets demo_gsm8k_chat_gen
 
   # Python 脚本
-  opencompass  ./configs/eval_api_demo.py
+  opencompass  examples/eval_api_demo.py
 
 
   # 现已支持 o1_mini_2024_09_12/o1_preview_2024_09_12  模型, 默认情况下 max_completion_tokens=8192.