Minor fixes, add OpenAI Harmony to requirements

Author: nv-alicheng

examples/06_GPT-OSS-120B_SGLang_Example/eval_accuracy.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+from pathlib import Path
+
+from inference_endpoint.dataset_manager.predefined.gpqa import GPQA
+from inference_endpoint.evaluation.extractor import ABCDExtractor
+from inference_endpoint.evaluation.scoring import PassAt1Scorer
+
+
+def main(args):
+    # Load the dataset
+    ds = GPQA.load_from_file(args.dataset_path)
+    ds.load()
+
+    # Create the scorer
+    scorer = PassAt1Scorer(
+        GPQA.DATASET_ID,
+        ds,
+        args.report_dir,
+        extractor=ABCDExtractor,
+    )
+
+    # Score the dataset
+    score = scorer.score()
+    print(f"Pass@1 Score: {score}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Evaluate accuracy of the SGLang endpoint on the GPQA dataset",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "--dataset-path",
+        type=Path,
+        help="Path to the dataset",
+        default="datasets/gpqa/diamond/gpqa_diamond.parquet",
+    )
+    parser.add_argument(
+        "--report-dir",
+        type=Path,
+        help="Path to the report directory",
+        default="gpqa_sglang_report",
+    )
+    args = parser.parse_args()
+    main(args)

requirements/base.txt

@@ -30,6 +30,7 @@ numpy==2.3.4
 datasets==4.1.1
 sentencepiece==0.2.1
 protobuf==6.33.0
+openai_harmony==0.0.8
 
 # Color support for cross-platform terminals
 colorama==0.4.6

src/inference_endpoint/evaluation/extractor.py

@@ -25,8 +25,8 @@ class Extractor(ABC):
     numeric value plain or inside a LaTeX block.
     """
 
-    @abstractmethod
     @classmethod
+    @abstractmethod
     def extract(cls, text: str) -> str | None:
         raise NotImplementedError
 

src/inference_endpoint/evaluation/scoring.py

@@ -71,7 +71,7 @@ def get_outputs(self):
             for line in f:
                 event = orjson.loads(line.strip())
                 if event["event_type"] == SampleEvent.COMPLETE.value:
-                    outputs.append(event["data"])
+                    outputs.append(event)
         df = pd.DataFrame(outputs, columns=["sample_uuid", "value"])
         df.rename(columns={"value": "output"}, inplace=True)
         return df