init transition to HF dataset

Author: Zilinghan

pyproject.toml

@@ -32,12 +32,13 @@ dependencies = [
     "pytest",
     "pytest-cov",
     "litellm",
+    "inspect-ai",
+    "datasets",
     # requirements for execution
     "numpy",
     "scipy",
     "matplotlib",
     "sympy",
-    "inspect-ai",
 ]
 
 # Classifiers help users find your project by categorizing it.

src/scicode/parse/parse.py

@@ -8,6 +8,7 @@
 import scipy
 import numpy as np
 from sympy import Symbol
+from datasets import load_dataset
 
 OrderedContent = list[tuple[str, str]]
 
@@ -56,6 +57,10 @@ def read_from_jsonl(file_path):
             data.append(json.loads(line.strip()))
     return data
 
+def read_from_hf_dataset(split='validation'):
+    dataset = load_dataset('Zilinghan/scicode', split=split)
+    return dataset
+
 def rm_comments(string: str) -> str:
     ret_lines = []
     lines = string.split('\n')