feat: refactor: model provider

Author: terryyz

bigcodebench/evaluate.py

@@ -115,7 +115,7 @@ def evaluate(
     split: str,
     subset: str,
     samples: Optional[str] = None,
-    remote_execute: bool = True,
+    local_execute: bool = False,
     remote_execute_api: str = "https://bigcode-bigcodebench-evaluator.hf.space/",
     pass_k: str = "1,5,10",
     save_pass_rate: bool = True,
@@ -135,16 +135,14 @@ def evaluate(
             **model_kwargs,
         )
     assert samples is not None, "No samples provided"
-    
-    extra = subset + "_" if subset != "full" else ""
-    
+        
     if os.path.isdir(samples):
-        result_path = os.path.join(samples, f"{extra}eval_results.json")
+        result_path = os.path.join(samples, "eval_results.json")
     else:
         assert samples.endswith(".jsonl")
-        result_path = samples.replace(".jsonl", f"_{extra}eval_results.json")
+        result_path = samples.replace(".jsonl", "_eval_results.json")
 
-    if remote_execute:
+    if not local_execute:
 
         client = Client(remote_execute_api)
         results, pass_at_k = client.predict(
@@ -351,7 +349,7 @@ def stucking_checker():
             json.dump(results, f, indent=2)
 
     if save_pass_rate:
-        pass_at_k_path = result_path.replace("_eval_results.json", "_pass_at_k.json")
+        pass_at_k_path = result_path.replace("eval_results.json", "pass_at_k.json")
 
         if os.path.isfile(pass_at_k_path):
             saved_pass_at_k = json.load(open(pass_at_k_path, "r"))

bigcodebench/gen/util/anthropic_request.py

@@ -44,4 +44,4 @@ def make_auto_request(client: anthropic.Client, *args, **kwargs) -> Message:
             print(e)
             signal.alarm(0)
             time.sleep(1)
-    return ret
+    return ret

bigcodebench/gen/util/google_request.py

@@ -0,0 +1,45 @@
+import time
+
+import google.generativeai as genai
+from google.api_core.exceptions import GoogleAPICallError, ResourceExhausted
+
+
+def make_request(
+    client: genai.GenerativeModel, temperature, messages, max_new_tokens=2048
+) -> genai.types.GenerateContentResponse:
+    messages = [{"role": m["role"], "parts": [m["content"]]} for m in messages]
+    response = client.generate_content(
+        messages,
+        generation_config=genai.types.GenerationConfig(
+            candidate_count=1,
+            max_output_tokens=max_new_tokens,
+            temperature=temperature,
+        ),
+        safety_settings=[
+            {"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_NONE"},
+            {"category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_NONE"},
+            {"category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_NONE"},
+            {"category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_NONE"},
+        ],
+    )
+
+    return response.text
+
+
+def make_auto_request(*args, **kwargs) -> genai.types.GenerateContentResponse:
+    ret = None
+    while ret is None:
+        try:
+            ret = make_request(*args, **kwargs)
+        except ResourceExhausted as e:
+            print("Rate limit exceeded. Waiting...", e.message)
+            time.sleep(10)
+        except GoogleAPICallError as e:
+            print(e.message)
+            time.sleep(1)
+        except Exception as e:
+            print("Unknown error. Waiting...")
+            print(e)
+            time.sleep(1)
+    return ret
+

bigcodebench/gen/util/mistral_request.py

@@ -0,0 +1,12 @@
+from mistralai.client import MistralClient
+from mistralai.models.chat_completion import ChatMessage
+
+def make_auto_request(client: MistralClient, *args, **kwargs) -> ChatMessage:
+    ret = None
+    while ret is None:
+        try:
+            ret = client.chat(*args, **kwargs)
+        except Exception as e:
+            print(e)
+            time.sleep(1)
+    return ret

bigcodebench/generate.py

@@ -154,13 +154,21 @@ def run_codegen(
         assert id_range[0] < id_range[1], "id_range must be increasing"
         id_range = tuple(id_range)
 
+    # Make project dir
+    os.makedirs(root, exist_ok=True)
+    
+    instruction_prefix = "Please provide a self-contained Python script that solves the following problem in a markdown code block:"
+    response_prefix = "Below is a Python script with a self-contained function that solves the problem and passes corresponding tests:"
+    
     # Make dir for codes generated by each model
     model_runner = make_model(
         model=model,
         backend=backend,
         subset=subset,
         split=split,
         temperature=temperature,
+        instruction_prefix=instruction_prefix,
+        response_prefix=response_prefix,
         base_url=base_url,
         tp=tp,
         trust_remote_code=trust_remote_code,