SWE-bench

Author: cte

evals/apps/swe-bench/.gitignore

@@ -0,0 +1,3 @@
+.venv
+
+datasets

evals/apps/swe-bench/.python-version

@@ -0,0 +1 @@
+3.13

evals/apps/swe-bench/eslint.config.mjs

@@ -0,0 +1,4 @@
+import { config } from "@evals/eslint-config/base"
+
+/** @type {import("eslint").Linter.Config} */
+export default [...config]

evals/apps/swe-bench/package.json

@@ -0,0 +1,14 @@
+{
+	"name": "@evals/swe-bench",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"dev": "tsx src/index.ts",
+		"dataset": "uv run scripts/dataset.py"
+	},
+	"dependencies": {},
+	"devDependencies": {
+		"@evals/eslint-config": "workspace:^",
+		"@evals/typescript-config": "workspace:^"
+	}
+}

evals/apps/swe-bench/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "swe-bench"
+version = "0.1.0"
+requires-python = ">=3.13"
+dependencies = [
+    "datasets>=3.5.1",
+]

evals/apps/swe-bench/scripts/dataset.py

@@ -0,0 +1,33 @@
+from datasets import Dataset, load_dataset
+from typing import Any, Dict, cast
+import json
+import os
+
+
+def main():
+    swebench = cast(Dataset, load_dataset("princeton-nlp/SWE-bench", split="test"))
+
+    # print(f"Count: {len(swebench)}")
+    # print(f"Features: {swebench.features}")
+    # print(f"Repos: {swebench.unique('repo')}")
+
+    datasets_dir = "datasets"
+    os.makedirs(datasets_dir, exist_ok=True)
+    data_file = os.path.join(datasets_dir, "swebench_test.jsonl")
+
+    if os.path.exists(data_file):
+        print(f"dataset already exists in {data_file}")
+        return
+
+    with open(data_file, "w") as f:
+        for i, row in enumerate(swebench):
+            example = dict(cast(Dict[str, Any], row))
+            f.write(json.dumps(example) + "\n")
+            if i % 100 == 0:
+                print(f"Processed {i} rows...")
+
+    print(f"dataset successfully written to {data_file}")
+
+
+if __name__ == "__main__":
+    main()

evals/apps/swe-bench/src/index.ts

@@ -0,0 +1,31 @@
+import fs from "fs"
+import path from "path"
+import readline from "readline"
+import { fileURLToPath } from "url"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+async function main() {
+	const datasetPath = path.join(__dirname, "..", "datasets", "swebench_test.jsonl")
+
+	try {
+		const input = fs.createReadStream(datasetPath, { encoding: "utf-8" })
+		const rl = readline.createInterface({ input, crlfDelay: Infinity })
+
+		for await (const line of rl) {
+			console.log(`Line: ${line}`)
+			// console.log(JSON.parse(line))
+			// break
+		}
+
+		rl.close()
+		input.close()
+	} catch (error) {
+		console.error("Error reading file:", error)
+	}
+}
+
+main().catch((error) => {
+	console.error("Unhandled error:", error)
+})

evals/apps/swe-bench/tsconfig.json

@@ -0,0 +1,5 @@
+{
+	"extends": "@evals/typescript-config/base.json",
+	"include": ["src"],
+	"exclude": ["node_modules"]
+}

evals/apps/swe-bench/uv.lock

@@ -399,8 +399,6 @@ export const providerSettingsSchema = z.object({
 	// OpenAI Native
 	openAiNativeApiKey: z.string().optional(),
 	openAiNativeBaseUrl: z.string().optional(),
-	// XAI
-	xaiApiKey: z.string().optional(),
 	// Mistral
 	mistralApiKey: z.string().optional(),
 	mistralCodestralUrl: z.string().optional(),