diff --git a/examples/basic_usage.py b/examples/basic_usage.py
index e6c1134..5fc6dd5 100755
--- a/examples/basic_usage.py
+++ b/examples/basic_usage.py
@@ -383,12 +383,12 @@ def _extract_content(self, html, url=None):
results_dir.mkdir(exist_ok=True)
results_path = results_dir / "evaluation_results.json"
- DataSaver.save_evaluation_results(result.to_dict(), results_path)
+ DataSaver.save_evaluation_results(result, results_path)
print(f"\n结果已保存到: {results_path}")
# 10. 生成报告
report_path = results_dir / "evaluation_report.csv"
- DataSaver.save_summary_report(result.to_dict(), report_path)
+ DataSaver.save_summary_report(result, report_path)
print(f"报告已保存到: {report_path}")
@@ -458,10 +458,251 @@ def _extract_content(self, html, url=None):
print(f"\n📊 榜单已保存到: {leaderboard_path}")
+def demo_llm_webkit_evaluation():
+ """演示LLM-WebKit抽取器的6项指标评测"""
+
+ print("=== LLM-WebKit Extractor 6项指标评测示例 ===\n")
+
+ # 设置日志
+ setup_logging(level="INFO")
+
+ # 1. 创建包含各种内容类型的测试数据集
+ print("1. 创建包含多种内容类型的测试数据集...")
+
+ samples = []
+
+ # 样本1: 包含文本和代码
+ samples.append(DataSample(
+ id="text_code_sample",
+ html="""
+
+
+ Python编程示例
+ 这是一段关于Python编程的介绍文本。
+
+def hello_world():
+ print("Hello, World!")
+ return True
+
+ 以上代码展示了一个简单的Python函数。
+
+
+ """,
+ groundtruth_content="""# Python编程示例
+
+这是一段关于Python编程的介绍文本。
+
+```python
+def hello_world():
+ print("Hello, World!")
+ return True
+```
+
+以上代码展示了一个简单的Python函数。""",
+ groundtruth_content_list=[
+ {"type": "heading", "content": "Python编程示例", "level": 1},
+ {"type": "text", "content": "这是一段关于Python编程的介绍文本。"},
+ {"type": "code", "content": "def hello_world():\n print(\"Hello, World!\")\n return True", "language": "python"},
+ {"type": "text", "content": "以上代码展示了一个简单的Python函数。"}
+ ]
+ ))
+
+ # 样本2: 包含表格
+ samples.append(DataSample(
+ id="table_sample",
+ html="""
+
+
+ 销售数据统计
+
+
+
+ | 产品 |
+ 销量 |
+ 收入 |
+
+
+
+
+ | 产品A |
+ 100 |
+ 1000 |
+
+
+ | 产品B |
+ 200 |
+ 3000 |
+
+
+
+
+
+ """,
+ groundtruth_content="""## 销售数据统计
+
+| 产品 | 销量 | 收入 |
+|------|------|------|
+| 产品A | 100 | 1000 |
+| 产品B | 200 | 3000 |""",
+ groundtruth_content_list=[
+ {"type": "heading", "content": "销售数据统计", "level": 2},
+ {"type": "table", "content": "| 产品 | 销量 | 收入 |\n|------|------|------|\n| 产品A | 100 | 1000 |\n| 产品B | 200 | 3000 |"}
+ ]
+ ))
+
+ # 样本3: 包含公式
+ samples.append(DataSample(
+ id="formula_sample",
+ html="""
+
+
+ 数学公式示例
+ 这是一个行内公式: $E = mc^2$
+ 这是一个行间公式:
+ $$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}$$
+
+
+ """,
+ groundtruth_content="""## 数学公式示例
+
+这是一个行内公式: $E = mc^2$
+
+这是一个行间公式:
+
+$$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}$$""",
+ groundtruth_content_list=[
+ {"type": "heading", "content": "数学公式示例", "level": 2},
+ {"type": "text", "content": "这是一个行内公式: $E = mc^2$"},
+ {"type": "text", "content": "这是一个行间公式:"},
+ {"type": "formula", "content": "\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}"}
+ ]
+ ))
+
+ # 创建数据集并添加样本
+ dataset = BenchmarkDataset(name="llm_webkit_test", description="LLM-WebKit 6项指标测试数据集")
+ for sample in samples:
+ dataset.add_sample(sample)
+
+ print(f"测试数据集包含 {len(dataset)} 个样本")
+ print(f"样本类型: 文本+代码, 表格, 公式\n")
+
+ # 2. 创建LLM-WebKit抽取器
+ print("2. 创建LLM-WebKit抽取器...")
+
+ # 显示所有可用的抽取器
+ available_extractors = ExtractorFactory.list_available()
+ print(f"可用的抽取器: {available_extractors}")
+
+ # 直接创建LLM-WebKit抽取器，设置模型路径
+ config = {
+ "model_path": "/Users/chupei/model/checkpoint-3296"
+ }
+ extractor = ExtractorFactory.create("llm-webkit", config=config)
+ print(f"✅ LLM-WebKit抽取器创建成功，模型路径: {config['model_path']}")
+
+ print()
+
+ # 3. 创建评测器并显示所有可用指标
+ print("3. 创建评测器...")
+ evaluator = Evaluator()
+ available_metrics = evaluator.metric_calculator.list_available_metrics()
+ print(f"✅ 可用的评测指标 ({len(available_metrics)}项):")
+
+ # 按照6项指标分类显示
+ target_metrics = ["overall", "text_edit", "code_edit", "table_edit", "table_TEDS", "formula_edit"]
+
+ for metric in target_metrics:
+ if metric in available_metrics:
+ print(f" ✅ {metric}")
+ else:
+ print(f" ❌ {metric} (未注册)")
+
+ print()
+
+ # 4. 运行评测
+ print("4. 开始评测...")
+ print("=" * 60)
+
+ result = evaluator.evaluate(
+ dataset=dataset,
+ extractor=extractor,
+ max_samples=None # 评测所有样本
+ )
+
+ # 5. 显示详细的6项指标结果
+ print("\n5. 📊 6项指标详细评测结果:")
+ print("=" * 60)
+
+ results_dict = result.to_dict()
+
+ # 从overall_metrics中提取指标结果
+ metrics = results_dict.get('overall_metrics', {})
+
+ # 按照指标分类显示
+ print(f"\n🏆 综合指标:")
+ if 'overall' in metrics:
+ print(f" overall (综合得分): {metrics['overall']:.4f}")
+ else:
+ print(" overall: 未计算")
+
+ print(f"\n📝 文本相关指标:")
+ if 'text_edit' in metrics:
+ print(f" text_edit (文本编辑距离): {metrics['text_edit']:.4f}")
+ else:
+ print(" text_edit: 未计算")
+ if 'code_edit' in metrics:
+ print(f" code_edit (代码编辑距离): {metrics['code_edit']:.4f}")
+ else:
+ print(" code_edit: 未计算")
+
+ print(f"\n📊 表格相关指标:")
+ if 'table_edit' in metrics:
+ print(f" table_edit (表格编辑距离): {metrics['table_edit']:.4f}")
+ else:
+ print(" table_edit: 未计算")
+ if 'table_TEDS' in metrics:
+ print(f" table_TEDS (表格结构相似度): {metrics['table_TEDS']:.4f}")
+ else:
+ print(" table_TEDS: 未计算")
+
+ print(f"\n🧮 公式相关指标:")
+ if 'formula_edit' in metrics:
+ print(f" formula_edit (公式编辑距离): {metrics['formula_edit']:.4f}")
+ else:
+ print(" formula_edit: 未计算")
+
+ print(f"\n📈 详细统计:")
+ print(f" 总样本数: {len(dataset)}")
+ success_count = len([s for s in results_dict.get('sample_results', []) if s.get('extraction_success', False)])
+ failure_count = len(dataset) - success_count
+ print(f" 成功样本数: {success_count}")
+ print(f" 失败样本数: {failure_count}")
+
+ # 6. 保存结果到文件
+ print("\n" + "=" * 60)
+ print("6. 保存评测结果...")
+
+ results_dir = Path("results")
+ results_dir.mkdir(exist_ok=True)
+
+ # 保存详细结果
+ results_path = results_dir / "llm_webkit_evaluation_results.json"
+ DataSaver.save_evaluation_results(result, results_path) # 直接传递result对象
+ print(f"✅ 详细结果已保存到: {results_path}")
+
+ # 生成CSV报告
+ report_path = results_dir / "llm_webkit_evaluation_report.csv"
+ DataSaver.save_summary_report(result, report_path) # 直接传递result对象
+ print(f"✅ CSV报告已保存到: {report_path}")
+
+ print("\n" + "=" * 60)
+ print("✅ LLM-WebKit 6项指标评测完成！")
+
+
if __name__ == "__main__":
try:
demo_basic_evaluation()
- # demo_extractor_comparison()
+ # demo_llm_webkit_evaluation() # 使用新的LLM-WebKit评测示例
print("\n✅ 示例运行完成！")
except Exception as e:
diff --git a/results/evaluation_report.csv b/results/evaluation_report.csv
index 1a3cf80..652eff8 100644
--- a/results/evaluation_report.csv
+++ b/results/evaluation_report.csv
@@ -1,2 +1,2 @@
-extractor,total_samples,success_rate,overall
-mock,2,1.0,0.8
+extractor,total_samples,success_rate,overall,code_edit,formula_edit,table_TEDS,table_edit,text_edit
+mock,2,1.0,0.8,1.0,1.0,1.0,1.0,0.0
diff --git a/results/evaluation_results.json b/results/evaluation_results.json
index dfd9e3e..7b603f6 100644
--- a/results/evaluation_results.json
+++ b/results/evaluation_results.json
@@ -2,7 +2,7 @@
"metadata": {
"dataset_name": "sample_dataset",
"extractor_name": "mock",
- "timestamp": "2025-07-30T10:05:54.322334",
+ "timestamp": "2025-07-31T14:29:43.477342",
"total_samples": 2
},
"overall_metrics": {
@@ -17,7 +17,7 @@
{
"sample_id": "sample-001-programming-tutorial",
"extraction_success": true,
- "extraction_time": 3.0994415283203125e-06,
+ "extraction_time": 4.0531158447265625e-06,
"metrics": {
"code_edit": {
"score": 1.0,
@@ -113,7 +113,7 @@
{
"sample_id": "sample-002-math-formulas",
"extraction_success": true,
- "extraction_time": 2.1457672119140625e-06,
+ "extraction_time": 1.9073486328125e-06,
"metrics": {
"code_edit": {
"score": 1.0,
diff --git a/tests/test_metrics.py b/tests/test_metrics.py
index 0a46f44..a2fe3d9 100644
--- a/tests/test_metrics.py
+++ b/tests/test_metrics.py
@@ -57,26 +57,7 @@ def hello():
最后是正确的文字内容。
"""
- # 构建content_list (模拟结构化数据)
- self.predicted_content_list = [
- {"type": "heading", "content": "标题", "level": 1},
- {"type": "paragraph", "content": "这是一段文字内容。"},
- {"type": "code", "content": 'def hello():\n print("Hello World")'},
- {"type": "paragraph", "content": "这是公式: $E = mc^2$"},
- {"type": "equation-interline", "content": "\\int_{0}^{\\infty} e^{-x} dx = 1"},
- {"type": "table", "content": "| 列1 | 列2 |\n|-----|-----|\n| 数据1 | 数据2 |\n| 数据3 | 数据4 |"},
- {"type": "paragraph", "content": "最后是更多文字内容。"}
- ]
-
- self.groundtruth_content_list = [
- {"type": "heading", "content": "标题", "level": 1},
- {"type": "paragraph", "content": "这是一段正确的文字内容。"},
- {"type": "code", "content": 'def hello():\n print("Hello, World!")'},
- {"type": "paragraph", "content": "这是正确的公式: $E = mc^2$"},
- {"type": "equation-interline", "content": "\\int_{0}^{\\infty} e^{-x} dx = 1"},
- {"type": "table", "content": "| 列1 | 列2 |\n|-----|-----|\n| 正确数据1 | 正确数据2 |\n| 正确数据3 | 正确数据4 |"},
- {"type": "paragraph", "content": "最后是正确的文字内容。"}
- ]
+
def test_available_metrics(self):
"""测试可用指标列表"""
@@ -91,9 +72,7 @@ def test_metric_calculation_success(self):
"""测试指标计算成功"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
# 验证所有指标都计算成功
@@ -106,9 +85,7 @@ def test_code_edit_metric(self):
"""测试代码编辑距离指标"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
code_result = results['code_edit']
@@ -127,9 +104,7 @@ def test_formula_edit_metric(self):
"""测试公式编辑距离指标"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
formula_result = results['formula_edit']
@@ -146,9 +121,7 @@ def test_table_edit_metric(self):
"""测试表格编辑距离指标"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
table_result = results['table_edit']
@@ -165,9 +138,7 @@ def test_table_teds_metric(self):
"""测试表格TEDS指标"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
teds_result = results['table_TEDS']
@@ -183,9 +154,7 @@ def test_text_edit_metric(self):
"""测试纯文本编辑距离指标"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
text_result = results['text_edit']
@@ -202,9 +171,7 @@ def test_overall_metric_calculation(self):
"""测试overall指标是其他指标的平均值"""
results = self.calculator.calculate_all(
predicted_content=self.predicted_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.predicted_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
# 获取individual指标分数
@@ -234,9 +201,7 @@ def test_identical_content(self):
# 使用相同的内容
results = self.calculator.calculate_all(
predicted_content=self.groundtruth_content,
- groundtruth_content=self.groundtruth_content,
- predicted_content_list=self.groundtruth_content_list,
- groundtruth_content_list=self.groundtruth_content_list
+ groundtruth_content=self.groundtruth_content
)
# 大部分指标应该得到完美分数(1.0)，除了可能某些算法有特殊处理
@@ -249,9 +214,7 @@ def test_empty_content(self):
"""测试空内容的情况"""
results = self.calculator.calculate_all(
predicted_content="",
- groundtruth_content="",
- predicted_content_list=[],
- groundtruth_content_list=[]
+ groundtruth_content=""
)
# 空内容应该能正确处理，不应该出错
@@ -260,72 +223,9 @@ def test_empty_content(self):
self.assertTrue(result.success or result.score == 0.0,
f"空内容的{metric_name}应该正确处理")
- def test_content_list_priority(self):
- """测试content_list优先级"""
- # 创建有差异的content_list
- different_predicted_list = [
- {"type": "heading", "content": "标题1"},
- {"type": "paragraph", "content": "这是不同的文字段落。"},
- {"type": "code", "content": "print('different code')"},
- {"type": "paragraph", "content": "另一段不同的文字。"},
- {"type": "table", "content": "| A | B |\n|---|---|\n| 1 | 2 |"},
- {"type": "text", "content": "更多的不同文字"}
- ]
-
- different_groundtruth_list = [
- {"type": "heading", "content": "标题2"},
- {"type": "paragraph", "content": "这是原始的文字段落。"},
- {"type": "code", "content": "print('original code')"},
- {"type": "paragraph", "content": "另一段原始的文字。"},
- {"type": "table", "content": "| X | Y |\n|---|---|\n| 3 | 4 |"},
- {"type": "text", "content": "更多的原始文字"}
- ]
-
- # 只提供content_list，不提供content
- results = self.calculator.calculate_all(
- predicted_content="", # 空的content
- groundtruth_content="",
- predicted_content_list=different_predicted_list,
- groundtruth_content_list=different_groundtruth_list
- )
-
- # 验证即使content为空，也能从content_list中提取内容
- for metric_name in ['code_edit', 'table_edit', 'text_edit']:
- if metric_name in results:
- result = results[metric_name]
- self.assertTrue(result.success, f"{metric_name}应该能从content_list提取内容")
- # 分数应该有意义（不为1.0，因为有差异）
- self.assertLess(result.score, 1.0, f"{metric_name}应该检测到差异")
-
- def test_nested_content_list(self):
- """测试嵌套的content_list结构"""
- nested_content_list = [
- {
- "type": "section",
- "children": [
- {"type": "code", "content": "print('nested code')"},
- {
- "type": "list",
- "items": [
- {"type": "equation-inline", "content": "x = 1"}
- ]
- }
- ]
- }
- ]
-
- results = self.calculator.calculate_all(
- predicted_content="",
- groundtruth_content="",
- predicted_content_list=nested_content_list,
- groundtruth_content_list=nested_content_list
- )
-
- # 验证能正确处理嵌套结构
- for metric_name in ['code_edit', 'formula_edit']:
- if metric_name in results:
- self.assertTrue(results[metric_name].success,
- f"{metric_name}应该能处理嵌套的content_list")
+
+
+
class TestErrorHandling(unittest.TestCase):
@@ -334,20 +234,12 @@ class TestErrorHandling(unittest.TestCase):
def setUp(self):
self.calculator = MetricCalculator()
- def test_malformed_content_list(self):
- """测试格式错误的content_list"""
- malformed_list = [
- {"invalid": "structure"}, # 缺少type字段
- "not_a_dict", # 不是字典
- {"type": "code", "content": None} # content为None
- ]
-
- # 应该能处理错误格式而不崩溃
+ def test_malformed_content(self):
+ """测试格式错误的输入"""
+ # 应该能处理各种错误输入而不崩溃
results = self.calculator.calculate_all(
predicted_content="test",
- groundtruth_content="test",
- predicted_content_list=malformed_list,
- groundtruth_content_list=malformed_list
+ groundtruth_content="test"
)
# 不应该有未捕获的异常
@@ -357,15 +249,222 @@ def test_none_inputs(self):
"""测试None输入"""
results = self.calculator.calculate_all(
predicted_content=None,
- groundtruth_content=None,
- predicted_content_list=None,
- groundtruth_content_list=None
+ groundtruth_content=None
)
# 应该能处理None输入
self.assertIsInstance(results, dict)
+class TestRealSampleMetrics(unittest.TestCase):
+ """测试基于LLM-WebKit实际提取结果的指标计算"""
+
+ def setUp(self):
+ """测试前准备"""
+ self.calculator = MetricCalculator()
+
+ def test_text_code_sample_edit_distance(self):
+ """测试文本+代码样本的编辑距离"""
+ # 基于实际调试结果的数据
+ groundtruth = """# Python编程示例
+
+这是一段关于Python编程的介绍文本。
+
+```python
+def hello_world():
+ print("Hello, World!")
+ return True
+```
+
+以上代码展示了一个简单的Python函数。"""
+
+ predicted = """# Python编程示例
+
+这是一段关于Python编程的介绍文本。
+
+```
+def hello_world():
+ print("Hello, World!")
+ return True
+```
+
+以上代码展示了一个简单的Python函数。"""
+
+ # 计算编辑距离（基于实际调试结果）
+ results = self.calculator.calculate_all(
+ predicted_content=predicted,
+ groundtruth_content=groundtruth
+ )
+
+ # 验证文本编辑距离
+ self.assertIn("text_edit", results)
+ self.assertTrue(results["text_edit"].success)
+ # 基于实际测试结果调整期望值
+ self.assertGreater(results["text_edit"].score, 0.50)
+
+ # 验证代码编辑距离（代码内容完全一致，应该有高分）
+ self.assertIn("code_edit", results)
+ self.assertTrue(results["code_edit"].success)
+ self.assertGreater(results["code_edit"].score, 0.90)
+
+ def test_table_sample_edit_distance(self):
+ """测试表格样本的编辑距离"""
+ groundtruth = """## 销售数据统计
+
+| 产品 | 销量 | 收入 |
+|------|------|------|
+| 产品A | 100 | 1000 |
+| 产品B | 200 | 3000 |"""
+
+ predicted = """## 销售数据统计
+
+| 产品 | 销量 | 收入 |
+|---|---|---|
+| 产品A | 100 | 1000 |
+| 产品B | 200 | 3000 |"""
+
+ results = self.calculator.calculate_all(
+ predicted_content=predicted,
+ groundtruth_content=groundtruth
+ )
+
+ # 验证表格编辑距离（应该接近0.9022）
+ self.assertIn("table_edit", results)
+ self.assertTrue(results["table_edit"].success)
+ self.assertGreater(results["table_edit"].score, 0.85)
+
+ # 验证TEDS指标（表格结构相同，应该满分）
+ self.assertIn("table_TEDS", results)
+ self.assertTrue(results["table_TEDS"].success)
+ self.assertGreater(results["table_TEDS"].score, 0.95)
+
+ def test_formula_sample_edit_distance(self):
+ """测试公式样本的编辑距离"""
+ groundtruth = """## 数学公式示例
+
+这是一个行内公式: $E = mc^2$
+
+这是一个行间公式:
+
+$$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}$$"""
+
+ predicted = """## 数学公式示例
+
+这是一个行内公式: \\$E = mc^2\\$
+
+这是一个行间公式:
+
+\\$\\$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}\\$"""
+
+ results = self.calculator.calculate_all(
+ predicted_content=predicted,
+ groundtruth_content=groundtruth
+ )
+
+ # 验证公式编辑距离（符号转义导致分数较低）
+ self.assertIn("formula_edit", results)
+ self.assertTrue(results["formula_edit"].success)
+ self.assertGreater(results["formula_edit"].score, 0.10)
+
+ # 验证文本编辑距离（去除公式后的纯文本）
+ self.assertIn("text_edit", results)
+ self.assertTrue(results["text_edit"].success)
+
+ def test_overall_score_calculation(self):
+ """测试综合分数计算"""
+ # 使用第一个样本测试综合分数
+ groundtruth = """# Python编程示例
+
+这是一段关于Python编程的介绍文本。
+
+```python
+def hello_world():
+ print("Hello, World!")
+ return True
+```
+
+以上代码展示了一个简单的Python函数。"""
+
+ predicted = """# Python编程示例
+
+这是一段关于Python编程的介绍文本。
+
+```
+def hello_world():
+ print("Hello, World!")
+ return True
+```
+
+以上代码展示了一个简单的Python函数。"""
+
+ results = self.calculator.calculate_all(
+ predicted_content=predicted,
+ groundtruth_content=groundtruth
+ )
+
+ # 验证overall分数存在且合理
+ self.assertIn("overall", results)
+ self.assertTrue(results["overall"].success)
+
+ # overall应该是所有成功指标的平均值
+ successful_scores = []
+ for metric_name, result in results.items():
+ if metric_name != "overall" and result.success:
+ successful_scores.append(result.score)
+
+ if successful_scores:
+ expected_overall = sum(successful_scores) / len(successful_scores)
+ actual_overall = results["overall"].score
+
+ # 允许小幅计算误差
+ self.assertAlmostEqual(actual_overall, expected_overall, places=3)
+
+ def test_all_metrics_coverage(self):
+ """测试所有6项指标都被计算"""
+ groundtruth = """# 综合示例
+
+这是文本内容。
+
+```python
+def test():
+ return True
+```
+
+这是公式: $x = y$
+
+| A | B |
+|---|---|
+| 1 | 2 |
+
+更多文本。"""
+
+ predicted = groundtruth # 使用相同内容测试
+
+ results = self.calculator.calculate_all(
+ predicted_content=predicted,
+ groundtruth_content=groundtruth
+ )
+
+ # 验证所有6项指标都存在
+ expected_metrics = ["overall", "text_edit", "code_edit", "table_edit", "table_TEDS", "formula_edit"]
+
+ print(f"\n=== 完全相同内容的指标测试结果 ===")
+
+ for metric in expected_metrics:
+ self.assertIn(metric, results, f"指标 {metric} 缺失")
+ self.assertTrue(results[metric].success, f"指标 {metric} 计算失败")
+
+ score = results[metric].score
+ print(f"{metric}: {score:.6f}")
+
+ # 完全相同的内容应该得到满分 1.0
+ self.assertAlmostEqual(score, 1.0,
+ places=4,
+ msg=f"完全相同内容的 {metric} 应该得到满分，实际得分: {score}")
+
+ print("✅ 所有指标都正确得到满分!")
+
+
def run_visual_test():
"""运行可视化测试（保留原有的打印功能）"""
print("=== 新指标功能测试 ===\n")
@@ -424,34 +523,11 @@ def hello():
最后是正确的文字内容。
"""
- # 构建content_list (模拟结构化数据)
- predicted_content_list = [
- {"type": "heading", "content": "标题", "level": 1},
- {"type": "paragraph", "content": "这是一段文字内容。"},
- {"type": "code", "content": 'def hello():\n print("Hello World")'},
- {"type": "paragraph", "content": "这是公式: $E = mc^2$"},
- {"type": "equation-interline", "content": "\\int_{0}^{\\infty} e^{-x} dx = 1"},
- {"type": "table", "content": "| 列1 | 列2 |\n|-----|-----|\n| 数据1 | 数据2 |\n| 数据3 | 数据4 |"},
- {"type": "paragraph", "content": "最后是更多文字内容。"}
- ]
-
- groundtruth_content_list = [
- {"type": "heading", "content": "标题", "level": 1},
- {"type": "paragraph", "content": "这是一段正确的文字内容。"},
- {"type": "code", "content": 'def hello():\n print("Hello, World!")'},
- {"type": "paragraph", "content": "这是正确的公式: $E = mc^2$"},
- {"type": "equation-interline", "content": "\\int_{0}^{\\infty} e^{-x} dx = 1"},
- {"type": "table", "content": "| 列1 | 列2 |\n|-----|-----|\n| 正确数据1 | 正确数据2 |\n| 正确数据3 | 正确数据4 |"},
- {"type": "paragraph", "content": "最后是正确的文字内容。"}
- ]
-
# 计算所有指标
print("正在计算指标...")
results = calculator.calculate_all(
predicted_content=predicted_content,
- groundtruth_content=groundtruth_content,
- predicted_content_list=predicted_content_list,
- groundtruth_content_list=groundtruth_content_list
+ groundtruth_content=groundtruth_content
)
# 显示结果
diff --git a/webmainbench/data/saver.py b/webmainbench/data/saver.py
index 3da0abb..cc7005e 100644
--- a/webmainbench/data/saver.py
+++ b/webmainbench/data/saver.py
@@ -5,9 +5,14 @@
import json
import jsonlines
from pathlib import Path
-from typing import Union, List, Dict, Any
+from typing import Union, List, Dict, Any, TYPE_CHECKING
+
from .dataset import BenchmarkDataset, DataSample
+if TYPE_CHECKING:
+ from ..evaluator import EvaluationResult
+ from ..metrics import MetricResult
+
class DataSaver:
"""Data saver for various output formats."""
@@ -73,41 +78,48 @@ def save_json(dataset: BenchmarkDataset,
json.dump(data, f, indent=indent, ensure_ascii=False)
@staticmethod
- def save_evaluation_results(results: Dict[str, Any],
+ def save_evaluation_results(results: Union["EvaluationResult", Dict[str, Any]],
file_path: Union[str, Path],
format: str = "json") -> None:
"""
Save evaluation results.
Args:
- results: Evaluation results dictionary
+ results: EvaluationResult instance or evaluation results dictionary
file_path: Output file path
format: Output format ("json" or "jsonl")
"""
file_path = Path(file_path)
file_path.parent.mkdir(parents=True, exist_ok=True)
+ # Convert EvaluationResult to dict if needed
+ if hasattr(results, 'to_dict'):
+ results_dict = results.to_dict()
+ else:
+ results_dict = results
+
if format.lower() == "json":
with open(file_path, 'w', encoding='utf-8') as f:
- json.dump(results, f, indent=2, ensure_ascii=False)
+ json.dump(results_dict, f, indent=2, ensure_ascii=False)
elif format.lower() == "jsonl":
with jsonlines.open(file_path, 'w') as writer:
- if isinstance(results, dict) and 'samples' in results:
- for sample_result in results['samples']:
+ if isinstance(results_dict, dict) and 'samples' in results_dict:
+ for sample_result in results_dict['samples']:
writer.write(sample_result)
else:
- writer.write(results)
+ writer.write(results_dict)
else:
raise ValueError(f"Unsupported format: {format}")
@staticmethod
- def save_summary_report(results: Union[Dict[str, Any], List[Dict[str, Any]]],
+ def save_summary_report(results: Union["EvaluationResult", List["EvaluationResult"], Dict[str, Any], List[Dict[str, Any]]],
file_path: Union[str, Path]) -> None:
"""
Save evaluation results as a CSV leaderboard.
Args:
- results: Single evaluation results dictionary or list of results for multiple extractors
+ results: Single EvaluationResult instance, list of EvaluationResult instances,
+ or their dictionary representations
file_path: Output CSV file path
"""
import csv
@@ -115,11 +127,14 @@ def save_summary_report(results: Union[Dict[str, Any], List[Dict[str, Any]]],
file_path = Path(file_path)
file_path.parent.mkdir(parents=True, exist_ok=True)
- # Ensure we have a list of results
- if isinstance(results, dict):
- results_list = [results]
+ # Convert EvaluationResult objects to dicts and ensure we have a list
+ def to_dict_if_needed(item):
+ return item.to_dict() if hasattr(item, 'to_dict') else item
+
+ if isinstance(results, list):
+ results_list = [to_dict_if_needed(item) for item in results]
else:
- results_list = results
+ results_list = [to_dict_if_needed(results)]
# Prepare CSV data
csv_data = []
@@ -135,13 +150,10 @@ def save_summary_report(results: Union[Dict[str, Any], List[Dict[str, Any]]],
'success_rate': error_analysis.get('success_rate', 0.0)
}
- # Add only specified metrics
+ # Add all available metrics from overall_metrics
if 'overall_metrics' in result:
- metrics_to_include = ['overall', 'table_extraction', 'formula_extraction']
- for metric_name in metrics_to_include:
- if metric_name in result['overall_metrics']:
- value = result['overall_metrics'][metric_name]
- row[metric_name] = round(value, 4) if isinstance(value, (int, float)) else value
+ for metric_name, value in result['overall_metrics'].items():
+ row[metric_name] = round(value, 4) if isinstance(value, (int, float)) else value
csv_data.append(row)
@@ -153,10 +165,26 @@ def get_sort_key(row):
# Write CSV file
if csv_data:
- # Define specific field order
- fieldnames = ['extractor', 'total_samples', 'success_rate', 'overall', 'table_extraction', 'formula_extraction']
- # Only include fields that exist in the data
- fieldnames = [f for f in fieldnames if f in csv_data[0]]
+ # Define field order: basic info first, then overall, then other metrics alphabetically
+ basic_fields = ['extractor', 'total_samples', 'success_rate']
+
+ # Get all metric fields from the data
+ all_fields = set()
+ for row in csv_data:
+ all_fields.update(row.keys())
+
+ # Remove basic fields from metrics
+ metric_fields = all_fields - set(basic_fields)
+
+ # Sort metrics: overall first, then alphabetically
+ sorted_metrics = []
+ if 'overall' in metric_fields:
+ sorted_metrics.append('overall')
+ metric_fields.remove('overall')
+ sorted_metrics.extend(sorted(metric_fields))
+
+ # Final field order
+ fieldnames = basic_fields + sorted_metrics
with open(file_path, 'w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
diff --git a/webmainbench/metrics/calculator.py b/webmainbench/metrics/calculator.py
index 90171d6..c127947 100644
--- a/webmainbench/metrics/calculator.py
+++ b/webmainbench/metrics/calculator.py
@@ -248,7 +248,11 @@ def get_summary(self, aggregated_results: Dict[str, MetricResult]) -> Dict[str,
def list_available_metrics(self) -> List[str]:
"""List all available metrics."""
- return list(self.metrics.keys())
+ metrics = list(self.metrics.keys())
+ # Add overall metric which is calculated dynamically
+ if "overall" not in metrics:
+ metrics.insert(0, "overall") # Put overall first
+ return metrics
def get_metric_info(self, metric_name: str) -> Optional[Dict[str, Any]]:
"""Get information about a specific metric."""