|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Translation QA rules — injectable into verify_translation.py via import. |
| 3 | +New checks: inline math, inline code, translationese, glossary consistency, register. |
| 4 | +""" |
| 5 | +import json, os, re, sys |
| 6 | +from collections import defaultdict |
| 7 | +from dataclasses import dataclass |
| 8 | +from pathlib import Path |
| 9 | +from typing import Sequence |
| 10 | + |
| 11 | +# ═══════════════════════════════════════════════════════════════════ |
| 12 | +# 1. Translationese pattern bank |
| 13 | +# ═══════════════════════════════════════════════════════════════════ |
| 14 | + |
| 15 | +TRANSLATIONESE_PATTERNS = [ |
| 16 | + # (regex, suggestion, severity) |
| 17 | + (r"作为一个\b", "删掉多余的'一个'，改为'作为'", "P2"), |
| 18 | + (r"对于.{1,20}来说", "删掉'对于...来说'，直接写主语", "P2"), |
| 19 | + (r"让我们来看看", "改为'来看' 或 '我们来看'", "P2"), |
| 20 | + (r"被称之为", "改为'叫做' 或 '被称为'", "P2"), |
| 21 | + (r"这是一件非常重要的事情", "精简为'这很重要'", "P2"), |
| 22 | + (r"其被", "主动语态优先，避免'其被'", "P2"), |
| 23 | + (r"人们通常", "简化或省略", "P2"), |
| 24 | + (r"实际上，我们可以", "改为'其实可以' 或删除", "P2"), |
| 25 | + (r"值得注意的是", "改为'注意'", "P2"), |
| 26 | + (r"\b等等\b", "检查是否必要，中文少用省略", "P2"), |
| 27 | + (r"呢\b", "检查是否翻译腔语气词", "P2"), |
| 28 | + (r"的的", "重复'的'——合并或删除", "P2"), |
| 29 | + (r"被\w{2,4}所", "避免'被...所'的文言残留", "P2"), |
| 30 | + (r"进行\w{2,6}(?:处理|操作|分析|研究|实验|训练|推理|优化)", "删掉虚动词'进行'，直接用实义动词", "P2"), |
| 31 | + (r"东西", "改为更准确的名词", "P2"), |
| 32 | +] |
| 33 | + |
| 34 | +# ═══════════════════════════════════════════════════════════════════ |
| 35 | +# 2. Glossary loader |
| 36 | +# ═══════════════════════════════════════════════════════════════════ |
| 37 | + |
| 38 | +@dataclass |
| 39 | +class GlossaryEntry: |
| 40 | + en: str |
| 41 | + zh_preferred: str |
| 42 | + zh_variants: list[str] |
| 43 | + notes: str = "" |
| 44 | + |
| 45 | +def load_glossary(glossary_path: Path) -> dict[str, GlossaryEntry]: |
| 46 | + """Parse GLOSSARY.md into {en_term: GlossaryEntry} dict.""" |
| 47 | + if not glossary_path.exists(): |
| 48 | + return {} |
| 49 | + |
| 50 | + text = glossary_path.read_text(encoding="utf-8") |
| 51 | + entries: dict[str, GlossaryEntry] = {} |
| 52 | + |
| 53 | + for line in text.split('\n'): |
| 54 | + line = line.strip() |
| 55 | + if not line.startswith('|') or line.startswith('|--') or line.startswith('|---'): |
| 56 | + continue |
| 57 | + parts = [p.strip() for p in line.split('|')] |
| 58 | + # Columns: | # | EN | ZH | Notes | |
| 59 | + if len(parts) < 4: |
| 60 | + continue |
| 61 | + en_idx = 2 if len(parts) >= 4 else 1 |
| 62 | + zh_idx = 3 if len(parts) >= 4 else 2 |
| 63 | + en = parts[en_idx] if en_idx < len(parts) else "" |
| 64 | + zh = parts[zh_idx] if zh_idx < len(parts) else "" |
| 65 | + notes = parts[4] if len(parts) > 4 else "" |
| 66 | + |
| 67 | + if not en or not zh or en in ('EN', 'English'): |
| 68 | + continue |
| 69 | + |
| 70 | + # Split ZH variants: "嵌入 / 嵌入向量" → ["嵌入", "嵌入向量"] |
| 71 | + zh_variants = [v.strip() for v in re.split(r'[\s]*[/／][\s]*', zh)] |
| 72 | + preferred = zh_variants[0] if zh_variants else zh |
| 73 | + |
| 74 | + entries[en.lower()] = GlossaryEntry( |
| 75 | + en=en, zh_preferred=preferred, |
| 76 | + zh_variants=zh_variants, notes=notes.strip() |
| 77 | + ) |
| 78 | + |
| 79 | + return entries |
| 80 | + |
| 81 | +# ═══════════════════════════════════════════════════════════════════ |
| 82 | +# 3. New check functions |
| 83 | +# ═══════════════════════════════════════════════════════════════════ |
| 84 | + |
| 85 | +def check_inline_math(source_text: str, target_text: str) -> tuple[bool, list[str]]: |
| 86 | + """Verify $...$ inline math from EN is preserved in ZH. ZH can add math (F12).""" |
| 87 | + src_math = set(re.findall(r'(?<!\$)\$(?!\$)([^$\n]+?)\$(?!\$)', source_text)) |
| 88 | + tgt_math = set(re.findall(r'(?<!\$)\$(?!\$)([^$\n]+?)\$(?!\$)', target_text)) |
| 89 | + # Only flag EN math that is MISSING from ZH (deletions only, additions OK) |
| 90 | + missing = src_math - tgt_math |
| 91 | + return len(missing) == 0, sorted(missing)[:10] |
| 92 | + |
| 93 | +def check_inline_code(source_text: str, target_text: str) -> tuple[bool, list[str]]: |
| 94 | + """Verify inline `code` spans are unchanged. Returns (ok, changed_items).""" |
| 95 | + # Only match single-token inline code (single word, no spaces, no newlines) |
| 96 | + src_codes = set(re.findall(r'`([^`\s\n]+?)`', source_text)) |
| 97 | + tgt_codes = set(re.findall(r'`([^`\s\n]+?)`', target_text)) |
| 98 | + changed = sorted(src_codes - tgt_codes) |
| 99 | + return len(changed) == 0, changed[:10] |
| 100 | + |
| 101 | +def check_heading_parity(source_text: str, target_text: str) -> tuple[bool, dict]: |
| 102 | + """Verify heading count and depth match between source and target.""" |
| 103 | + src_headings = re.findall(r'^(#{1,6})\s', source_text, re.MULTILINE) |
| 104 | + tgt_headings = re.findall(r'^(#{1,6})\s', target_text, re.MULTILINE) |
| 105 | + src_counts = {h: src_headings.count(h) for h in set(src_headings)} |
| 106 | + tgt_counts = {h: tgt_headings.count(h) for h in set(tgt_headings)} |
| 107 | + ok = (len(src_headings) == len(tgt_headings) and src_counts == tgt_counts) |
| 108 | + return ok, {"src_total": len(src_headings), "tgt_total": len(tgt_headings), |
| 109 | + "src_by_depth": src_counts, "tgt_by_depth": tgt_counts} |
| 110 | + |
| 111 | +def check_translationese(text: str) -> list[dict]: |
| 112 | + """Scan target text for translationese patterns.""" |
| 113 | + findings = [] |
| 114 | + for pattern, suggestion, severity in TRANSLATIONESE_PATTERNS: |
| 115 | + for m in re.finditer(pattern, text): |
| 116 | + findings.append({ |
| 117 | + "pattern": pattern, "match": m.group(0), |
| 118 | + "position": m.start(), "suggestion": suggestion, |
| 119 | + "severity": severity |
| 120 | + }) |
| 121 | + return findings |
| 122 | + |
| 123 | +def check_glossary_consistency(text: str, glossary: dict[str, 'GlossaryEntry']) -> list[dict]: |
| 124 | + """Check that glossary terms are used consistently in target text. |
| 125 | + For each EN term, verify only one ZH variant appears in any given file. |
| 126 | + """ |
| 127 | + findings = [] |
| 128 | + for en_term, entry in glossary.items(): |
| 129 | + variants_used = set() |
| 130 | + for variant in entry.zh_variants: |
| 131 | + if variant in text: |
| 132 | + variants_used.add(variant) |
| 133 | + if len(variants_used) > 1: |
| 134 | + findings.append({ |
| 135 | + "en_term": en_term, |
| 136 | + "variants_found": sorted(variants_used), |
| 137 | + "preferred": entry.zh_preferred, |
| 138 | + "message": f"'{en_term}' translated inconsistently: {sorted(variants_used)}" |
| 139 | + }) |
| 140 | + return findings |
| 141 | + |
| 142 | +def check_register_score(text: str) -> float: |
| 143 | + """Score text on a 'conversational vs formal' scale. Higher = more conversational.""" |
| 144 | + conversational = len(re.findall(r'我们|想象|注意|就是|其实|比如|不过|当然|对吧', text)) |
| 145 | + formal = len(re.findall(r'其|该|之|所|者|并非|及其|予以|进行', text)) |
| 146 | + total = conversational + formal |
| 147 | + if total == 0: |
| 148 | + return 0.5 # neutral |
| 149 | + return conversational / total |
| 150 | + |
| 151 | +# ═══════════════════════════════════════════════════════════════════ |
| 152 | +# 4. Chapter mapping: EN dir → ZH dir |
| 153 | +# ═══════════════════════════════════════════════════════════════════ |
| 154 | + |
| 155 | +CHAPTER_MAP = [ |
| 156 | + ("chapter 01 - vectors", "zh/第01章 - 向量"), |
| 157 | + ("chapter 02 - matrices", "zh/第02章 - 矩阵"), |
| 158 | + ("chapter 03 - calculus", "zh/第03章 - 微积分"), |
| 159 | + ("chapter 04 - statistics", "zh/第04章 - 统计学"), |
| 160 | + ("chapter 05 - probability", "zh/第05章 - 概率论"), |
| 161 | + ("chapter 06 - machine learning", "zh/第06章 - 机器学习"), |
| 162 | + ("chapter 07 - computational linguistics", "zh/第07章 - 计算语言学"), |
| 163 | + ("chapter 08 - computer vision", "zh/第08章 - 计算机视觉"), |
| 164 | + ("chapter 09 - audio and speech", "zh/第09章 - 音频与语音"), |
| 165 | + ("chapter 10 - multimodal learning", "zh/第10章 - 多模态学习"), |
| 166 | + ("chapter 11 - autonomous systems", "zh/第11章 - 自主系统"), |
| 167 | + ("chapter 12 - graph neural networks", "zh/第12章 - 图神经网络"), |
| 168 | + ("chapter 13 - computing and OS", "zh/第13章 - 计算与操作系统"), |
| 169 | + ("chapter 14 - data structures and algorithms", "zh/第14章 - 数据结构与算法"), |
| 170 | + ("chapter 15 - production software engineering", "zh/第15章 - 生产级软件工程"), |
| 171 | + ("chapter 16 - SIMD and GPU programming", "zh/第16章 - SIMD 与 GPU 编程"), |
| 172 | + ("chapter 17 - AI inference", "zh/第17章 - AI 推理"), |
| 173 | + ("chapter 18 - ML systems design", "zh/第18章 - 机器学习系统设计"), |
| 174 | + ("chapter 19 - applied AI", "zh/第19章 - 应用 AI"), |
| 175 | + ("chapter 20 - bleeding edge AI", "zh/第20章 - 前沿 AI"), |
| 176 | + ("chapter 21 - alignment, safety & interpretability", "zh/第21章 - 对齐、安全与可解释性"), |
| 177 | + ("chapter 22 - llm evaluation methodology", "zh/第22章 - LLM Evaluation 方法学"), |
| 178 | + ("chapter 23 - ai agent and tool use", "zh/第23章 - AI Agent 与工具使用"), |
| 179 | + ("chapter 24 - numerical analysis and convex optimisation", "zh/第24章 - 数值分析与凸优化补遗"), |
| 180 | + ("chapter 25 - ai system interview guide", "zh/第25章 - AI 系统实战面试指南"), |
| 181 | +] |
| 182 | + |
| 183 | + |
| 184 | +def discover_chapter_pairs(repo_root: Path) -> list[tuple[Path, Path]]: |
| 185 | + """Auto-discover EN+ZH chapter pairs from repo root.""" |
| 186 | + pairs = [] |
| 187 | + for en_name, zh_name in CHAPTER_MAP: |
| 188 | + en_path = repo_root / en_name |
| 189 | + zh_path = repo_root / zh_name |
| 190 | + if en_path.is_dir() and zh_path.is_dir(): |
| 191 | + pairs.append((en_path, zh_path)) |
| 192 | + return pairs |
| 193 | + |
| 194 | + |
| 195 | +# ═══════════════════════════════════════════════════════════════════ |
| 196 | +# 5. Export for verify_translation.py injection |
| 197 | +# ═══════════════════════════════════════════════════════════════════ |
| 198 | + |
| 199 | +__all__ = [ |
| 200 | + "TRANSLATIONESE_PATTERNS", "GlossaryEntry", "load_glossary", |
| 201 | + "check_inline_math", "check_inline_code", "check_heading_parity", |
| 202 | + "check_translationese", "check_glossary_consistency", "check_register_score", |
| 203 | + "CHAPTER_MAP", "discover_chapter_pairs", |
| 204 | +] |
0 commit comments