Use new text annotation format

See https://github.com/nk2028/tshet-uinh-data/wiki/%E6%A0%A1%E5%8B%98%E7%AC%A6%E8%99%9F%E6%A0%BC%E5%BC%8F

The fields 反切 and 字頭 in 廣韻.csv, as well as all source data tables,
now use this standardized format.

Author: syimyuzya

build.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterable
 import csv
 import dataclasses
 from dataclasses import dataclass
@@ -89,14 +90,83 @@ def load_patches() -> dict[tuple[str, str], Patch]:
     return patches
 
 
+def split_head_with_ids(s: str) -> tuple[str, str]:
+    if not s:
+        raise ValueError('empty string')
+    if s[0] in (
+        '⿰',
+        '⿱',
+        '⿴',
+        '⿵',
+        '⿶',
+        '⿷',
+        '⿸',
+        '⿹',
+        '⿺',
+        '⿻',
+        '⿼',
+        '⿽',
+        '㇯',
+    ):
+        num_parts = 2
+    elif s[0] in ('⿲', '⿳'):
+        num_parts = 3
+    elif s[0] in ('⿾', '⿿', '〾'):
+        num_parts = 1
+    else:
+        return s[0], s[1:]
+    idc = s[0]
+    parts = []
+    rest = s[1:]
+    for i in range(num_parts):
+        # if not rest:
+        #     break
+        part, rest = split_head_with_ids(rest)
+        parts.append(part)
+    return idc + ''.join(parts), rest
+
+
+def iter_chars_with_ids(s: str) -> Iterable[str]:
+    while s:
+        head, s = split_head_with_ids(s)
+        yield head
+
+
+# NOTE Only handles simple annotations for now.
+def remove_annotations(original: str) -> str:
+    original = original.replace('`', '')
+    chars = list(iter_chars_with_ids(original))
+    n = len(chars)
+    removable = [False] * n
+    i = 0
+    while i < len(chars):
+        ch = chars[i]
+        if ch in ('［', '］'):
+            removable[i] = True
+            i += 1
+        elif ch == '｛':
+            j = chars.index('｝', i + 1)
+            removable[i : j + 1] = (True,) * (j + 1 - i)
+            i = j + 1
+        elif ch == '〈':
+            j = chars.index('〉', i + 1)
+            removable[i] = removable[j] = True
+            k = j - i - 1
+            assert not any(removable[i - k : i])
+            removable[i - k : i] = (True,) * k
+            i = j + 1
+        else:
+            i += 1
+    return ''.join(ch for ch, rm in zip(chars, removable) if not rm)
+
+
 @dataclass
 class 廣韻Row:
     小韻號: str
     小韻字號: str
     韻目原貌: str
     音韻地位: str
     反切: str
-    字頭原貌: str
     字頭: str
     字頭說明: str
     釋義: str
@@ -124,7 +194,7 @@ def main():
         poem_小韻內字序 = 字序_data[字序_key].poem_小韻內字序
         if not poem_小韻內字序:
             poem_反切 = poem_data[(原書小韻號, '1')]['廣韻反切(覈校後)']
-            含原貌字頭 = ''
+            字頭 = ''
             釋義 = ''
             釋義參照 = ''
         else:
@@ -139,7 +209,7 @@ def main():
                 字頭覈校說明,
                 poem_反切,
                 字頭原貌,
-                含原貌字頭,
+                字頭,
                 釋義,
                 釋義補充,
                 韻目原貌,
@@ -156,7 +226,7 @@ def main():
                 )
             )
             if 字頭覈校說明 == '校':
-                含原貌字頭 = f'[{字頭原貌}/{含原貌字頭}]'
+                字頭 = f'{字頭原貌}〈{字頭}〉'
             if not 釋義:
                 釋義參照 = '下'
             elif 釋義補充:
@@ -167,22 +237,22 @@ def main():
         # 修正
         字頭說明 = ''
         if (patch := patches.get(字序_key)) is not None:
-            assert patch.原字頭 == 含原貌字頭, (
-                f'patching 小韻 #{原書小韻號}/{小韻字號} 字 "{patch.原字頭}", but the actual 字 is "{含原貌字頭}"'
+            assert patch.原字頭 == 字頭, (
+                f'patching 小韻 #{原書小韻號}/{小韻字號} 字 "{patch.原字頭}", but the actual 字 is "{字頭}"'
             )
             patch_coverage.add(字序_key)
             assert patch.校正字頭, (
                 f'patching 小韻 #{原書小韻號}/{小韻字號} 字 "{patch.原字頭}", but 校正字頭 is missing'
             )
-            if patch.校正字頭.startswith('['):
-                assert re.fullmatch(r'\[.+/.+\]', patch.校正字頭), (
-                    f'invalid 校正字頭: "{patch.校正字頭}"'
-                )
+            # TODO Stricter format check
+            assert re.fullmatch(
+                r'｛.+｝|［.+］|.+〈.+〉|[^｛｝［］〈〉]+', patch.校正字頭
+            ), f'invalid 校正字頭: "{patch.校正字頭}"'
             if '～' in patch.校正字頭:
-                assert not 含原貌字頭.startswith('['), (
-                    f'cannot use "～" in 校正字頭 when 字頭 contains correction: "{含原貌字頭}"'
+                assert 字頭 and 字頭[-1] not in tuple('｝］〉'), (
+                    f'cannot use "～" in 校正字頭 when 字頭 contains correction or is empty: "{字頭}"'
                 )
-            含原貌字頭 = patch.校正字頭.replace('～', 含原貌字頭)
+            字頭 = patch.校正字頭.replace('～', 字頭)
 
             # 字頭說明 is an added field, thus it does not have an original value
             字頭說明 = patch.字頭說明
@@ -191,32 +261,29 @@ def main():
                 assert patch.原釋義 == 釋義, (
                     f'patching 釋義 on 小韻 #{原書小韻號}/{小韻字號} 字 "{patch.原字頭}", but the actual 釋義 is "{釋義}"'
                 )
-                corrected = re.sub(r'\[.+?/(?:-|(.+?))\]|[{}]', r'\1', patch.校正釋義)
-                釋義 = corrected
+                釋義 = remove_annotations(patch.校正釋義)
             if patch.校正釋義參照 or patch.原釋義參照:
                 assert patch.原釋義參照 == 釋義參照, (
                     f'patching 釋義參照 on 小韻 #{原書小韻號}/{小韻字號} 字 "{patch.原字頭}", but the actual 釋義參照 is "{釋義參照}"'
                 )
                 釋義參照 = patch.校正釋義參照
-        elif 字序_data[字序_key].sbgy_字.endswith('/-]'):
-            assert not 含原貌字頭.startswith('[')
-            含原貌字頭 = f'[{含原貌字頭}/-]'
+        elif 字序_data[字序_key].sbgy_字.endswith('｝'):
+            assert 字頭[-1:] not in tuple('｝］〉')
+            字頭 = f'｛{字頭}｝'
 
         字_check = 字序_data[字序_key].字
-        assert 含原貌字頭 == 字_check, (
-            f'字頭 mismatch between 字序表 and (patched) 廣韻 data: "{字_check}" != "{含原貌字頭}" (小韻 {原書小韻號}/{小韻字號})'
+        assert 字頭 == 字_check, (
+            f'字頭 mismatch between 字序表 and (patched) 廣韻 data: "{字_check}" != "{字頭}" (小韻 {原書小韻號}/{小韻字號})'
         )
-        if 含原貌字頭.startswith('['):
-            字頭原貌, 字頭 = 含原貌字頭[1:-1].split('/')
-            字頭 = '' if 字頭 == '-' else 字頭
-            字頭原貌 = '' if 字頭原貌 == '-' else 字頭原貌
+        if 字頭[-1] in ('｝', '］'):
+            字頭或原貌 = 字頭[1:-1]
+        elif 字頭[-1] == '〉':
+            字頭或原貌 = 字頭[字頭.index('〈') + 1 : -1]
         else:
-            字頭 = 含原貌字頭
-            字頭原貌 = ''
+            字頭或原貌 = 字頭
 
         # 小韻號
         # NOTE 字頭 & 細分轄字 in 小韻表.tsv does not contain 字頭原貌 (yet)
-        字頭或原貌 = 字頭 or 字頭原貌
         if 原書小韻號 in 細分號_by_原書小韻:
             for 細分 in 細分號_by_原書小韻[原書小韻號]:
                 小韻號 = 原書小韻號 + 細分
@@ -246,7 +313,7 @@ def main():
 
         # 釋義中反切
         if 小韻字號 == '1' and 反切:
-            反切原貌 = re.sub(r'\[.\]|<.>|⦉.⦊|\(.\)|⦅.⦆', '', 反切)
+            反切原貌 = re.sub(r'［.］|〈.〉|〘.〙|（.）|｟.｠', '', 反切)
             if 反切原貌 != poem_反切:
                 assert 釋義.count(poem_反切 + '切') == 1, (
                     f'釋義 not containing {反切}切 exactly once: {釋義}'
@@ -259,7 +326,6 @@ def main():
             韻目原貌,
             音韻地位,
             反切,
-            字頭原貌,
             字頭,
             字頭說明,
             釋義,

check.py

@@ -11,8 +11,8 @@
 )
 PATTERN_反切 = re.compile(
     r"""(?x)(
-        \[.\] |  # 脫字
-        . ( <.> | ⦉.⦊ | \(.\) | ⦅.⦆ )*  # 原貌及校正
+        ［.］ |  # 脫字
+        . ( 〈.〉 | 〘.〙 | （.） | ｟.｠ )*  # 原貌及校正
     ){2}"""
 )
 PATTERN_IDC = re.compile(r'[\u2ff0-\u2fff\u303e\u31ef]')
@@ -29,7 +29,7 @@ def contains_ascii(s: str):
     with open('韻書/廣韻.csv') as f:
         assert (
             next(f).rstrip('\n')
-            == '小韻號,小韻字號,韻目原貌,音韻地位,反切,字頭原貌,字頭,字頭說明,釋義,釋義參照'
+            == '小韻號,小韻字號,韻目原貌,音韻地位,反切,字頭,字頭說明,釋義,釋義參照'
         )
         for line in f:
             (
@@ -38,7 +38,6 @@ def contains_ascii(s: str):
                 韻目原貌,
                 音韻地位描述,
                 反切,
-                字頭原貌,
                 字頭,
                 字頭說明,
                 釋義,
@@ -50,13 +49,9 @@ def contains_ascii(s: str):
                 f'invalid 小韻字號: {小韻字號}'
             )
             assert len(韻目原貌) == 1, f'invalid 韻目原𩩕: {韻目原貌}'
-            assert 字頭原貌 != 字頭, f'字頭原貌 same as 字頭: {字頭}'
-            for field, 字 in (('字頭原貌', 字頭原貌), ('字頭', 字頭)):
-                if not 字:
-                    continue
-                assert 字 != '-' and (len(字) == 1 or PATTERN_IDC.match(字)), (
-                    f'invalid {field}: {字}'
-                )
+            assert re.fullmatch(r'｛.+｝|［.+］|.+〈.+〉|[^｛｝［］〈〉]+', 字頭), (
+                f'invalid 字頭: {字頭}'
+            )
 
             assert PATTERN_描述.fullmatch(音韻地位描述) is not None, (
                 f'invalid 音韻地位: {音韻地位描述}'

src/patches.csv

@@ -1,38 +1,38 @@
 原書小韻號,小韻字號,原字頭,校正字頭,原釋義,校正釋義,原釋義參照,校正釋義參照,字頭說明,備注
 11,2,,融,,上同,,上,,左下為「𢆉」形，poem表因未入U而缺
 130,4,襹,～,襹毛羽衣皃,𧞬襹毛羽衣皃,,,,poem表承「宋本廣韻データ」用 PUA 字元 U+EE42，當為 U+277AC「𧞬」
-141,1,𤿎,[～/𢻹],,,,,,《形聲考》校
+141,1,𤿎,～〈𢻹〉,,,,,,《形聲考》校
 177,6,,𢊿,,上同,,上,,poem表作「⿸广⿳𥫗⺫攵」，因未入U而缺
-213,12,婔,[～/-],,,,,「婓」（滂三C微平）之或體,周祖謨校
-231,13,㶛,[～/-],,,,,本紐重出當刪,周祖謨校
-318,1,攜,～,提也離也又姓出何氏姓苑戶圭切二十四三,提也離也又姓出何氏姓苑{戶圭}切二十[四/三],,,,poem表釋義誤錄
+213,12,婔,｛～｝,,,,,「婓」（滂三C微平）之或體,周祖謨校
+231,13,㶛,｛～｝,,,,,本紐重出當刪,周祖謨校
+318,1,攜,～,提也離也又姓出何氏姓苑戶圭切二十四三,提也離也又姓出何氏姓苑`戶圭`切二十四〈三〉,,,,poem表釋義誤錄
 511,3,𩁚,𩁢,,,,,,poem表兩字形顛倒
 511,4,𩁢,𩁚,,,,,,poem表兩字形顛倒
-597,1,𤜼,[～/-],,,,,「犳」（章開三陽入）之訛字,無效小韻
-646,1,𡰝,[～/𡰖],,,,,,《形聲考》校
+597,1,𤜼,｛～｝,,,,,「犳」（章開三陽入）之訛字,無效小韻
+646,1,𡰝,～〈𡰖〉,,,,,,《形聲考》校
 949,9,䔖,～,,,,上,,poem表遺漏釋義補充
-961,1a1,,嬹,,女字,,,,周祖謨補
-1380,2,皷,[～/鼓],說文曰郭也春分之音萬物郭皮甲而出故謂之皷周禮六皷靁皷靈皷路皷鼖皷鼛皷晉皷亦作鼔,說文曰郭也春分之音萬物郭皮甲而出故謂之鼓周禮六鼓靁鼓靈鼓路鼓鼖鼓鼛鼓晉鼓亦作𡔷,,,,周祖謨校；poem表此二字認同有誤
+961,1a1,,［嬹］,,女字,,,,周祖謨補
+1380,2,皷,～〈鼓〉,說文曰郭也春分之音萬物郭皮甲而出故謂之皷周禮六皷靁皷靈皷路皷鼖皷鼛皷晉皷亦作鼔,說文曰郭也春分之音萬物郭皮甲而出故謂之鼓周禮六鼓靁鼓靈鼓路鼓鼖鼓鼛鼓晉鼓亦作𡔷,,,,周祖謨校；poem表此二字認同有誤
 1380,3,鼓,鼔,說文曰擊皷也,說文曰擊鼓也,,,,poem表此二字認同有誤
 1619,2,𤣗,～,𤣗,,,下,,poem表釋義誤錄
 1883,2,,⿱𱡘正,,俗,,上,,poem表因未入U而缺
-1929,11,𦱙,[～/莥],蔨實亦作𦶆,蔨實亦作[莥/𦱙],,,,周祖謨校
-2021,1,㶒,[～/-],,,,,「㴸」之音，「㶒」（書開三侵上）之字,無效小韻
-2046,17,𣄉,[～/𣃳],掩光又於葉切,[掩光/掩也]又於葉切,,,,周祖謨校；《形聲考》云蓋為「掩」之變
-2369,1,計,～,籌計說文會也筭也又姓後漢有計子古詣切十二,籌計說文會也筭也又姓後漢有計子勳{古詣切}十二,,,,poem表承「宋本廣韻データ」用 PUA 字元 U+ECE0，當為  U+52F3「勳」
+1929,11,𦱙,～〈莥〉,蔨實亦作𦶆,蔨實亦作莥〈𦱙〉,,,,周祖謨校
+2021,1,㶒,｛～｝,,,,,「㴸」之音，「㶒」（書開三侵上）之字,無效小韻
+2046,17,𣄉,～〈𣃳〉,掩光又於葉切,掩光〈也〉又於葉切,,,,周祖謨校；《形聲考》云蓋為「掩」之變
+2369,1,計,～,籌計說文會也筭也又姓後漢有計子古詣切十二,籌計說文會也筭也又姓後漢有計子勳`古詣`切十二,,,,poem表承「宋本廣韻データ」用 PUA 字元 U+ECE0，當為  U+52F3「勳」
 2533,2,概,槩,,,,,,取消poem表「部件換位」調整，與整理反切之用字相統一
-2988,1,盛,～,多也長也又姓後漢西羌傳有北海太守盛苞其先姓奭避元帝諱改姓盛承正切又音成三,多也長也又姓後漢西羌傳有北[海/地]太守盛苞其先姓奭避元帝諱改姓盛{承正}切又音成三,,,,周祖謨校
-2991,1,𣢝,[～/欦],,,,,,《形聲考》校
-3113,1,馾,～,冠幘一曰馬步近前丁紺切三,[冠幘一曰馬步近前/馬睡皃]{丁紺}切[三/四],,,,周祖謨校
-3113,1a1,,帎,,冠幘近前,,,,周祖謨補
+2988,1,盛,～,多也長也又姓後漢西羌傳有北海太守盛苞其先姓奭避元帝諱改姓盛承正切又音成三,多也長也又姓後漢西羌傳有北海〈地〉太守盛苞其先姓奭避元帝諱改姓盛`承正`切又音成三,,,,周祖謨校
+2991,1,𣢝,～〈欦〉,,,,,,《形聲考》校
+3113,1,馾,～,冠幘一曰馬步近前丁紺切三,｛冠幘一曰馬步近前｝［馬睡皃］`丁紺`切三〈四〉,,,,周祖謨校
+3113,1a1,,［帎］,,冠幘近前,,,,周祖謨補
 3276,15,,⿰隺犬,,至也高也,,,,poem表因未入U而缺
 3291,7,,⿱芖雨,,俗,,上,,poem表作「⿱共雨」，因未入U而缺
 3292,2,鷝,鵯,,,,,,poem表誤校；此實為偽字，保持原樣即可
-3373,1,𣅝,[～/-],,,,,「突」之音，「𠬛」（明一魂入）之訛字,無效小韻
-3389,4,紇,～,絲下也又孔子父名又虜複姓三氏北齊開府紇奚永樂又有紇干氏紇骨氏又虜三字姓後魏有賊師紇豆陵伊利又胡結切,絲下也又孔子父名又虜複姓三氏北齊開府紇奚永樂又有紇[干/于]氏紇骨氏又虜三字姓後魏有賊師紇豆陵伊利又胡結切,,,,周祖謨校；poem表承「宋本廣韻データ」誤錄
-3390,1,搰,～,掘地也戶骨切于十,掘地也{戶骨}切[十一/十],,,,周祖謨校；poem表承「宋本廣韻データ」誤錄
+3373,1,𣅝,｛～｝,,,,,「突」之音，「𠬛」（明一魂入）之訛字,無效小韻
+3389,4,紇,～,絲下也又孔子父名又虜複姓三氏北齊開府紇奚永樂又有紇干氏紇骨氏又虜三字姓後魏有賊師紇豆陵伊利又胡結切,絲下也又孔子父名又虜複姓三氏北齊開府紇奚永樂又有紇干〈于〉氏紇骨氏又虜三字姓後魏有賊師紇豆陵伊利又胡結切,,,,周祖謨校；poem表承「宋本廣韻データ」誤錄
+3390,1,搰,～,掘地也戶骨切于十,掘地也`戶骨`切十｛一｝,,,,周祖謨校；poem表承「宋本廣韻データ」誤錄
 3390,4,𦗣,～,耳黧,耳聲,,,,周祖謨校；《形聲考》云此字當為戶骨⦅兀⦆<瓦>切為「𦖍」（下瓦反）字之訛
-3390,5,𦖼,[～/-],,,,,,周祖謨校
-3829,5,𣄉,[～/𣃳],掩光名掩也,[掩光名/掩也],,,,周祖謨校；《形聲考》云蓋為「掩」之變
-3829,6,厭,～,厭伏亦惡夢又於琰切六,厭伏亦惡夢又於琰切[六/-],,,,周祖謨校；poem表誤錄
-3830,6,貼,～,以物之質錢,以物[之/-]質錢,,,,周祖謨校；poem表誤錄
+3390,5,𦖼,｛～｝,,,,,,周祖謨校
+3829,5,𣄉,～〈𣃳〉,掩光名掩也,掩｛光名｝［也］,,,,周祖謨校；《形聲考》云蓋為「掩」之變
+3829,6,厭,～,厭伏亦惡夢又於琰切六,厭伏亦惡夢又於琰切｛六｝,,,,周祖謨校；poem表誤錄
+3830,6,貼,～,以物之質錢,以物｛之｝質錢,,,,周祖謨校；poem表誤錄