Maintenance: PDF generator fixed to not bail out with new section mapping (WIP still appendix mappings broken)

Signed-off-by: Stefan Hagen <stefan@hagen.link>

Author: sthagen

csaf_2.1/prose/edit/bin/lapidify.py

@@ -10,7 +10,9 @@
 import yaml
 
 ENCODING = 'utf-8'
+ENC_ERRS = 'ignore'
 NL = '\n'
+RS = chr(30)  # Record Separator
 CB_END = '}'
 COLON = ':'
 DASH = '-'
@@ -22,7 +24,7 @@
 SPACE = ' '
 TM = '™'
 
-DEBUG = bool(os.getenv('LAPIDIFY_DEBUG', ''))
+DEBUG = bool(os.getenv('DEBUG_LAPIDIFY', ''))
 TARGETS = (
     PDF := 'pdf',
     GFM_PLUS := 'gfm+vendor_hacks',
@@ -32,11 +34,13 @@
 DUMP_LUT = bool(os.getenv('DUMP_LUT', ''))
 
 # Configuration and runtime parameter candidates:
+GREMLINS = ' .,;?!_()[]{}<>\\/$:"\'`´'
 BINDER_AT = pathlib.Path('etc') / 'bind.txt'
 SOURCE_AT = pathlib.Path('src')
 BUILD_AT = pathlib.Path('build')
 SECTION_DISPLAY_TO_LABEL_AT = pathlib.Path('etc') / 'section-display-to-label.json'
 SECTION_LABEL_TO_DISPLAY_AT = pathlib.Path('etc') / 'section-label-to-display.json'
+SECTION_DISPLAY_TO_TEXT_AT = pathlib.Path('etc') / 'section-display-to-text.json'
 EG_GLOBAL_TO_LABEL_AT = pathlib.Path('etc') / 'example-global-to-local.json'
 EG_LABEL_TO_GLOBAL_AT = pathlib.Path('etc') / 'example-local-to-global.json'
 
@@ -81,6 +85,7 @@
 """
 CLEAN_MD_START = '# Introduction'
 FENCED_BLOCK_FLIP_FLOP = '```'
+APPENDIX_INNER_PATTERN = re.compile(r'(?P<display>[A-Z][\.0-9]+)\ +(?P<rest>.+)')
 LOGO_URL = 'https://docs.oasis-open.org/templates/OASISLogo-v3.0.png'
 LOGO_LOCAL_PATH = 'images/OASISLogo-v3.0.png'
 TOP_LOGO_LINE = f'![OASIS Logo]({LOGO_URL})'
@@ -137,7 +142,7 @@
 
 def load_binder(binder_at: Union[str, pathlib.Path], ignores: Union[list[str], None] = None) -> list[pathlib.Path]:
     """Load the linear binder text file into a list of file paths."""
-    with open(binder_at, 'rt', encoding=ENCODING) as resource:
+    with open(binder_at, 'rt', encoding=ENCODING, errors=ENC_ERRS) as resource:
         collation = (pathlib.Path(entry.strip()) for entry in resource.readlines() if entry.strip())
     return [path for path in collation if str(path) not in ignores] if ignores else list(collation)
 
@@ -165,25 +170,44 @@ def detect_meta(text_lines: list[str]) -> tuple[META_TOC_TYPE, list[str]]:
 
 def load_document(path: Union[str, pathlib.Path]) -> tuple[META_TOC_TYPE, list[str]]:
     """Load the text file into a list of strings and harvest any YAML meta info (if present remove the lines)."""
-    with open(path, 'rt', encoding=ENCODING) as resource:
+    with open(path, 'rt', encoding=ENCODING, errors=ENC_ERRS) as resource:
         return detect_meta(resource.readlines())
 
 
 def dump_assembly(text_lines: list[str], to_path: Union[str, pathlib.Path]) -> None:
     """Dump the lines of text into the text file at path."""
-    with open(to_path, 'wt', encoding=ENCODING) as resource:
+    with open(to_path, 'wt', encoding=ENCODING, errors=ENC_ERRS) as resource:
         resource.write(''.join(text_lines))
 
 
-def label_derive_from(text: str) -> str:
-    """Transform text to kebab style conventional label assuming no newlines present."""
-    good_nuff = (' ', '.', ',', ';', '?', '!', '_', '(', ')', '[', ']', '{', '}', '<', '>', '\\', '/', '$', ':')
-    slug = text.strip()
-    for bad in good_nuff:
-        slug = slug.replace(bad, DASH)
-    parts = slug.split(DASH)
-    slug = DASH.join(s for s in parts if s)  #  and s != DASH)
-    return slug.lower()
+def slugify(
+    text: str,
+    connector: str = DASH,
+    marker: str = RS,
+    gremlins: str = GREMLINS,
+    policy: str = 'lower',
+) -> str:
+    """Derive kebab style slug from text.
+
+    Implementer notes:
+
+    - Every character not in gremlins is kept.
+    - Incoming connector chars (default dashes) are preserved by
+      sandwich transform to and from marker char (default ASCII RS).
+      If the marker char occurs in the text, it will be replaced
+      with the connector char during the back transform.
+    """
+    ds = connector
+    rs = marker
+
+    sl = text.strip().replace(ds, rs)
+    for gremlin in gremlins:
+        sl = sl.replace(gremlin, ds)
+
+    return getattr(
+        ds.join(s.replace(rs, ds) for s in sl.split(ds) if s and s != ds),
+        policy
+    )()
 
 
 def label_in(text: str) -> bool:
@@ -217,25 +241,31 @@ def code_block_label_in(text: str) -> bool:
 
 def load_label_to_display_lut(path: Union[str, pathlib.Path] = SECTION_LABEL_TO_DISPLAY_AT) -> dict[str, str]:
     """Load the LUT for section labels -> display."""
-    with pathlib.Path(path).open('rt', encoding=ENCODING) as handle:
+    with pathlib.Path(path).open('rt', encoding=ENCODING, errors=ENC_ERRS) as handle:
         return json.load(handle)
 
 
 def load_display_to_label_lut(path: Union[str, pathlib.Path] = SECTION_DISPLAY_TO_LABEL_AT) -> dict[str, str]:
     """Load the LUT for section display -> labels."""
-    with pathlib.Path(path).open('rt', encoding=ENCODING) as handle:
+    with pathlib.Path(path).open('rt', encoding=ENCODING, errors=ENC_ERRS) as handle:
+        return json.load(handle)
+
+
+def load_display_to_text_lut(path: Union[str, pathlib.Path] = SECTION_DISPLAY_TO_TEXT_AT) -> dict[str, str]:
+    """Load the LUT for section display -> labels."""
+    with pathlib.Path(path).open('rt', encoding=ENCODING, errors=ENC_ERRS) as handle:
         return json.load(handle)
 
 
 def load_eg_label_to_global_lut(path: Union[str, pathlib.Path] = EG_LABEL_TO_GLOBAL_AT) -> dict[str, str]:
     """Load the LUT for example labels -> global."""
-    with pathlib.Path(path).open('rt', encoding=ENCODING) as handle:
+    with pathlib.Path(path).open('rt', encoding=ENCODING, errors=ENC_ERRS) as handle:
         return json.load(handle)
 
 
 def load_eg_global_to_label_lut(path: Union[str, pathlib.Path] = EG_GLOBAL_TO_LABEL_AT) -> dict[str, str]:
     """Load the LUT for example global -> labels."""
-    with pathlib.Path(path).open('rt', encoding=ENCODING) as handle:
+    with pathlib.Path(path).open('rt', encoding=ENCODING, errors=ENC_ERRS) as handle:
         return json.load(handle)
 
 
@@ -281,6 +311,13 @@ def insert_any_section_reference(record: str) -> str:
                     raise RuntimeError(f'false positive sec ref in ({record.rstrip(NL)})')
                 label = found['label']
                 if label not in SEC_LABEL_TEXT:
+                    print(f'ERROR: in insert-any-section-reference ({record=})')
+                    print(f'ERROR-CONTEXT: {record=} - {trigger_text=}')
+                    print(f'ERROR-CONTEXT: {record=} - {label=}')
+                    for skey in SEC_LABEL_TEXT:
+                        if skey.startswith(label[:len(label) // 2]):
+                            print(f'DEBUG: - similar {skey=} exists')
+                    print(f'DEBUG: You may want to execute grep -n {label} src/*.md')
                     raise RuntimeError(f'missing register label for sec ref in ({record.rstrip(NL)})')
                 text = SEC_LABEL_TEXT[label]
                 sem_ref = f'[sec](#{label})'
@@ -402,7 +439,7 @@ def main(args: list[str]) -> int:
                     in_definition = True
                     # prepare the data triplet
                     term = line.strip()
-                    label = 'def;' + label_derive_from(term)
+                    label = 'def;' + slugify(term)
                     definition = ''
                     continue
                 if in_definition:
@@ -527,7 +564,7 @@ def main(args: list[str]) -> int:
                     label = text.split(TOK_LAB, 1)[1].rstrip(CB_END)
                     # reduced_text = text.split(TOK_LAB, 1)[0]
                 else:
-                    label = label_derive_from(text)
+                    label = slugify(text)
                 clean_sec_cnt_disp = (f'{sec_cnt_disp}' if is_plain else sec_cnt_disp).rstrip(FULL_STOP)
                 SEC_LABEL_TEXT[label] = clean_sec_cnt_disp
                 SECTION_DISPLAY_TO_LABEL[clean_sec_cnt_disp] = label
@@ -595,12 +632,32 @@ def main(args: list[str]) -> int:
             pl_anchor = TOK_EG.replace('$thing$', magic_label)
             line = line.rstrip(NL) + pl_anchor + NL
             # now the UX bonus:
-            sec_disp = 'sec-' + display_from[section].replace(FULL_STOP, '-')  # type: ignore
+            try:
+                sec_disp_context_part = display_from[section]  # type: ignore
+            except KeyError as err:
+                print(f'ERROR: {slot=} in example-refs-processing ({err})')
+                print(f'ERROR-CONTEXT: {slot=} - {line=}')
+                print(f'ERROR-CONTEXT: {slot=} - {section=}')
+                for skey in display_from:
+                    if skey.startswith(section[:len(section) // 2]):  # type: ignore
+                        print(f'DEBUG: - similar {skey=} exists')
+                return 1
+            sec_disp = 'sec-' + sec_disp_context_part.replace(FULL_STOP, '-')  # type: ignore
             sec_disp_num_label = f'{sec_disp}-eg-{num}'
             sec_disp_num_anchor = TOK_EG.replace('$thing$', sec_disp_num_label)
             line = line.rstrip(NL) + sec_disp_num_anchor + NL
             # now the global counter extra:
-            global_example_num = eg_global_from[magic_label]
+            try:
+                global_example_num = eg_global_from[magic_label]
+            except KeyError as err:
+                print(f'ERROR: {slot=} in example-refs-global-counter-lookup ({err})')
+                print(f'ERROR-CONTEXT: {slot=} - {line=}')
+                print(f'ERROR-CONTEXT: {slot=} - {magic_label=}')
+                for ekey in eg_global_from:
+                    if ekey.startswith(magic_label[:len(magic_label) // 2]):
+                        print(f'DEBUG: - similar {ekey=} exists')
+                return 1
+
             global_example_num_label = f'example-{global_example_num}'
             global_example_num_anchor = TOK_EG.replace('$thing$', global_example_num_label)
             line = line.rstrip(NL) + global_example_num_anchor + NL
@@ -710,16 +767,16 @@ def main(args: list[str]) -> int:
     BUILD_AT.mkdir(parents=True, exist_ok=True)
     dump_assembly(lines, BUILD_AT / 'pdf.md')
 
-    with open(BUILD_AT / 'toc-mint.json', 'wt', encoding=ENCODING) as handle:
+    with open(BUILD_AT / 'toc-mint.json', 'wt', encoding=ENCODING, errors=ENC_ERRS) as handle:
         json.dump(mint, handle, indent=2)
 
     if DUMP_LUT:
-        with SECTION_DISPLAY_TO_LABEL_AT.open('wt', encoding=ENCODING) as handle:
+        with SECTION_DISPLAY_TO_LABEL_AT.open('wt', encoding=ENCODING, errors=ENC_ERRS) as handle:
             json.dump(SECTION_DISPLAY_TO_LABEL, handle, indent=2)
         section_label_to_display = {
             label: disp for label, disp in sorted((label, disp) for disp, label in SECTION_DISPLAY_TO_LABEL.items())
         }
-        with SECTION_LABEL_TO_DISPLAY_AT.open('wt', encoding=ENCODING) as handle:
+        with SECTION_LABEL_TO_DISPLAY_AT.open('wt', encoding=ENCODING, errors=ENC_ERRS) as handle:
             json.dump(section_label_to_display, handle, indent=2)
 
     return 0