PROOF OF CONCEPT -- comments welcome

Per discussion from
https://github.com/open-mpi/ompi/wiki/Meeting-2025-03-14, we would
like to make relocating an Open MPI installation easier.
Specifically, if we can avoid the need to propagate various *_PREFIX
environment variables / command line options down through multiple
layers and subsystems (OMPI, PRRTE, PMIx), that would
eliminate/simplify a bunch of our code.

Text help files are something that Open MPI currently has to find in
the install tree filesystem at run-time.

This commit is a proposal:

1. Developers still maintain help messages in the various help_*.txt
   files around the code base.

   Maintaining descriptive, user-friendly help messages in text files
   (instead of manually hand-coding long strings in C) has proven to
   be quite useful.  We do not want to lose this capability.

2. During "make" (in opal/util), those help_*.txt files are encoded
   into an indexed array of C strings in
   opal/util/show_help_content.c.

3. opal_show_help*() then can look up the appropriate help strings via
   filename / topic tuples, just like it used to -- these strings now
   just happen to be in C variables instead of text files.

A single change will need to be made wherever a help_*.txt file is
used (e.g., in components, but also in various base directories):
update Makefile.am to remove the help_*.txt file from dist_*_DATA and
it add to EXTRA_DIST.

**NOTE:** This Makefile.am change has not been made across the code
	  base yet.  Once we agree on the prototype/ideas, the
	  Makefile.am change can be implemented across the code base.

This work is intended for main / v6.0.x -- not for v5.0.x.

Signed-off-by: Jeff Squyres <[email protected]>

Author: jsquyres

.gitignore

@@ -313,7 +313,7 @@ opal/tools/wrappers/opalCC-wrapper-data.txt
 opal/tools/wrappers/opal_wrapper
 opal/tools/wrappers/opal.pc
 
-opal/util/show_help_lex.c
+opal/util/show_help_content.c
 opal/util/keyval/keyval_lex.c
 
 test/monitoring/aggregate_profile.pl

opal/mca/btl/tcp/Makefile.am

@@ -20,7 +20,7 @@
 # $HEADER$
 #
 
-dist_opaldata_DATA = help-mpi-btl-tcp.txt
+EXTRA_DIST = help-mpi-btl-tcp.txt
 
 sources = \
     btl_tcp.c \

opal/util/Makefile.am

@@ -27,16 +27,16 @@
 # $HEADER$
 #
 
+include $(top_srcdir)/Makefile.ompi-rules
+
 SUBDIRS = \
         json \
         keyval
 
-dist_opaldata_DATA = \
+EXTRA_DIST = \
         help-opal-util.txt \
-        json/help-json.txt
-
-AM_LFLAGS = -Popal_show_help_yy
-LEX_OUTPUT_ROOT = lex.opal_show_help_yy
+        json/help-json.txt \
+        convert-help-files-to-c-code.py
 
 noinst_LTLIBRARIES = libopalutil.la libopalutil_core.la
 
@@ -76,7 +76,6 @@ headers = \
         proc.h \
         qsort.h \
         show_help.h \
-        show_help_lex.h \
         stacktrace.h \
         string_copy.h \
         sys_limits.h \
@@ -121,12 +120,17 @@ libopalutil_core_la_SOURCES = \
         qsort.c \
         sha256.c \
         show_help.c \
-        show_help_lex.l \
+        show_help_content.c \
         stacktrace.c \
         string_copy.c \
         sys_limits.c \
         uri.c
 
+show_help_content.c: convert-help-files-to-c-code.py
+	$(OMPI_V_GEN) $(abs_srcdir)/convert-help-files-to-c-code.py \
+	    --root $(abs_top_srcdir) \
+	    --out show_help_content.c
+
 if OPAL_COMPILE_TIMING
 libopalutil_core_la_SOURCES += timings.c
 endif
@@ -138,18 +142,12 @@ libopalutil_core_la_DEPENDENCIES = \
         json/libopalutil_json.la \
         keyval/libopalutilkeyval.la
 
-# flex prior to version 2.6.6 uses the POSIX extension fileno()
-# without providing the proper feature test macro, so
-# we add the _POSIX_C_SOURCE macro here.
-# See https://github.com/westes/flex/issues/263
-libopalutil_core_la_CFLAGS = -D_POSIX_C_SOURCE=200809L
-
 # Conditionally install the header files
 
 if WANT_INSTALL_HEADERS
 opaldir = $(opalincludedir)/$(subdir)
 opal_HEADERS = $(headers)
 endif
 
-maintainer-clean-local:
-	rm -f show_help_lex.c
+# This file is generated
+CLEANFILES = show_help_content.c

opal/util/convert-help-files-to-c-code.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2025      Jeffrey M. Squyres.  All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+import os
+import sys
+import argparse
+
+def find_help_files(root, verbose=False):
+    # Search for help-*.txt files across the source tree, skipping
+    # some directories (e.g., 3rd-party)
+    help_files = []
+    skip_dirs = ['.git', '3rd-party']
+    for root_dir, dirs, files in os.walk(root):
+        for sd in skip_dirs:
+            if sd in dirs:
+                dirs.remove(sd)
+
+        for file in files:
+            if file.startswith("help-") and file.endswith(".txt"):
+                full_path = os.path.join(root_dir, file)
+                help_files.append(full_path)
+                if verbose:
+                    print(f"Found: {full_path}")
+    return help_files
+
+def parse_ini_files(file_paths, verbose=False):
+    # Parse INI-style files, returning a dictionary with filenames as
+    # keys.  Don't use the Python configparse module in order to
+    # reduce dependencies (i.e., so that we don't have to pip install
+    # anything to run this script).
+    data = {}
+    for file_path in file_paths:
+        sections = {}
+        current_section = None
+        with open(file_path) as file:
+            for line in file:
+                line = line.strip()
+                if line.startswith('#') or not line:
+                    continue
+                if line.startswith('[') and line.endswith(']'):
+                    current_section = line[1:-1]
+                    sections[current_section] = list()
+                elif current_section is not None:
+                    sections[current_section].append(line)
+
+        data[os.path.basename(file_path)] = sections
+
+        if verbose:
+            print(f"Parsed: {file_path} ({len(sections)} sections found)")
+
+    return data
+
+def generate_c_code(parsed_data):
+    # Generate C code with an array of filenames and their
+    # corresponding INI sections.
+    c_code = f"""// THIS FILE IS GENERATED AUTOMATICALLY! EDITS WILL BE LOST!
+// This file generated by {sys.argv[0]}
+
+"""
+    # Rather than escaping the C code {} in f strings, make this a
+    # separate (non-f-string) addition to c_code.
+    c_code += """#include <stdio.h>
+#include <string.h>
+
+typedef struct {
+    const char *section;
+    const char *content;
+} ini_entry;
+
+typedef struct {
+    const char *filename;
+    ini_entry *entries;
+} file_entry;
+
+"""
+
+    ini_arrays = []
+    file_entries = []
+
+    for idx, (filename, sections) in enumerate(parsed_data.items()):
+        var_name = filename.replace('-', '_').replace('.', '_')
+
+        ini_entries = []
+        for section, content_list in sections.items():
+            content = '\n'.join(content_list)
+            c_content = content.replace('"','\\"').replace("\n", '\\n"\n"')
+            ini_entries.append(f'    {{ "{section}", "{c_content}" }}')
+        ini_entries.append(f'    {{ NULL, NULL }}')
+
+        ini_array_name = f"ini_entries_{idx}"
+        ini_arrays.append(f"static ini_entry {ini_array_name}[] = {{\n" + ",\n".join(ini_entries) + "\n};\n")
+        file_entries.append(f'    {{ "{filename}", {ini_array_name} }}')
+    file_entries.append(f'    {{ NULL, NULL }}')
+
+    c_code += "\n".join(ini_arrays) + "\n"
+    c_code += "static file_entry help_files[] = {\n" + ",\n".join(file_entries) + "\n};\n"
+
+    c_code += """
+
+const char *opal_show_help_get_content(const char *filename, const char* topic)
+{
+    file_entry *fe;
+    ini_entry *ie;
+
+    for (int i = 0; help_files[i].filename != NULL; ++i) {
+        fe = &(help_files[i]);
+        if (strcmp(fe->filename, filename) == 0) {
+            for (int j = 0; fe->entries[j].section != NULL; ++j) {
+                ie = &(fe->entries[j]);
+                if (strcmp(ie->section, topic) == 0) {
+                    return ie->content;
+                }
+            }
+        }
+    }
+
+    return NULL;
+}
+"""
+
+    return c_code
+
+#-------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate C code from help text INI files.")
+    parser.add_argument("--root",
+                        required=True,
+                        help="Root directory to search for help-*.txt files")
+    parser.add_argument("--out",
+                        required=True,
+                        help="Output C file")
+    parser.add_argument("--verbose",
+                        action="store_true",
+                        help="Enable verbose output")
+    args = parser.parse_args()
+
+    if args.verbose:
+        print(f"Searching in: {args.root}")
+
+    file_paths = find_help_files(args.root, args.verbose)
+    parsed_data = parse_ini_files(file_paths, args.verbose)
+    c_code = generate_c_code(parsed_data)
+
+    if os.path.exists(args.out):
+        with open(args.out) as f:
+            existing_content = f.read()
+
+            if existing_content == c_code:
+                if args.verbose:
+                    print(f"Help string content has not changed; not re-writing {args.out}")
+                exit(0)
+
+    with open(args.out, "w") as f:
+        f.write(c_code)
+
+    if args.verbose:
+        print(f"Generated C code written to {args.out}")
+
+if __name__ == "__main__":
+    main()