tool: include contents of codelists in compiled specifications

Author: colmjude

bin/csv_helpers.py

@@ -53,3 +53,50 @@ def write_csv(
         # Write the data rows
         for row in data:
             writer.writerow(row)
+
+
+def csv_to_markdown(
+    filename="data/output.csv",
+    fields=None,
+    exclude_fields=None,
+    first_field="reference",
+    last_field=None,
+    encoding="utf-8",
+):
+    """
+    Convert CSV to markdown table, optionally limiting to specific fields
+    fields: List of field names to include in output. If None, includes all fields.
+    """
+    with open(filename, newline="", encoding=encoding) as csvfile:
+        reader = csv.reader(csvfile)
+        headers = next(reader)
+
+        # Start with headers as read from file
+        headers_to_use = list(headers)
+
+        # If fields specified, restrict to those fields (in CSV order)
+        if fields:
+            headers_to_use = [h for h in headers_to_use if h in fields]
+
+        # If exclude_fields specified, remove them if present
+        if exclude_fields:
+            headers_to_use = [h for h in headers_to_use if h not in exclude_fields]
+
+        # Determine the indices of columns to output (preserve CSV ordering)
+        field_indices = [headers.index(h) for h in headers_to_use]
+
+        # Initialize markdown table with selected headers
+        markdown_table = "| " + " | ".join(headers_to_use) + " |\n"
+        markdown_table += "| " + " | ".join(["---"] * len(headers_to_use)) + " |\n"
+
+        # Add rows with only selected fields
+        for row in reader:
+            selected_values = [row[i] for i in field_indices]
+            # Clean values (strip quotes if present)
+            cleaned_values = [
+                val.strip("'") if val.startswith("'") and val.endswith("'") else val
+                for val in selected_values
+            ]
+            markdown_table += "| " + " | ".join(cleaned_values) + " |\n"
+
+    return markdown_table

bin/generate_info_model.py

@@ -1,4 +1,7 @@
+import os
+
 from applications import get_application_module_refs
+from csv_helpers import csv_to_markdown
 from fields import (
     format_field_display_name,
     get_applicable_app_types,
@@ -23,7 +26,6 @@ def format_fields_table(field_entries, fields_spec, table_type="main", app_type=
                 "| reference | name | description | requirement | notes |",
                 "| --- | --- | --- | --- | --- |",
             ]
-        else:
             lines = [
                 "| reference | name | description | only for application | requirement | notes |",
                 "| --- | --- | --- | --- | --- | --- |",
@@ -141,13 +143,67 @@ def get_codelists_for_app(module_refs, fields, specification):
     return codelists
 
 
+def create_codelist_table(codelist_obj):
+    lines = []
+    name = codelist_obj.get("name", "Unknown")
+
+    source = codelist_obj.get("source", "")
+
+    # Heading
+    heading = f"### {name}"
+    lines.append(heading)
+
+    # Source information
+    if not source:
+        lines.append("\n_codelist source not provided_\n")
+    else:
+        if isinstance(source, str) and (
+            source.startswith("http://") or source.startswith("https://")
+        ):
+            lines.append(f"\nThis codelist is sourced from [{source}]({source})\n")
+        else:
+            path = source
+            if not os.path.isabs(path):
+                repo_root = os.getcwd()
+                path = os.path.join(repo_root, path)
+
+            if not os.path.exists(path):
+                lines.append(f"\nSource file not found: {source}\n")
+                return "\n".join(lines)
+
+            try:
+                md_table = csv_to_markdown(path)
+                lines.append("")
+                lines.append(md_table)
+            except Exception as e:
+                lines.append(f"\nError reading source file {source}: {e}\n")
+
+    return "\n".join(lines) if lines else ""
+
+
 def generate_codelist_md_str(codelists):
+    """
+    Generates a Markdown-formatted string with the codelists required for the
+    specification.
+
+    Args:
+        codelists (list): A list of objects representing codelists. Each
+            object should be convertible to a string or be a dict-like object.
+
+    Returns:
+        str: A Markdown-formatted string enumerating the codelist names, or an
+        empty string if no codelists are provided.
+    """
     # list out the codelist names
     if not codelists:
         return ""
-    lines = ["This are the codelist required to support this specification:\n"]
-    for codelist in codelists:
-        lines.append(f"- {codelist}")
+
+    lines = []
+    lines.append("Below are the codelists required to support this specification:\n")
+    # Sort codelists by their 'name' attribute
+    sorted_codelists = sorted(codelists, key=lambda c: c.get("name", ""))
+    for codelist in sorted_codelists:
+        lines.append(create_codelist_table(codelist))
     return "\n".join(lines)
 
 
@@ -157,6 +213,8 @@ def generate_application(app_ref, specification):
     """
     applications = specification.get("application", {})
     fields = specification.get("field", {})
+    codelists = specification.get("codelist", {})
+
     app = applications.get(app_ref)
     if not app:
         print(f"Application '{app_ref}' not found in specification.")
@@ -210,7 +268,10 @@ def generate_application(app_ref, specification):
     # 6. Required Codelists
     if inc_codelists:
         out.append("## Required codelists\n")
-        out.append(generate_codelist_md_str(inc_codelists))
+        inc_codelist_objs = [
+            codelists.get(ref) for ref in inc_codelists if codelists.get(ref)
+        ]
+        out.append(generate_codelist_md_str(inc_codelist_objs))
 
     return "\n".join(out)
 

bin/integrity_checks/applications.py

@@ -14,6 +14,7 @@
 # 6. if extends then must be a valid application reference in application schemas
 # 7. define which fields are overridden and which are added to
 # 8. listing app types should list base types and their 'children' applications
+# 9. check the application is part of the official application or sub application dataset
 
 
 def check_application_names(applications):