Rename scripts and add json parser. (#160)

Author: cpyle0819

aws_doc_sdk_examples_tools/agent/README.md

@@ -0,0 +1,82 @@
+# Ailly Prompt Workflow
+
+This project provides two scripts to **generate**, **run**, and **parse** [Ailly](https://www.npmjs.com/package/@ailly/cli) prompts based on your AWS DocGen snippets.
+
+## Overview
+
+1. **Generate** Ailly prompts from DocGen snippets (`make_prompts.py`).
+2. **Run** the Ailly CLI on the generated prompts (`npx @ailly/cli`).
+3. **Parse** the Ailly outputs back into structured JSON (`parse_json_files.py`).
+
+---
+
+## Prerequisites
+
+- Python 3.8+
+- Node.js & npm (for `npx`)
+
+---
+
+## Step 1: Generate Ailly prompts
+
+Use `make_prompts.py` to create a directory of Markdown files and a `.aillyrc` configuration file.
+
+```bash
+python make_prompts.py \
+  --doc-gen-root /path/to/your/docgen/project \
+  --system-prompts "You are a helpful assistant..." \
+  --out .ailly_prompts
+```
+
+**Arguments:**
+- `--doc-gen-root`: Path to your DocGen project root.
+- `--system-prompts`: One or more system prompts, either as strings or file paths.
+- `--out`: (Optional) Output directory. Defaults to `.ailly_prompts`.
+
+This will:
+- Create Markdown files for each snippet.
+- Create a `.aillyrc` file with the provided system prompts.
+
+---
+
+## Step 2: Run Ailly CLI
+
+Run Ailly on the generated prompts. From inside the .ailly_prompts directory:
+
+```bash
+npx @ailly/cli --root .ailly_prompts
+```
+
+This will create `{file}.ailly.md` output files in the `.ailly_prompts` directory (or whatever output directory you specified).
+
+---
+
+## Step 3: Parse Ailly output
+
+After Ailly has generated response files (`*.ailly.md`), parse them into JSON objects using `parse_json_files.py`:
+
+```bash
+python parse_json_files.py .ailly_prompts/*.ailly.md --out output.json
+```
+
+**Arguments:**
+- Positional: List of files to parse (you can use a glob like `*.ailly.md`).
+- `--out`: (Optional) Output file path for the resulting JSON array. Defaults to `out.json`.
+
+---
+
+## Example Full Workflow
+
+```bash
+# Step 1: Generate prompts
+python make_prompts.py \
+  --doc-gen-root ~/projects/aws-docgen-root \
+  --system-prompts system_prompt.txt \
+  --out .ailly_prompts
+
+# Step 2: Run Ailly
+npx @ailly/cli --root .ailly_prompts
+
+# Step 3: Parse results
+python parse_json_files.py .ailly_prompts/*.ailly.md --out results.json
+```

aws_doc_sdk_examples_tools/scripts/make_prompts.py renamed to aws_doc_sdk_examples_tools/agent/make_prompts.py

@@ -54,7 +54,9 @@ def parse_prompts_arg(values: List[str]) -> List[str]:
     return prompts
 
 
-def main(doc_gen_root: Path, system_prompts: List[str], out: str = ".ailly_prompts") -> None:
+def main(
+    doc_gen_root: Path, system_prompts: List[str], out: str = ".ailly_prompts"
+) -> None:
     """Generate prompts and configuration files for Ailly."""
     out_path = Path(out)
     setup_ailly(system_prompts, out_path)
@@ -68,17 +70,18 @@ def main(doc_gen_root: Path, system_prompts: List[str], out: str = ".ailly_promp
         description="Write Ailly prompts for DocGen snippets and parse the results."
     )
     parser.add_argument(
-        "--doc-gen-root", required=True,
-        help="Path to a DocGen ready project."
+        "--doc-gen-root", required=True, help="Path to a DocGen ready project."
     )
     parser.add_argument(
-        "--system-prompts", nargs="+", required=True,
-        help="List of prompt strings or file paths to store in a .aillyrc file."
+        "--system-prompts",
+        nargs="+",
+        required=True,
+        help="List of prompt strings or file paths to store in a .aillyrc file.",
     )
     parser.add_argument(
         "--out",
         default=".ailly_prompts",
-        help="Directory where Ailly prompt files will be written. Defaults to '.ailly_prompts'."
+        help="Directory where Ailly prompt files will be written. Defaults to '.ailly_prompts'.",
     )
 
     args = parser.parse_args()

aws_doc_sdk_examples_tools/agent/parse_json_files.py

@@ -0,0 +1,77 @@
+import argparse
+import json
+import logging
+from os.path import basename
+from pathlib import Path
+from typing import List, Dict, Optional
+
+logging.basicConfig(level=logging.WARNING)
+logger = logging.getLogger(__name__)
+
+
+def extract_json_from_text(text: str) -> Optional[Dict]:
+    brace_stack: List[str] = []
+    json_start = None
+
+    for i, char in enumerate(text):
+        if char == "{":
+            if not brace_stack:
+                json_start = i
+            brace_stack.append(char)
+        elif char == "}":
+            if brace_stack:
+                brace_stack.pop()
+                if not brace_stack and json_start is not None:
+                    json_candidate = text[json_start : i + 1]
+                    try:
+                        return json.loads(json_candidate)
+                    except json.JSONDecodeError:
+                        logger.warning(
+                            "Found a brace-balanced block, but it isn't valid JSON."
+                        )
+                        # Continue searching
+    return None
+
+
+def process_files(file_paths: List[str]) -> Dict[str, Dict]:
+    results = {}
+    for path in file_paths:
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()
+            json_data = extract_json_from_text(content)
+            if json_data is not None:
+                results[basename(path)] = json_data
+            else:
+                logger.warning(f"No valid JSON object found in file: {f.name}")
+        except Exception as e:
+            logger.warning(f"Error processing file {path}: {e}")
+    return results
+
+
+def write_objects(object: Dict, out: Path):
+    out.write_text(json.dumps(object, indent=2))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract JSON objects from one or more text files."
+    )
+
+    parser.add_argument("files", nargs="+", help="List of files to process")
+
+    parser.add_argument(
+        "--out",
+        default="out.json",
+        help="File path where the resultant json will be written.",
+    )
+
+    args = parser.parse_args()
+
+    json_objects = process_files(args.files)
+    out = Path(args.out)
+    write_objects(json_objects, out)
+
+
+if __name__ == "__main__":
+    main()