diff --git a/.gitignore b/.gitignore index e5a4825..704c243 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__ build/ dist/ .ailly_iam_policy +*.log \ No newline at end of file diff --git a/aws_doc_sdk_examples_tools/lliam/README.md b/aws_doc_sdk_examples_tools/lliam/README.md index 6955f13..cddd9f8 100644 --- a/aws_doc_sdk_examples_tools/lliam/README.md +++ b/aws_doc_sdk_examples_tools/lliam/README.md @@ -1,78 +1,108 @@ -# Ailly Prompt Workflow +# Lliam - LLM-powered IAM Policy Metadata Enhancement -This project automates the process of generating, running, parsing, and applying [Ailly](https://www.npmjs.com/package/@ailly/cli) prompt outputs to an AWS DocGen project. It combines all steps into one streamlined command using a single Python script. +Lliam automates the process of generating, running, parsing, and applying [Ailly](https://www.npmjs.com/package/@ailly/cli) prompt outputs to enhance AWS DocGen IAM policy examples with rich metadata. --- ## 📦 Overview -This tool: -1. **Generates** Ailly prompts from DocGen snippets. -2. **Runs** Ailly CLI to get enhanced metadata. -3. **Parses** Ailly responses into structured JSON. -4. **Updates** your DocGen examples with the new metadata. +Lliam provides a multi-step workflow to: +1. **Generate** Ailly prompts from DocGen IAM policy snippets +2. **Execute** Ailly CLI to get LLM-enhanced metadata +3. **Update** your DocGen examples with the enhanced metadata -All of this is done with one command. +The workflow is designed around three main commands that can be run independently or in sequence. --- ## ✅ Prerequisites - Python 3.8+ -- Node.js and npm (for `npx`) -- A DocGen project directory +- Node.js and npm (for `npx @ailly/cli`) +- A DocGen project directory with IAM policy examples --- ## 🚀 Usage -From your project root, run: +### Step 1: Create Prompts + +Generate Ailly prompts from your DocGen IAM policy examples: ```bash -python -m aws_doc_sdk_examples_tools.agent.bin.main \ +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app create-prompts \ /path/to/your/docgen/project \ --system-prompts path/to/system_prompt.txt ``` -### 🔧 Arguments +### Step 2: Run Ailly -- `iam_tributary_root`: Path to the root directory of your IAM policy tributary -- `--system-prompts`: List of system prompt files or strings to include in the Ailly configuration -- `--skip-generation`: Skip the prompt generation and Ailly execution steps (useful for reprocessing existing outputs) +Execute Ailly to generate enhanced metadata: -Run `python -m aws_doc_sdk_examples_tools.agent.bin.main update --help` for more info. +```bash +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app run-ailly +``` ---- +Optionally process specific batches: +```bash +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app run-ailly --batches "batch_01,batch_02" +``` -## 🗂 What This Does +### Step 3: Update Repository -Under the hood, this script: +Apply the enhanced metadata back to your DocGen examples: -1. Creates a directory `.ailly_iam_policy` containing: - - One Markdown file per snippet. - - A `.aillyrc` configuration file. +```bash +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app update-reservoir \ + /path/to/your/docgen/project +``` -2. Runs `npx @ailly/cli` to generate `.ailly.md` outputs. +Optionally update specific batches or packages: +```bash +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app update-reservoir \ + /path/to/your/docgen/project \ + --batches "batch_01,batch_02" \ + --packages "package1,package2" +``` + +--- -3. Parses the Ailly `.ailly.md` files into a single `iam_updates.json` file. +## 🗂 Workflow Details -4. Updates each matching `Example` in the DocGen instance with: - - `title` - - `title_abbrev` - - `synopsis` +1. **Prompt Generation**: + - Scans DocGen examples for IAM policies with default metadata prefixes + - Creates batched Markdown files in `.ailly_iam_policy/` + - Generates `.aillyrc` configuration with system prompts + +2. **Ailly Execution**: + - Runs `npx @ailly/cli` on generated prompts + - Processes batches to manage large datasets + - Generates `.ailly.md` response files + +3. **Metadata Update**: + - Parses Ailly responses for enhanced metadata + - Updates DocGen examples with `title`, `title_abbrev`, `synopsis`, and `description` + - Applies service-specific suffixes to metadata --- -## 💡 Example +## 💡 Example Workflow ```bash -python -m aws_doc_sdk_examples_tools.agent.bin.main \ +# Generate prompts from IAM policy examples +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app create-prompts \ ~/projects/AWSIAMPolicyExampleReservoir \ - --system-prompts prompts/system_prompt.txt + --system-prompts prompts/iam_system_prompt.txt + +# Run Ailly on all generated prompts +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app run-ailly + +# Update the repository with enhanced metadata +python -m aws_doc_sdk_examples_tools.lliam.entry_points.lliam_app update-reservoir \ + ~/projects/AWSIAMPolicyExampleReservoir ``` -This will: -- Write prompts and config to `.ailly_iam_policy/` -- Run Ailly and capture results -- Parse and save output as `.ailly_iam_policy/iam_updates.json` -- Apply updates to your DocGen examples +This workflow will: +- Extract IAM policy snippets and create batched prompts in `.ailly_iam_policy/` +- Execute Ailly to generate enhanced metadata +- Parse responses and update your DocGen examples with rich metadata diff --git a/aws_doc_sdk_examples_tools/lliam/entry_points/lliam_app.py b/aws_doc_sdk_examples_tools/lliam/entry_points/lliam_app.py new file mode 100644 index 0000000..4dccb76 --- /dev/null +++ b/aws_doc_sdk_examples_tools/lliam/entry_points/lliam_app.py @@ -0,0 +1,108 @@ +from pathlib import Path +from typing import List, Optional +from typing_extensions import Annotated +from datetime import datetime +import logging +import typer + +from aws_doc_sdk_examples_tools.lliam.config import ( + AILLY_DIR, + BATCH_PREFIX +) +from aws_doc_sdk_examples_tools.lliam.domain import commands +from aws_doc_sdk_examples_tools.lliam.service_layer import messagebus, unit_of_work + +logging.basicConfig( + level=logging.INFO, + filename=f"lliam-run-{datetime.now().strftime('%Y%m%d_%H%M%S')}.log", + filemode="w", +) + +logger = logging.getLogger(__name__) +app = typer.Typer(name="Lliam") + + +@app.command() +def create_prompts(iam_tributary_root: str, system_prompts: List[str] = []): + doc_gen_root = iam_tributary_root + cmd = commands.CreatePrompts( + doc_gen_root=doc_gen_root, + system_prompts=system_prompts, + out_dir=AILLY_DIR, + ) + uow = unit_of_work.FsUnitOfWork() + messagebus.handle(cmd, uow) + + +@app.command() +def run_ailly( + batches: Annotated[ + Optional[str], + typer.Option( + help="Batch names to process (comma-separated list)" + ), + ] = None, +) -> None: + """ + Run ailly to generate IAM policy content and process the results. + If batches is specified, only those batches will be processed. + If batches is omitted, all batches will be processed. + """ + requested_batches = parse_batch_names(batches) + cmd = commands.RunAilly(batches=requested_batches) + messagebus.handle(cmd) + + +@app.command() +def update_reservoir( + iam_tributary_root: str, + batches: Annotated[ + Optional[str], + typer.Option( + help="Batch names to process (comma-separated list)" + ), + ] = None, + packages: Annotated[ + Optional[str], typer.Option(help="Comma delimited list of packages to update") + ] = None, +) -> None: + """ + Update the doc_gen reservoir with processed IAM policy updates. + If batches is specified, only those batch directories will be processed. + If batches is omitted, all available update files will be processed. + """ + doc_gen_root = Path(iam_tributary_root) + batch_names = parse_batch_names(batches) + package_names = parse_package_names(packages) + cmd = commands.UpdateReservoir( + root=doc_gen_root, batches=batch_names, packages=package_names + ) + messagebus.handle(cmd) + + +def parse_batch_names(batch_names_str: Optional[str]) -> List[str]: + """ + Parse batch names from a comma-separated string. + """ + if not batch_names_str: + return [] + + batch_names = [] + + for name in batch_names_str.split(","): + maybe_batch_name = name.strip() + assert maybe_batch_name.startswith(BATCH_PREFIX) + batch_names.append(maybe_batch_name) + + return batch_names + + +def parse_package_names(package_names_str: Optional[str]) -> List[str]: + if not package_names_str: + return [] + + return [n.strip() for n in package_names_str.split(",")] + + +if __name__ == "__main__": + app() diff --git a/aws_doc_sdk_examples_tools/lliam/service_layer/messagebus.py b/aws_doc_sdk_examples_tools/lliam/service_layer/messagebus.py new file mode 100644 index 0000000..4e0b7eb --- /dev/null +++ b/aws_doc_sdk_examples_tools/lliam/service_layer/messagebus.py @@ -0,0 +1,39 @@ +from typing import Callable, Dict, Optional, Type + +from aws_doc_sdk_examples_tools.lliam.domain import commands +from aws_doc_sdk_examples_tools.lliam.service_layer import ( + create_prompts, + update_doc_gen, + run_ailly, + unit_of_work, +) + +# Only handling Commands for now. +Message = commands.Command + + +def handle( + message: commands.Command, uow: Optional[unit_of_work.FsUnitOfWork] = None +): + queue = [message] + + while queue: + message = queue.pop(0) + if isinstance(message, commands.Command): + handle_command(message, uow) + else: + raise Exception(f"{message} was not a Command") + + +def handle_command( + command: commands.Command, uow: Optional[unit_of_work.FsUnitOfWork] +): + handler = COMMAND_HANDLERS[type(command)] + handler(command, uow) + + +COMMAND_HANDLERS: Dict[Type[commands.Command], Callable] = { + commands.CreatePrompts: create_prompts.create_prompts, + commands.RunAilly: run_ailly.handle_run_ailly, + commands.UpdateReservoir: update_doc_gen.handle_update_reservoir, +}