diff --git a/src/schemas/PIPELINE_SCHEMAS.md b/src/schemas/PIPELINE_SCHEMAS.md new file mode 100644 index 0000000..d2b8b86 --- /dev/null +++ b/src/schemas/PIPELINE_SCHEMAS.md @@ -0,0 +1,681 @@ +# ACE Pipeline Standardized Schemas + +This document defines the standardized input and output formats for each stage of the ACE pipeline. These schemas ensure consistency across different implementations and enable interoperability between pipeline stages. + +## Pipeline Stages + +The ACE pipeline consists of multiple stages, where each stage consumes the output from the previous stage: + +0. **Experiment Setup** - Initialize experiment and create domain metadata +1. **Area Generation** - Generate domain areas +2. **Capability Generation** - Generate capabilities for each area +3. **Task Generation** - Generate tasks for each capability +4. **Solution Generation** - Generate solutions for each task +5. **Validation** - Validate solutions against tasks + +**Note:** Experiment configuration must remain consistent throughout the pipeline. Once set during experiment setup, it should not be changed to avoid inconsistencies. + +## Implementation Approach + +**Pipeline Pattern:** +Each stage follows a consistent pattern: +1. **Consumes Previous Stage Output**: Each stage (except Stage 0) loads data from the previous stage's output files using provided load functions +2. **Stage Implementation**: Produces dataclass objects (or lists of dataclasses) + metadata +3. **Save Function**: Takes dataclass objects + metadata → saves to JSON file using provided save functions + +**Important:** All stage implementations must follow this pattern to ensure the pipeline is clean, consistent, and maintainable. This enables interoperability between different implementations, resumability of failed runs, and clear traceability through the pipeline. + +**Note:** The dataclasses, save functions (`save_(data, metadata, output_path)`), and load functions (`load_(file_path) -> `) for each stage will be provided and must be used. Do not implement custom serialization or data structures - use the standardized schemas to ensure consistency across the pipeline. Dataclasses provide type safety, validation, and clear structure. JSON is the serialization format. + +**Iteration Note:** Some stages operate on subsets (one area, capability, or task at a time) and require an outer orchestrator/loop script to iterate over all items: +- **Stage 2 (Capability Generation)**: Operates on one area at a time - orchestrator loops over all areas from Stage 1 +- **Stage 3 (Task Generation)**: Operates on one capability at a time - orchestrator loops over all capabilities from Stage 2 +- **Stage 4 (Solution Generation)**: Operates on one task at a time - orchestrator loops over all tasks from Stage 3 +- **Stage 5 (Validation)**: Operates on one task at a time - orchestrator loops over all solutions from Stage 4 + +The stage implementation itself handles a single item, and the orchestrator manages the iteration across all items. + +--- + +## Naming Conventions + +All identifiers and tags in the pipeline follow standardized formats: + +### Tags +- **Format**: `_YYYYMMDD_HHMMSS` (e.g., `_20251009_122040`) +- **Usage**: Used for versioning outputs in Stages 1-5 +- **Generation**: Automatically generated when a new run is created (timestamp-based) + +### Domain IDs +- **Format**: `domain_` + zero-padded 3-digit number (e.g., `domain_000`) +- **Assignment**: Sequential starting from `domain_000` +- **Scope**: Unique within an experiment (typically only one domain per experiment) + +### Area IDs +- **Format**: `area_` + zero-padded 3-digit number (e.g., `area_000`, `area_001`) +- **Assignment**: Sequential starting from `area_000` when areas are generated +- **Scope**: Unique within an experiment + +### Capability IDs +- **Format**: `cap_` + zero-padded 3-digit number (e.g., `cap_000`, `cap_001`) +- **Assignment**: Sequential starting from `cap_000` within each area when capabilities are generated +- **Scope**: Unique within an area (but can repeat across areas, e.g., `area_000/cap_000/` and `area_001/cap_000/`) + +### Task IDs +- **Format**: `task_` + zero-padded 3-digit number (e.g., `task_000`, `task_001`) +- **Assignment**: Sequential starting from `task_000` within each capability when tasks are generated +- **Scope**: Unique within a capability + + +--- + +## Directory Structure + +All outputs are stored in the following flat directory structure, with each stage having its own top-level directory and tags for different generation runs: + +``` +/ + / + experiment.json # Experiment metadata (all configuration) + domain/ + domain.json # Domain metadata (contains domain_id) + areas/ + / # Tag from area generation (e.g., _20251009_122040) + areas.json # All areas for this area generation run (output from Stage 1) + capabilities/ + / # Tag from capability generation (e.g., _20251009_131252) + / # One directory per area (e.g., area_000, area_001) + capabilities.json # All capabilities for this area in this generation run + tasks/ + / # Tag from task generation (e.g., _20251014_114358) + / # One directory per area (e.g., area_000, area_001) + / # One directory per capability (e.g., cap_000, cap_001) + tasks.json # All tasks for this capability in this generation run + solutions/ + / # Tag from solution generation (e.g., _20251016_182128) + / # One directory per area (e.g., area_000, area_001) + / # One directory per capability (e.g., cap_000, cap_001) + / # One directory per task (e.g., task_000, task_001) + solution.json # Solution for this task + validation/ + / # Tag from validation run (e.g., _20251017_091500) + / # One directory per area (e.g., area_000, area_001) + / # One directory per capability (e.g., cap_000, cap_001) + / # One directory per task (e.g., task_000, task_001) + validation.json # Validation result for this task +``` + +**Example:** +``` +agentic_outputs/ + r0_10x10/ + experiment.json # Experiment configuration + domain/ + domain.json # Domain metadata + areas/ + _20251009_122040/ # Tag from first area generation + areas.json # All areas from this generation + _20251010_143022/ # Tag from second area generation (different set of areas) + areas.json # All areas from this generation + capabilities/ + _20251009_131252/ # Tag from first capability generation + area_000/ + capabilities.json # Capabilities for area_000 + area_001/ + capabilities.json # Capabilities for area_001 + _20251011_091500/ # Tag from second capability generation + area_000/ + capabilities.json # Capabilities for area_000 + tasks/ + _20251014_114358/ # Tag from first task generation + area_000/ + cap_000/ + tasks.json # Tasks for area_000/cap_000 + cap_001/ + tasks.json # Tasks for area_000/cap_001 + area_001/ + cap_000/ + tasks.json # Tasks for area_001/cap_000 + _20251015_120000/ # Tag from second task generation + area_000/ + cap_000/ + tasks.json # Tasks for area_000/cap_000 + solutions/ + _20251016_182128/ # Tag from solution generation + area_000/ + cap_000/ + task_000/ + solution.json + task_001/ + solution.json + area_001/ + cap_000/ + task_000/ + solution.json + validation/ + _20251017_091500/ # Tag from validation run + area_000/ + cap_000/ + task_000/ + validation.json + task_001/ + validation.json + area_001/ + cap_000/ + task_000/ + validation.json +``` + +**File Naming:** +- Experiment: `experiment.json` (no versioning, one file per experiment, contains all configuration) +- Domain: `domain.json` (no versioning, one file per experiment) +- Areas: `areas.json` (versioned by tag: `areas//areas.json`) +- Capabilities: `capabilities.json` (versioned by tag: `capabilities///capabilities.json`) +- Tasks: `tasks.json` (versioned by tag: `tasks////tasks.json`) +- Solutions: `solution.json` (versioned by tag: `solutions/////solution.json`) +- Validation: `validation.json` (versioned by tag: `validation/////validation.json`) + +**Resumability Benefits:** +- Each area has its own directory - easy to see which areas are processed +- Each capability has its own directory - easy to see which capabilities are complete +- Missing files are immediately visible in the directory structure +- Can resume from any area or capability by checking if directory/files exist +- Tags allow multiple runs/versions to coexist +- Can check latest tag to determine most recent run + +**Versioning Strategy:** +- Each stage generates a new tag when run (see Tags in Naming Conventions section) +- Tags are independent per stage (areas can have different tag than capabilities) +- **Input tags**: Each stage requires tag(s) from previous stage(s) to load input data + - Stage 1 (Areas): No input tag (uses domain.json) + - Stage 2 (Capabilities): Requires `areas_tag` from Stage 1 + - Stage 3 (Tasks): Requires `capabilities_tag` from Stage 2 + - Stage 4 (Solutions): Requires `tasks_tag` from Stage 3 + - Stage 5 (Validation): Requires `solutions_tag` from Stage 4 (task information is included in solution files) +- **Resume tags**: Optional - If provided, stage loads existing output and continues incomplete generation + - Checks for existing files with resume tag + - Identifies which items are incomplete (e.g., missing capabilities, tasks, solutions) + - Continues generation only for incomplete items + - Preserves existing completed items +- **New tags**: If no resume tag provided, generates new tag and creates fresh output + +--- + +## Dataclasses + +All dataclasses used across pipeline stages are defined below. Stage implementations must use these standardized dataclasses. + +**Note:** All ID and tag formats (domain_id, area_id, capability_id, task_id, tags) are defined in the [Naming Conventions](#naming-conventions) section. Individual field descriptions below do not repeat these format definitions. + +### PipelineMetadata + +**File:** [`metadata_schemas.py`](metadata_schemas.py) + +All pipeline outputs include a `metadata` object (represented by the `PipelineMetadata` dataclass) that provides pipeline execution context and traceability. + +**Required Fields:** +- `experiment_id`: String (required, experiment identifier) +- `output_base_dir`: String (required, base output directory for all pipeline outputs) +- `timestamp`: String (required, ISO 8601 format, e.g., "2025-11-06T12:00:00Z") +- `input_stage_tag`: String (optional, tag of the input data used from previous stage) - Present when stage uses input from previous stage, null for Stage 0 +- `output_stage_tag`: String (optional, tag for this output) - Present for versioned stages (Stages 1-5), null for Stage 0 (not versioned) +- `resume`: Boolean (required, indicates if this run was resumed from a previous checkpoint) + +**Optional Fields:** +- Additional optional fields may be added as needed for pipeline-specific metadata + +**Note:** +- Stage-specific identifiers (domain_id, area_id, capability_id, task_id) are stored in the actual data objects (Domain, Area, Capability, Task), NOT in PipelineMetadata +- PipelineMetadata focuses on pipeline execution context, not the content being processed + +### Experiment + +**File:** [`experiment_schemas.py`](experiment_schemas.py) + +**Fields:** +- `experiment_id`: String (required, experiment identifier) +- `domain`: String (required, human-readable domain name) +- `domain_id`: String (required) +- `pipeline_type`: String (optional, e.g., "agentic", "diverse_task") - identifies the pipeline variant +- `configuration`: Dict[str, Any] (required, complete configuration used for this experiment - structure varies by pipeline type) + +### Domain + +**File:** [`domain_schemas.py`](domain_schemas.py) + +**Fields:** +- `name`: String (required, human-readable domain name) +- `domain_id`: String (required) +- `description`: String (optional, domain description) + +### Area + +**File:** [`area_schemas.py`](area_schemas.py) + +**Fields:** +- `name`: String (required, human-readable area name) +- `area_id`: String (required) +- `domain`: Domain (required, Domain dataclass object) +- `description`: String (required, area description) +- `generation_metadata`: Dict (optional, nested dictionary containing process-specific information) + - This field can contain any generation-specific data (e.g., generation method, parameters, intermediate steps) + - Structure is flexible and depends on the generation method + +**Note:** When serialized to JSON, the `domain` object is flattened to `domain` (string) and `domain_id` (string) fields. + +### Capability + +**File:** [`capability_schemas.py`](capability_schemas.py) + +**Fields:** +- `name`: String (required, capability name) +- `capability_id`: String (required) +- `area`: Area (required, Area dataclass object) +- `description`: String (required, capability description) +- `generation_metadata`: Dict (optional, nested dictionary containing process-specific information) + - This field can contain any generation-specific data (e.g., generation method, parameters, intermediate steps) + - Structure is flexible and depends on the generation method + +**Note:** When serialized to JSON, the `area` object is flattened to `area` (string), `area_id` (string), `domain` (string), and `domain_id` (string) fields. + +### Task + +**File:** [`task_schemas.py`](task_schemas.py) + +**Fields:** +- `task_id`: String (required, unique within capability) +- `task`: String (required, the task/problem text) +- `capability`: Capability (required, Capability dataclass object) + +**Note:** When serialized to JSON, the `capability` object is flattened to `capability` (string), `capability_id` (string), `area` (string), `area_id` (string), `domain` (string), and `domain_id` (string) fields. + +### TaskSolution + +**File:** [`solution_schemas.py`](solution_schemas.py) + +**Fields:** +- `task_id`: String (required) +- `task`: String (required, the task/problem text from Stage 3) +- `solution`: String (required, the final solution) +- `reasoning`: String (required, explanation of the solution) +- `task_obj`: Task (required, Task dataclass object with full hierarchy) +- `numerical_answer`: String (optional, JSON string with numerical results) +- `generation_metadata`: Dict (optional, nested dictionary containing process-specific information) + - This field can contain any generation-specific data (e.g., debate rounds, agent interactions, pipeline type) + - Structure is flexible and depends on the generation method (agentic, single-agent, etc.) + +**Note:** When serialized to JSON, the `task_obj` object is flattened to `capability` (string), `capability_id` (string), `area` (string), `area_id` (string), `domain` (string), and `domain_id` (string) fields. + +### ValidationResult + +**File:** [`validation_schemas.py`](validation_schemas.py) + +**Fields:** +- `task_id`: String (required) +- `task`: String (required, the task/problem text from Stage 3) +- `verification`: Boolean (required, overall validation status - whether the solution is verified/valid) +- `feedback`: String (required, detailed feedback on the validation) +- `task_obj`: Task (required, Task dataclass object with full hierarchy) +- `score`: Float (optional, validation score, typically 0.0 to 1.0) +- `generation_metadata`: Dict (optional, nested dictionary containing process-specific information) + - This field can contain any validation-specific data (e.g., validation method, criteria details, error details) + - Structure is flexible and depends on the validation method + +**Note:** When serialized to JSON, the `task_obj` object is flattened to `capability` (string), `capability_id` (string), `area` (string), `area_id` (string), `domain` (string), and `domain_id` (string) fields. + +--- + +## Stage 0: Experiment Setup + +### Input +All inputs come from a configuration YAML file (e.g., `src/cfg/agentic_config.yaml`). Important fields include: +- **Experiment ID**: String - The experiment identifier (e.g., "r0_10x10") +- **Domain Name**: String - The domain name (e.g., "personal finance", "mathematics") +- **Description**: String (optional) - Domain description +- **Output Base Directory**: String - Base output directory for all pipeline outputs (e.g., `global_cfg.output_dir` in agentic pipeline) + +**Note:** The `experiment_id` and `output_base_dir` from the config YAML file are consistent across all stages. All stage-specific configurations (e.g., `num_areas`, `num_capabilities_per_area`, `num_tasks_per_capability`) also come from this same config YAML file. + +### Tag Handling +- **No input tag required** (first stage) +- **No resume tag** - Not applicable (single domain JSON, always creates new files) + +### Outputs + +This stage creates two files: +1. `experiment.json` - Experiment metadata +2. `domain.json` - Domain metadata + +#### Output 1: `experiment.json` + +**Stage Output:** Experiment dataclass + PipelineMetadata +**Save Function:** `save_experiment(experiment: Experiment, metadata: PipelineMetadata, output_path: Path)` (see [`io_utils.py`](io_utils.py)) + +**File Path:** `//experiment.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T12:00:00Z", + "input_stage_tag": null, + "output_stage_tag": null, + "resume": false + }, + "experiment": { + "experiment_id": "r0_10x10", + "domain": "personal finance", + "domain_id": "domain_000", + "pipeline_type": "agentic", + "configuration": { + ... + } + } +} +``` + +**Schema:** See `Experiment` and `PipelineMetadata` dataclasses in the Dataclasses section above. + +#### Output 2: `domain.json` + +**Stage Output:** Domain dataclass object + PipelineMetadata +**Save Function:** `save_domain(domain: Domain, metadata: PipelineMetadata, output_path: Path)` (see [`io_utils.py`](io_utils.py)) + +**File Path:** `//domain/domain.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T12:00:00Z", + "input_stage_tag": null, + "output_stage_tag": null, + "resume": false + }, + "domain": { + "name": "personal finance", + "domain_id": "domain_000", + "description": "Personal finance domain covering budgeting, investing, retirement planning, etc." + } +} +``` + +--- + +## Stage 1: Area Generation + +### Input +- **Domain**: Domain object (from Stage 0) - Loaded from `domain/domain.json` +- **Configuration**: Dict - Stage-specific configuration from config YAML file (e.g., `num_areas`) + +### Tag Handling +- **Input tag**: Not applicable (uses `domain/domain.json` from Stage 0, which has no tag) +- **Resume tag**: Not applicable (single `areas.json` file with all areas, always creates new files) +- **New tag**: Generates new tag and creates `areas//areas.json` + +### Output: `areas.json` + +**Stage Output:** List[Area] dataclasses + PipelineMetadata +**Save Function:** `save_areas(areas: List[Area], metadata: PipelineMetadata, output_path: Path)` (see [`io_utils.py`](io_utils.py)) + +**File Path:** `//areas//areas.json` +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T12:00:00Z", + "input_stage_tag": null, + "output_stage_tag": "_20251009_122040", + "resume": false + }, + "areas": [ + { + "name": "Cash Flow & Budget Management", + "area_id": "area_000", + "description": "Design and monitor budgets using various methodologies...", + "domain": "personal finance", + "domain_id": "domain_000" + } + ] +} +``` + +--- + +## Stage 2: Capability Generation + +### Input +- **Areas tag**: String - Tag from Stage 1 output (e.g., `_20251009_122040`) + - Loads areas from `areas//areas.json` +- **Configuration**: Dict - Stage-specific configuration from config YAML file (e.g., `num_capabilities_per_area`) + +### Tag Handling +- **Resume tag**: Optional - If provided, goes to `capabilities//` directory + - For each area_id, checks if `capabilities///capabilities.json` exists + - If file exists, capabilities for that area were already generated successfully, so skip it + - If file doesn't exist, creates `/` subdirectory and generates capabilities for that area +- **New tag**: If no resume tag provided, generates new tag (cap_tag) for this capability generation run + - For each area, creates `capabilities///capabilities.json` + +### Output: `capabilities.json` (one per area) + +**Stage Output:** List[Capability] dataclasses + PipelineMetadata +**Save Function:** `save_capabilities(capabilities: List[Capability], metadata: PipelineMetadata, output_path: Path)` (see [`io_utils.py`](io_utils.py)) + +**File Path:** `//capabilities///capabilities.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T12:30:00Z", + "input_stage_tag": "_20251009_122040", + "output_stage_tag": "_20251009_131252", + "resume": false + }, + "capabilities": [ + { + "name": "budget_policy_and_structure", + "capability_id": "cap_000", + "description": "Define the strategic framework and methodology for budgeting...", + "area": "Cash Flow & Budget Management", + "area_id": "area_000", + "domain": "personal finance", + "domain_id": "domain_000" + } + ] +} +``` + +--- + +## Stage 3: Task Generation + +### Input +- **Capabilities tag**: String - Tag from Stage 2 output (e.g., `_20251009_131252`) + - Loads capabilities from `capabilities///capabilities.json` for each area +- **Configuration**: Dict - Stage-specific configuration from config YAML file (e.g., `num_final_problems_per_capability`) + +### Tag Handling +- **Resume tag**: Optional - If provided, goes to `tasks//` directory + - For each `` and ``, checks if `tasks////tasks.json` exists + - If file exists, tasks for that capability were already generated successfully, so skip it + - If file doesn't exist, creates `//` subdirectories and generates tasks for that capability +- **New tag**: If no resume tag provided, generates new tag (task_tag) for this task generation run + - For each capability, creates `tasks////tasks.json` + +### Output: `tasks.json` (one per capability) + +**Stage Output:** List[Task] dataclasses + PipelineMetadata +**Save Function:** `save_tasks(tasks: List[Task], metadata: PipelineMetadata, output_path: Path)` (see [`io_utils.py`](io_utils.py)) + +**File Path:** `//tasks////tasks.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T13:00:00Z", + "input_stage_tag": "_20251009_131252", + "output_stage_tag": "_20251014_114358", + "resume": false + }, + "tasks": [ + { + "task_id": "task_000", + "task": "You are advising a client who wants to set up a zero-based budget...", + "capability_id": "cap_000", + "capability": "budget_policy_and_structure", + "area": "Cash Flow & Budget Management", + "area_id": "area_000", + "domain": "personal finance", + "domain_id": "domain_000" + }, + { + "task_id": "task_001", + "task": "A family of four needs to restructure their budget...", + "capability_id": "cap_000", + "capability": "budget_policy_and_structure", + "area": "Cash Flow & Budget Management", + "area_id": "area_000", + "domain": "personal finance", + "domain_id": "domain_000" + } + ] +} +``` + +--- + +## Stage 4: Solution Generation + +### Input +- **Tasks tag**: String - Tag from Stage 3 output (e.g., `_20251014_114358`) + - For each area and capability, loads tasks from `tasks////tasks.json` +- **Configuration**: Dict - Stage-specific configuration from config YAML file (e.g., `max_rounds`) + +### Tag Handling +- **Resume tag**: Optional - If provided, goes to `solutions//` directory + - For each area_id, capability_id, and task_id combination, checks if `solutions/////solution.json` exists + - If file exists, solution for that task was already generated successfully, so skip it + - If file doesn't exist, creates `///` subdirectories and generates solution for that task +- **New tag**: If no resume tag provided, generates new tag (solution_tag) for this solution generation run + - For each task, creates `solutions/////solution.json` + +### Output: `solution.json` (one per task) + +**Stage Output:** TaskSolution dataclass + PipelineMetadata +**Save Function:** `save_solution(task_solution: TaskSolution, metadata: PipelineMetadata, output_path: Path)` + +**File Path:** `//solutions/////solution.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T13:30:00Z", + "input_stage_tag": "_20251014_114358", + "output_stage_tag": "_20251016_182128", + "resume": false + }, + "task_id": "task_000", + "task": "You are advising a client who wants to set up a zero-based budget...", + "capability": "budget_policy_and_structure", + "capability_id": "cap_000", + "area": "Cash Flow & Budget Management", + "area_id": "area_000", + "domain": "personal finance", + "domain_id": "domain_000", + "solution": "The optimal approach is to use a zero-based budgeting methodology...", + "reasoning": "Both agents agreed on the zero-based approach because...", + "numerical_answer": "{\"budget_allocation\": {...}}", + "generation_metadata": { + "pipeline_type": "agentic", + "consensus_reached": true, + "total_rounds": 2, + "agents": [ + { + "agent_id": "A", + "thought": "I need to analyze the client's financial situation...", + "final_answer": "{\"recommendation\": {...}}", + "round_number": 0 + }, + { + "agent_id": "B", + "thought": "The client's income and expenses suggest...", + "final_answer": "{\"recommendation\": {...}}", + "round_number": 0 + } + ] + } +} +``` + +--- + +## Stage 5: Validation + +### Input +- **Solutions tag**: String - Tag from Stage 4 output (e.g., `_20251016_182128`) + - For each area, capability, and task, loads solutions from `solutions/////solution.json` + - Task information is included in the solution files, so no separate tasks tag is needed +- **Configuration**: Dict - Stage-specific configuration from config YAML file (e.g., validation criteria) + +### Tag Handling +- **Input tag**: Required - `solutions_tag` from Stage 4 + - For each area, capability, and task, loads solutions from `solutions/////solution.json` + - Task information is included in the solution files +- **Resume tag**: Optional - If provided, goes to `validation//` directory + - For each `///solution.json` in `solutions/`, checks if `validation/////validation.json` exists + - If file exists, validation for that task was already completed successfully, so skip it + - If file doesn't exist, creates `///` subdirectories and generates validation for that task +- **New tag**: If no resume tag provided, generates new tag (validation_tag) for this validation run + - For each task, creates `validation/////validation.json` + +### Output: `validation.json` (one per task) + +**Stage Output:** ValidationResult dataclass + PipelineMetadata +**Save Function:** `save_validation(validation_result: ValidationResult, metadata: PipelineMetadata, output_path: Path)` + +**File Path:** `//validation/////validation.json` + +```json +{ + "metadata": { + "experiment_id": "r0_10x10", + "output_base_dir": "agentic_outputs", + "timestamp": "2025-11-06T14:00:00Z", + "input_stage_tag": "_20251016_182128", + "output_stage_tag": "_20251017_091500", + "resume": false + }, + "task_id": "task_000", + "task": "You are advising a client who wants to set up a zero-based budget...", + "capability": "budget_policy_and_structure", + "capability_id": "cap_000", + "area": "Cash Flow & Budget Management", + "area_id": "area_000", + "domain": "personal finance", + "domain_id": "domain_000", + "verification": true, + "feedback": "Solution addresses all aspects of the task...", + "score": 0.95, + "generation_metadata": { + "validation_method": "llm_based", + "criteria": { + "solution_completeness": true, + "solution_accuracy": true, + "reasoning_quality": true + } + } +} +``` + + +--- diff --git a/src/schemas/README.md b/src/schemas/README.md new file mode 100644 index 0000000..a2bc443 --- /dev/null +++ b/src/schemas/README.md @@ -0,0 +1,87 @@ +# ACE Pipeline Schemas + +This directory contains standardized schemas for all ACE pipeline stages, ensuring consistent data formats across different implementations. + +## Structure + +- **[`PIPELINE_SCHEMAS.md`](PIPELINE_SCHEMAS.md)** - Complete documentation of input/output formats for each stage +- **Python Dataclasses** - Type-safe data structures for each stage: + - [`experiment_schemas.py`](experiment_schemas.py) - Experiment (Stage 0) + - [`domain_schemas.py`](domain_schemas.py) - Domain (Stage 0) + - [`metadata_schemas.py`](metadata_schemas.py) - Common metadata (PipelineMetadata) + - [`area_schemas.py`](area_schemas.py) - Area generation (Stage 1) + - [`capability_schemas.py`](capability_schemas.py) - Capability generation (Stage 2) + - [`task_schemas.py`](task_schemas.py) - Task generation (Stage 3) + - [`solution_schemas.py`](solution_schemas.py) - Solution generation (Stage 4) + - [`validation_schemas.py`](validation_schemas.py) - Validation (Stage 5) +- **I/O Utilities** - Save and load functions: + - [`io_utils.py`](io_utils.py) - Functions to save/load all stage outputs (save/load functions for all 7 stage outputs) + +## Usage + +### Using Python Dataclasses + +```python +from src.schemas import ( + Domain, + Experiment, + PipelineMetadata, + Area, + Capability, + Task, + TaskSolution, + ValidationResult, +) + +# Create area +domain = Domain(name="Personal Finance", domain_id="domain_000") +area = Area( + name="Cash Flow & Budget Management", + area_id="area_000", + description="Design and monitor budgets...", + domain=domain, + # generation_metadata is optional +) + +# Convert to dict for JSON serialization +data = area.to_dict() + +# Load from dict +area = Area.from_dict(data) +``` + +### Using Save/Load Functions + +```python +from pathlib import Path +from src.schemas import ( + save_areas, + load_areas, + PipelineMetadata, + Area, +) + +# Save areas +areas = [Area(...), Area(...)] +metadata = PipelineMetadata( + experiment_id="r0_10x10", + output_base_dir="agentic_outputs", + timestamp="2025-11-06T12:00:00Z", + output_stage_tag="_20251009_122040" +) +save_areas(areas, metadata, Path("output/areas.json")) + +# Load areas +areas, metadata = load_areas(Path("output/areas.json")) +``` + +## Pipeline Stages + +0. **Experiment Setup** → `Experiment`, `Domain` +1. **Area Generation** → `Area` +2. **Capability Generation** → `Capability` +3. **Task Generation** → `Task` +4. **Solution Generation** → `TaskSolution` +5. **Validation** → `ValidationResult` + +See [`PIPELINE_SCHEMAS.md`](PIPELINE_SCHEMAS.md) for detailed specifications. diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py new file mode 100644 index 0000000..29e46fc --- /dev/null +++ b/src/schemas/__init__.py @@ -0,0 +1,65 @@ +"""Standardized schemas for ACE pipeline stages. + +This module provides standardized data structures for all pipeline stages, +ensuring consistent input/output formats regardless of internal implementation. +""" + +from src.schemas.area_schemas import Area +from src.schemas.capability_schemas import Capability +from src.schemas.domain_schemas import Domain +from src.schemas.experiment_schemas import Experiment +from src.schemas.io_utils import ( + load_areas, + load_capabilities, + load_domain, + load_experiment, + load_solution, + load_tasks, + load_validation, + save_areas, + save_capabilities, + save_domain, + save_experiment, + save_solution, + save_tasks, + save_validation, +) +from src.schemas.metadata_schemas import PipelineMetadata +from src.schemas.solution_schemas import TaskSolution +from src.schemas.task_schemas import Task +from src.schemas.validation_schemas import ValidationResult + + +__all__ = [ + # Metadata + "PipelineMetadata", + # Experiment schemas (Stage 0) + "Experiment", + "Domain", + # Area schemas + "Area", + # Capability schemas + "Capability", + # Task schemas + "Task", + # Solution schemas + "TaskSolution", + # Validation schemas + "ValidationResult", + # I/O functions - Save + "save_experiment", + "save_domain", + "save_areas", + "save_capabilities", + "save_tasks", + "save_solution", + "save_validation", + # I/O functions - Load + "load_experiment", + "load_domain", + "load_areas", + "load_capabilities", + "load_tasks", + "load_solution", + "load_validation", +] diff --git a/src/schemas/area_schemas.py b/src/schemas/area_schemas.py new file mode 100644 index 0000000..311eb8a --- /dev/null +++ b/src/schemas/area_schemas.py @@ -0,0 +1,50 @@ +"""Schemas for area generation stage (Stage 1). + +Defines Area dataclass for domain area. Areas are high-level categories +within a domain. +""" + +from dataclasses import dataclass, field +from typing import Dict, Optional + +from src.schemas.domain_schemas import Domain + + +@dataclass +class Area: + """Dataclass for domain area.""" + + name: str + area_id: str + domain: Domain + description: str + generation_metadata: Optional[Dict] = field(default_factory=dict) + + def to_dict(self): + """Convert to dictionary.""" + result = { + "name": self.name, + "area_id": self.area_id, + "domain": self.domain.name, + "domain_id": self.domain.domain_id, + "description": self.description, + } + if self.generation_metadata: + result["generation_metadata"] = self.generation_metadata + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + domain = Domain( + name=data["domain"], + domain_id=data["domain_id"], + description=data.get("domain_description"), + ) + return cls( + name=data["name"], + area_id=data["area_id"], + domain=domain, + description=data["description"], + generation_metadata=data.get("generation_metadata", {}), + ) diff --git a/src/schemas/capability_schemas.py b/src/schemas/capability_schemas.py new file mode 100644 index 0000000..8cfc74c --- /dev/null +++ b/src/schemas/capability_schemas.py @@ -0,0 +1,60 @@ +"""Schemas for capability generation stage (Stage 2). + +Defines Capability dataclass for capability within an area. Capabilities +are specific skills or abilities. +""" + +from dataclasses import dataclass, field +from typing import Dict, Optional + +from src.schemas.area_schemas import Area +from src.schemas.domain_schemas import Domain + + +@dataclass +class Capability: + """Dataclass for capability.""" + + name: str + capability_id: str + area: Area + description: str + generation_metadata: Optional[Dict] = field(default_factory=dict) + + def to_dict(self): + """Convert to dictionary.""" + result = { + "name": self.name, + "capability_id": self.capability_id, + "area": self.area.name, + "area_id": self.area.area_id, + "area_description": self.area.description, + "domain": self.area.domain.name, + "domain_id": self.area.domain.domain_id, + "description": self.description, + } + if self.generation_metadata: + result["generation_metadata"] = self.generation_metadata + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + domain = Domain( + name=data["domain"], + domain_id=data["domain_id"], + description=data.get("domain_description"), + ) + area = Area( + name=data["area"], + area_id=data["area_id"], + domain=domain, + description=data["area_description"], + ) + return cls( + name=data["name"], + capability_id=data["capability_id"], + area=area, + description=data["description"], + generation_metadata=data.get("generation_metadata", {}), + ) diff --git a/src/schemas/domain_schemas.py b/src/schemas/domain_schemas.py new file mode 100644 index 0000000..999990a --- /dev/null +++ b/src/schemas/domain_schemas.py @@ -0,0 +1,35 @@ +"""Schemas for domain (Stage 0). + +Defines Domain dataclass for domain. +""" + +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Domain: + """Dataclass for domain.""" + + name: str + domain_id: str + description: Optional[str] = None + + def to_dict(self): + """Convert to dictionary.""" + result = { + "name": self.name, + "domain_id": self.domain_id, + } + if self.description is not None: + result["description"] = self.description + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + return cls( + name=data["name"], + domain_id=data["domain_id"], + description=data.get("description"), + ) diff --git a/src/schemas/experiment_schemas.py b/src/schemas/experiment_schemas.py new file mode 100644 index 0000000..de92538 --- /dev/null +++ b/src/schemas/experiment_schemas.py @@ -0,0 +1,41 @@ +"""Schemas for experiment setup stage (Stage 0). + +Defines Experiment dataclass containing experiment configuration and metadata. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + + +@dataclass +class Experiment: + """Dataclass for experiment metadata and configuration.""" + + experiment_id: str + domain: str + domain_id: str + pipeline_type: Optional[str] = None + configuration: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self): + """Convert to dictionary.""" + result = { + "experiment_id": self.experiment_id, + "domain": self.domain, + "domain_id": self.domain_id, + "configuration": self.configuration, + } + if self.pipeline_type is not None: + result["pipeline_type"] = self.pipeline_type + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + return cls( + experiment_id=data["experiment_id"], + domain=data["domain"], + domain_id=data["domain_id"], + pipeline_type=data.get("pipeline_type"), + configuration=data.get("configuration", {}), + ) diff --git a/src/schemas/io_utils.py b/src/schemas/io_utils.py new file mode 100644 index 0000000..082b50b --- /dev/null +++ b/src/schemas/io_utils.py @@ -0,0 +1,277 @@ +"""I/O utilities for saving and loading pipeline stage outputs.""" + +import json +from pathlib import Path +from typing import List, Tuple + +from src.schemas.area_schemas import Area +from src.schemas.capability_schemas import Capability +from src.schemas.domain_schemas import Domain +from src.schemas.experiment_schemas import Experiment +from src.schemas.metadata_schemas import PipelineMetadata +from src.schemas.solution_schemas import TaskSolution +from src.schemas.task_schemas import Task +from src.schemas.validation_schemas import ValidationResult + + +def save_experiment( + experiment: Experiment, metadata: PipelineMetadata, output_path: Path +) -> None: + """Save experiment output to JSON file. + + Args: + experiment: Experiment dataclass + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + "experiment": experiment.to_dict(), + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_domain(domain: Domain, metadata: PipelineMetadata, output_path: Path) -> None: + """Save domain output to JSON file. + + Args: + domain: Domain dataclass + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + "domain": domain.to_dict(), + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_areas( + areas: List[Area], metadata: PipelineMetadata, output_path: Path +) -> None: + """Save areas output to JSON file. + + Args: + areas: List of Area dataclasses + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + "areas": [area.to_dict() for area in areas], + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_capabilities( + capabilities: List[Capability], metadata: PipelineMetadata, output_path: Path +) -> None: + """Save capabilities output to JSON file. + + Args: + capabilities: List of Capability dataclasses + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + "capabilities": [cap.to_dict() for cap in capabilities], + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_tasks( + tasks: List[Task], metadata: PipelineMetadata, output_path: Path +) -> None: + """Save tasks output to JSON file. + + Args: + tasks: List of Task dataclasses + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + "tasks": [task.to_dict() for task in tasks], + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_solution( + task_solution: TaskSolution, metadata: PipelineMetadata, output_path: Path +) -> None: + """Save solution output to JSON file. + + Args: + task_solution: TaskSolution dataclass + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + **task_solution.to_dict(), + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +def save_validation( + validation_result: ValidationResult, metadata: PipelineMetadata, output_path: Path +) -> None: + """Save validation output to JSON file. + + Args: + validation_result: ValidationResult dataclass + metadata: PipelineMetadata dataclass + output_path: Path to save the JSON file + """ + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "metadata": metadata.to_dict(), + **validation_result.to_dict(), + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + +# Load functions + + +def load_experiment(file_path: Path) -> Tuple[Experiment, PipelineMetadata]: + """Load experiment output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (Experiment, PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + experiment = Experiment.from_dict(data["experiment"]) + return experiment, metadata + + +def load_domain(file_path: Path) -> Tuple[Domain, PipelineMetadata]: + """Load domain output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (Domain, PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + domain = Domain.from_dict(data["domain"]) + return domain, metadata + + +def load_areas(file_path: Path) -> Tuple[List[Area], PipelineMetadata]: + """Load areas output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (List[Area], PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + areas = [Area.from_dict(area_data) for area_data in data["areas"]] + return areas, metadata + + +def load_capabilities( + file_path: Path, +) -> Tuple[List[Capability], PipelineMetadata]: + """Load capabilities output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (List[Capability], PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + capabilities = [Capability.from_dict(cap_data) for cap_data in data["capabilities"]] + return capabilities, metadata + + +def load_tasks(file_path: Path) -> Tuple[List[Task], PipelineMetadata]: + """Load tasks output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (List[Task], PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + tasks = [Task.from_dict(task_data) for task_data in data["tasks"]] + return tasks, metadata + + +def load_solution(file_path: Path) -> Tuple[TaskSolution, PipelineMetadata]: + """Load solution output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (TaskSolution, PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + # Solution files have flattened structure + # (metadata + all task_solution fields) + solution_data = {k: v for k, v in data.items() if k != "metadata"} + task_solution = TaskSolution.from_dict(solution_data) + return task_solution, metadata + + +def load_validation( + file_path: Path, +) -> Tuple[ValidationResult, PipelineMetadata]: + """Load validation output from JSON file. + + Args: + file_path: Path to the JSON file + + Returns + ------- + Tuple of (ValidationResult, PipelineMetadata) + """ + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + metadata = PipelineMetadata.from_dict(data["metadata"]) + # Validation files have flattened structure + # (metadata + all validation_result fields) + validation_data = {k: v for k, v in data.items() if k != "metadata"} + validation_result = ValidationResult.from_dict(validation_data) + return validation_result, metadata diff --git a/src/schemas/metadata_schemas.py b/src/schemas/metadata_schemas.py new file mode 100644 index 0000000..343e6b7 --- /dev/null +++ b/src/schemas/metadata_schemas.py @@ -0,0 +1,77 @@ +"""Metadata schemas for pipeline stages. + +This module defines PipelineMetadata, which provides execution context and traceability +for all pipeline stage outputs. It tracks experiment ID, timestamps, input/output +version tags, and resume state. Used by all save/load functions and serialized in +JSON output files. + +Note: PipelineMetadata tracks execution context, not content (content identifiers are +in the data objects themselves). +""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + + +@dataclass +class PipelineMetadata: + """Standard metadata for all pipeline stage outputs. + + Provides execution context, traceability, and resumability for pipeline stages. + Included with every stage output to track which experiment produced it, when it was + generated, which input version was used, and whether the run was resumed. + + Attributes + ---------- + experiment_id: Unique identifier for the experiment. + output_base_dir: Base directory path where all pipeline outputs are stored. + timestamp: ISO 8601 formatted timestamp (e.g., "2025-11-06T12:00:00Z"). + Auto-generated if not provided. + input_stage_tag: Optional tag for the input version from previous stage + (e.g., "_20251009_122040"). None for Stage 0. + output_stage_tag: Optional tag for this output version + (e.g., "_20251009_131252"). None for Stage 0. + resume: Boolean indicating if this run was resumed from a checkpoint. + """ + + experiment_id: str + output_base_dir: str + timestamp: str + input_stage_tag: Optional[str] = None + output_stage_tag: Optional[str] = None + resume: bool = False + + def __post_init__(self): + """Set default timestamp if not provided. + + Automatically generates a UTC timestamp in ISO 8601 format if not set. + """ + if not self.timestamp: + self.timestamp = datetime.utcnow().isoformat() + "Z" + + def to_dict(self): + """Convert metadata to dictionary for JSON serialization.""" + result = { + "experiment_id": self.experiment_id, + "output_base_dir": self.output_base_dir, + "timestamp": self.timestamp, + "resume": self.resume, + } + if self.input_stage_tag is not None: + result["input_stage_tag"] = self.input_stage_tag + if self.output_stage_tag is not None: + result["output_stage_tag"] = self.output_stage_tag + return result + + @classmethod + def from_dict(cls, data: dict): + """Create PipelineMetadata from dictionary (e.g., loaded from JSON).""" + return cls( + experiment_id=data["experiment_id"], + output_base_dir=data["output_base_dir"], + timestamp=data["timestamp"], + input_stage_tag=data.get("input_stage_tag"), + output_stage_tag=data.get("output_stage_tag"), + resume=data.get("resume", False), + ) diff --git a/src/schemas/solution_schemas.py b/src/schemas/solution_schemas.py new file mode 100644 index 0000000..e4547a7 --- /dev/null +++ b/src/schemas/solution_schemas.py @@ -0,0 +1,83 @@ +"""Schemas for solution generation stage (Stage 4). + +Defines TaskSolution dataclass for task solution, including solution text, +reasoning, and optional numerical answer. +""" + +from dataclasses import dataclass, field +from typing import Dict, Optional + +from src.schemas.area_schemas import Area +from src.schemas.capability_schemas import Capability +from src.schemas.domain_schemas import Domain +from src.schemas.task_schemas import Task + + +@dataclass +class TaskSolution: + """Dataclass for task solution.""" + + task_id: str + task: str + solution: str + reasoning: str + task_obj: Task + numerical_answer: Optional[str] = None + generation_metadata: Optional[Dict] = field(default_factory=dict) + + def to_dict(self): + """Convert to dictionary.""" + result = { + "task_id": self.task_id, + "task": self.task, + "solution": self.solution, + "reasoning": self.reasoning, + "capability_id": self.task_obj.capability.capability_id, + "capability": self.task_obj.capability.name, + "capability_description": self.task_obj.capability.description, + "area": self.task_obj.capability.area.name, + "area_id": self.task_obj.capability.area.area_id, + "area_description": self.task_obj.capability.area.description, + "domain": self.task_obj.capability.area.domain.name, + "domain_id": self.task_obj.capability.area.domain.domain_id, + } + if self.numerical_answer is not None: + result["numerical_answer"] = self.numerical_answer + if self.generation_metadata: + result["generation_metadata"] = self.generation_metadata + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + domain = Domain( + name=data["domain"], + domain_id=data["domain_id"], + description=data.get("domain_description"), + ) + area = Area( + name=data["area"], + area_id=data["area_id"], + domain=domain, + description=data["area_description"], + ) + capability = Capability( + name=data["capability"], + capability_id=data["capability_id"], + area=area, + description=data["capability_description"], + ) + task_obj = Task( + task_id=data["task_id"], + task=data["task"], + capability=capability, + ) + return cls( + task_id=data["task_id"], + task=data["task"], + solution=data["solution"], + reasoning=data["reasoning"], + task_obj=task_obj, + numerical_answer=data.get("numerical_answer"), + generation_metadata=data.get("generation_metadata", {}), + ) diff --git a/src/schemas/task_schemas.py b/src/schemas/task_schemas.py new file mode 100644 index 0000000..cbf8865 --- /dev/null +++ b/src/schemas/task_schemas.py @@ -0,0 +1,61 @@ +"""Schemas for task generation stage (Stage 3). + +Defines Task dataclass for task. Tasks are concrete evaluation items +that test a capability. +""" + +from dataclasses import dataclass + +from src.schemas.area_schemas import Area +from src.schemas.capability_schemas import Capability +from src.schemas.domain_schemas import Domain + + +@dataclass +class Task: + """Dataclass for task.""" + + task_id: str + task: str + capability: Capability + + def to_dict(self): + """Convert to dictionary.""" + return { + "task_id": self.task_id, + "task": self.task, + "capability_id": self.capability.capability_id, + "capability": self.capability.name, + "capability_description": self.capability.description, + "area": self.capability.area.name, + "area_id": self.capability.area.area_id, + "area_description": self.capability.area.description, + "domain": self.capability.area.domain.name, + "domain_id": self.capability.area.domain.domain_id, + } + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + domain = Domain( + name=data["domain"], + domain_id=data["domain_id"], + description=data.get("domain_description"), + ) + area = Area( + name=data["area"], + area_id=data["area_id"], + domain=domain, + description=data["area_description"], + ) + capability = Capability( + name=data["capability"], + capability_id=data["capability_id"], + area=area, + description=data["capability_description"], + ) + return cls( + task_id=data["task_id"], + task=data["task"], + capability=capability, + ) diff --git a/src/schemas/validation_schemas.py b/src/schemas/validation_schemas.py new file mode 100644 index 0000000..02ec8ee --- /dev/null +++ b/src/schemas/validation_schemas.py @@ -0,0 +1,83 @@ +"""Schemas for validation stage (Stage 5). + +Defines ValidationResult dataclass for validation result, including +verification status, feedback, and optional score. +""" + +from dataclasses import dataclass, field +from typing import Dict, Optional + +from src.schemas.area_schemas import Area +from src.schemas.capability_schemas import Capability +from src.schemas.domain_schemas import Domain +from src.schemas.task_schemas import Task + + +@dataclass +class ValidationResult: + """Dataclass for validation result.""" + + task_id: str + task: str + verification: bool + feedback: str + task_obj: Task + score: Optional[float] = None + generation_metadata: Optional[Dict] = field(default_factory=dict) + + def to_dict(self): + """Convert to dictionary.""" + result = { + "task_id": self.task_id, + "task": self.task, + "verification": self.verification, + "feedback": self.feedback, + "capability_id": self.task_obj.capability.capability_id, + "capability": self.task_obj.capability.name, + "capability_description": self.task_obj.capability.description, + "area": self.task_obj.capability.area.name, + "area_id": self.task_obj.capability.area.area_id, + "area_description": self.task_obj.capability.area.description, + "domain": self.task_obj.capability.area.domain.name, + "domain_id": self.task_obj.capability.area.domain.domain_id, + } + if self.score is not None: + result["score"] = self.score + if self.generation_metadata: + result["generation_metadata"] = self.generation_metadata + return result + + @classmethod + def from_dict(cls, data: dict): + """Create from dictionary.""" + domain = Domain( + name=data["domain"], + domain_id=data["domain_id"], + description=data.get("domain_description"), + ) + area = Area( + name=data["area"], + area_id=data["area_id"], + domain=domain, + description=data["area_description"], + ) + capability = Capability( + name=data["capability"], + capability_id=data["capability_id"], + area=area, + description=data["capability_description"], + ) + task_obj = Task( + task_id=data["task_id"], + task=data["task"], + capability=capability, + ) + return cls( + task_id=data["task_id"], + task=data["task"], + verification=data["verification"], + feedback=data["feedback"], + task_obj=task_obj, + score=data.get("score"), + generation_metadata=data.get("generation_metadata", {}), + )