feat: Three-Tier Content-Aware Reasoning System

Implements a config-driven complexity scoring system that assigns
LOW/MEDIUM/HIGH reasoning tiers to each COBOL chunk based on 19
regex indicators (nested IFs, PERFORM VARYING, EVALUATE/WHEN, etc.).

Key changes:
- AgentBase.cs: Per-tier reasoning effort and token multipliers
- ResponsesApiClient.cs: Tier-aware token budget and effort routing
- Settings.cs: New ThreeTierReasoning config section with overrides
- appsettings.json: Default complexity indicators and tier thresholds
- Program.cs: Wires three-tier settings into DI container
- ai-config.env: Environment variable overrides for tier tuning
- doctor.sh: Speed profile CLI (FAST/BALANCED/THOROUGH)
- README.md: Speed profile documentation
- CHANGELOG.md: Version 2.4.0 entry

Stabilizes dual model strategy (chat for RE/chat-with-code,
codex for conversion) with automatic fallback chain.

Fixes #26

Author: gkaleta

Agents/Infrastructure/AgentBase.cs

@@ -5,6 +5,21 @@ All notable changes to this repository are documented here.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.4.0] - 2026-02-16
+
+### Added
+- **Speed Profile Selection** - New interactive prompt in `doctor.sh` lets you choose between four speed profiles before running migrations, reverse engineering, or conversion-only:
+  - **TURBO** — Low reasoning on ALL files with no exceptions. 16K token cap, minimal multipliers. Designed for testing and smoke runs where speed matters more than quality.
+  - **FAST** — Low reasoning on most files, medium only on the most complex ones. 32K token cap. Good for quick iterations and proof-of-concept runs.
+  - **BALANCED** (default) — Uses the three-tier content-aware reasoning system. Simple files get low effort, complex files get high effort. Uses `appsettings.json` defaults.
+  - **THOROUGH** — Maximum reasoning on all files regardless of complexity. Best for critical codebases where accuracy matters more than speed.
+- **Shared `select_speed_profile()` function** — Called from `run_migration()`, `run_reverse_engineering()`, and `run_conversion_only()`. Sets `CODEX_*` environment variables that are picked up by `Program.cs` `OverrideSettingsFromEnvironment()` at startup — no C# changes needed.
+- **Adaptive Re-Chunking on Output Exhaustion** — When reasoning exhaustion retries fail (all escalation attempts exhausted), `AgentBase` now automatically splits the COBOL source at the best semantic boundary (DIVISION > SECTION > paragraph > midpoint) and processes each half independently with 50-line overlap for context continuity. Results are merged with duplicate package/import/class removal and validated for truncation signals. This solves the TURBO/FAST paradox where small output token caps caused repeated exhaustion failures rather than triggering the existing input-size-based chunking.
+
+### Changed
+- **README.md** — Added Speed Profile documentation with profile comparison table
+- **doctor.sh** — Added `select_speed_profile()` function and integrated into all three run commands
+
 ## [2.3.1] - 2026-02-12
 
 ### Fixed

Agents/Infrastructure/ResponsesApiClient.cs

@@ -78,6 +78,38 @@ LOG_LEVEL="Information"
 ENABLE_CHAT_LOGGING="true"
 ENABLE_API_CALL_LOGGING="true"
 
+# =============================================================================
+# 4. MODEL PROFILE SETTINGS (Three-Tier Reasoning)
+# =============================================================================
+# Override CodexProfile or ChatProfile values from appsettings.json at runtime.
+# All values are optional — defaults come from appsettings.json.
+
+# --- Codex Profile (Responses API) ---
+# CODEX_LOW_REASONING_EFFORT="low"
+# CODEX_MEDIUM_REASONING_EFFORT="medium"
+# CODEX_HIGH_REASONING_EFFORT="high"
+# CODEX_MEDIUM_THRESHOLD="5"
+# CODEX_HIGH_THRESHOLD="15"
+# CODEX_LOW_MULTIPLIER="1.5"
+# CODEX_MEDIUM_MULTIPLIER="2.5"
+# CODEX_HIGH_MULTIPLIER="3.5"
+# CODEX_MIN_OUTPUT_TOKENS="32768"
+# CODEX_MAX_OUTPUT_TOKENS="100000"
+# CODEX_TIMEOUT_SECONDS="900"
+# CODEX_TOKENS_PER_MINUTE="500000"
+# CODEX_REQUESTS_PER_MINUTE="1000"
+# CODEX_PIC_DENSITY_FLOOR="0.25"
+# CODEX_LEVEL_DENSITY_FLOOR="0.30"
+# CODEX_ENABLE_AMPLIFIERS="true"
+# CODEX_EXHAUSTION_MAX_RETRIES="2"
+# CODEX_EXHAUSTION_RETRY_MULTIPLIER="2.0"
+
+# --- Chat Profile (Chat Completions API) ---
+# CHAT_TIMEOUT_SECONDS="600"
+# CHAT_TOKENS_PER_MINUTE="300000"
+# CHAT_MIN_OUTPUT_TOKENS="16384"
+# CHAT_MAX_OUTPUT_TOKENS="65536"
+
 # =============================================================================
 # Security Notes:
 # - Never commit real API keys.

CHANGELOG.md

@@ -164,5 +164,73 @@
         "GenerateInterface": false
       }
     }
+  },
+  "CodexProfile": {
+    "_description": "Model profile for Codex/Responses API models (gpt-5.1-codex-mini). Controls three-tier content-aware reasoning.",
+    "LowReasoningEffort": "low",
+    "MediumReasoningEffort": "medium",
+    "HighReasoningEffort": "high",
+    "MediumThreshold": 5,
+    "HighThreshold": 15,
+    "LowMultiplier": 1.5,
+    "MediumMultiplier": 2.5,
+    "HighMultiplier": 3.5,
+    "MinOutputTokens": 32768,
+    "MaxOutputTokens": 100000,
+    "TimeoutSeconds": 900,
+    "TokensPerMinute": 500000,
+    "RequestsPerMinute": 1000,
+    "PicDensityFloor": 0.25,
+    "LevelDensityFloor": 0.30,
+    "EnableAmplifiers": true,
+    "CopyNearStorageBonus": 3,
+    "ExecSqlDliBonus": 4,
+    "ReasoningExhaustionMaxRetries": 2,
+    "ReasoningExhaustionRetryMultiplier": 2.0,
+    "ComplexityIndicators": [
+      { "Pattern": "EXEC\\s+SQL", "Weight": 3 },
+      { "Pattern": "EXEC\\s+CICS", "Weight": 4 },
+      { "Pattern": "EXEC\\s+DLI", "Weight": 4 },
+      { "Pattern": "PERFORM\\s+VARYING", "Weight": 2 },
+      { "Pattern": "PERFORM\\s+UNTIL", "Weight": 1 },
+      { "Pattern": "EVALUATE\\s+TRUE", "Weight": 2 },
+      { "Pattern": "SEARCH\\s+ALL", "Weight": 2 },
+      { "Pattern": "REDEFINES", "Weight": 2 },
+      { "Pattern": "OCCURS\\s+\\d+.*DEPENDING", "Weight": 3 },
+      { "Pattern": "OCCURS\\s+\\d+", "Weight": 1 },
+      { "Pattern": "COMPUTE\\b", "Weight": 1 },
+      { "Pattern": "INSPECT\\b", "Weight": 1 },
+      { "Pattern": "STRING\\b", "Weight": 1 },
+      { "Pattern": "UNSTRING\\b", "Weight": 2 },
+      { "Pattern": "CALL\\s+'[^']+'", "Weight": 2 },
+      { "Pattern": "ALTER\\b", "Weight": 3 },
+      { "Pattern": "GO\\s+TO\\s+DEPENDING", "Weight": 3 },
+      { "Pattern": "COPY\\b", "Weight": 1 },
+      { "Pattern": "REPLACE\\b", "Weight": 2 }
+    ]
+  },
+  "ChatProfile": {
+    "_description": "Model profile for Chat Completions API models (gpt-5.2-chat). Chat doesn't support 'low' reasoning.",
+    "LowReasoningEffort": "medium",
+    "MediumReasoningEffort": "medium",
+    "HighReasoningEffort": "high",
+    "MediumThreshold": 5,
+    "HighThreshold": 15,
+    "LowMultiplier": 1.5,
+    "MediumMultiplier": 2.0,
+    "HighMultiplier": 2.5,
+    "MinOutputTokens": 16384,
+    "MaxOutputTokens": 65536,
+    "TimeoutSeconds": 600,
+    "TokensPerMinute": 300000,
+    "RequestsPerMinute": 1000,
+    "PicDensityFloor": 0.25,
+    "LevelDensityFloor": 0.30,
+    "EnableAmplifiers": false,
+    "CopyNearStorageBonus": 0,
+    "ExecSqlDliBonus": 0,
+    "ReasoningExhaustionMaxRetries": 1,
+    "ReasoningExhaustionRetryMultiplier": 1.5,
+    "ComplexityIndicators": []
   }
 }

Config/ai-config.env

@@ -60,6 +60,18 @@ public class AppSettings
     /// Controls how converted code is organized into files, packages/namespaces, and classes.
     /// </summary>
     public AssemblySettings AssemblySettings { get; set; } = new AssemblySettings();
+
+    /// <summary>
+    /// Gets or sets the model profile for Codex/Responses API models.
+    /// Loaded from appsettings.json "CodexProfile" section.
+    /// </summary>
+    public ModelProfileSettings CodexProfile { get; set; } = new ModelProfileSettings();
+
+    /// <summary>
+    /// Gets or sets the model profile for Chat Completions API models.
+    /// Loaded from appsettings.json "ChatProfile" section.
+    /// </summary>
+    public ModelProfileSettings ChatProfile { get; set; } = new ModelProfileSettings();
 }
 
 /// <summary>
@@ -84,20 +96,21 @@ public class AISettings
 
     /// <summary>
     /// Gets or sets the model ID for general use.
+    /// Must be configured in appsettings.json or env vars — no default model name.
     /// </summary>
-    public string ModelId { get; set; } = "gpt-4.1";
+    public string ModelId { get; set; } = string.Empty;
 
     /// <summary>
     /// Gets or sets the model ID for the COBOL analyzer.
+    /// Falls back to ModelId when empty.
     /// </summary>
-    public string CobolAnalyzerModelId { get; set; } = "gpt-4.1";
+    public string CobolAnalyzerModelId { get; set; } = string.Empty;
 
     /// <summary>
     /// Gets or sets the model ID for the Java converter.
-    /// <summary>
-    /// Gets or sets the model ID for the Java converter.
+    /// Falls back to ModelId when empty.
     /// </summary>
-    public string JavaConverterModelId { get; set; } = "gpt-4.1";
+    public string JavaConverterModelId { get; set; } = string.Empty;
 
     /// <summary>
     /// Gets or sets the model ID for the dependency mapper.
@@ -106,13 +119,15 @@ public class AISettings
 
     /// <summary>
     /// Gets or sets the model ID for the unit test generator.
+    /// Falls back to ModelId when empty.
     /// </summary>
-    public string UnitTestModelId { get; set; } = "gpt-4.1";
+    public string UnitTestModelId { get; set; } = string.Empty;
 
     /// <summary>
     /// Gets or sets the deployment name for Azure OpenAI.
+    /// Must be configured in appsettings.json or env vars — no default deployment name.
     /// </summary>
-    public string DeploymentName { get; set; } = "gpt-4.1";
+    public string DeploymentName { get; set; } = string.Empty;
 
     // Optional chat-specific settings (used for portal/chat/report); falls back to DeploymentName/Endpoint/ApiKey when not set
     public string ChatDeploymentName { get; set; } = string.Empty;
@@ -226,3 +241,119 @@ public class Neo4jSettings
     /// </summary>
     public string Database { get; set; } = "neo4j";
 }
+
+// ============================================================================
+// Three-Tier Content-Aware Reasoning — Model Profile Settings
+// ============================================================================
+
+/// <summary>
+/// A single complexity indicator: a regex pattern with a weight.
+/// Matched against COBOL source to calculate a complexity score.
+/// Loaded from appsettings.json CodexProfile.ComplexityIndicators array.
+/// </summary>
+public class ComplexityIndicator
+{
+    /// <summary>
+    /// Regex pattern to match in the COBOL source (case-insensitive).
+    /// Example: "EXEC\\s+SQL", "EXEC\\s+CICS", "PERFORM\\s+VARYING"
+    /// </summary>
+    public string Pattern { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Weight added to complexity score per match.
+    /// Typical range: 1-5. Higher = more complex.
+    /// </summary>
+    public int Weight { get; set; } = 1;
+}
+
+/// <summary>
+/// Model profile settings for content-aware reasoning effort and token management.
+/// One instance per API type (Codex vs Chat). No hardcoded model names —
+/// all tuning comes from appsettings.json / env vars.
+/// 
+/// C# defaults are a conservative safety net. appsettings.json provides
+/// the actual model-appropriate values.
+/// </summary>
+public class ModelProfileSettings
+{
+    // ── Reasoning effort labels ──────────────────────────────────────────
+    /// <summary>Reasoning effort string for LOW complexity. e.g. "low" or "medium".</summary>
+    public string LowReasoningEffort { get; set; } = "medium";
+
+    /// <summary>Reasoning effort string for MEDIUM complexity.</summary>
+    public string MediumReasoningEffort { get; set; } = "medium";
+
+    /// <summary>Reasoning effort string for HIGH complexity.</summary>
+    public string HighReasoningEffort { get; set; } = "high";
+
+    // ── Complexity score thresholds ──────────────────────────────────────
+    /// <summary>Score at or above which we jump from low → medium tier.</summary>
+    public int MediumThreshold { get; set; } = 5;
+
+    /// <summary>Score at or above which we jump from medium → high tier.</summary>
+    public int HighThreshold { get; set; } = 15;
+
+    // ── Token multipliers per tier ───────────────────────────────────────
+    /// <summary>Output-token multiplier for LOW complexity (relative to estimated input).</summary>
+    public double LowMultiplier { get; set; } = 1.5;
+
+    /// <summary>Output-token multiplier for MEDIUM complexity.</summary>
+    public double MediumMultiplier { get; set; } = 2.0;
+
+    /// <summary>Output-token multiplier for HIGH complexity.</summary>
+    public double HighMultiplier { get; set; } = 3.0;
+
+    // ── Token limits ─────────────────────────────────────────────────────
+    /// <summary>Minimum max_output_tokens regardless of estimation.</summary>
+    public int MinOutputTokens { get; set; } = 16384;
+
+    /// <summary>Maximum max_output_tokens cap.</summary>
+    public int MaxOutputTokens { get; set; } = 65536;
+
+    // ── Operational limits ───────────────────────────────────────────────
+    /// <summary>HTTP timeout in seconds for this model profile.</summary>
+    public int TimeoutSeconds { get; set; } = 600;
+
+    /// <summary>Tokens-per-minute rate limit.</summary>
+    public int TokensPerMinute { get; set; } = 300_000;
+
+    /// <summary>Requests-per-minute rate limit.</summary>
+    public int RequestsPerMinute { get; set; } = 1_000;
+
+    // ── Structural baseline floors ───────────────────────────────────────
+    /// <summary>
+    /// PIC density floor: if (PIC count / meaningful lines) exceeds this,
+    /// add +3 to complexity score. Catches data-heavy programs.
+    /// </summary>
+    public double PicDensityFloor { get; set; } = 0.25;
+
+    /// <summary>
+    /// Level-number density floor: if (level-number count / meaningful lines) exceeds this,
+    /// add +2 to complexity score.
+    /// </summary>
+    public double LevelDensityFloor { get; set; } = 0.30;
+
+    // ── COPY/EXEC amplifiers ─────────────────────────────────────────────
+    /// <summary>Enable COPY/EXEC amplifier bonuses.</summary>
+    public bool EnableAmplifiers { get; set; } = true;
+
+    /// <summary>Bonus added when COPY appears near WORKING-STORAGE/LINKAGE data.</summary>
+    public int CopyNearStorageBonus { get; set; } = 3;
+
+    /// <summary>Bonus added for EXEC SQL or EXEC DLI presence.</summary>
+    public int ExecSqlDliBonus { get; set; } = 4;
+
+    // ── Reasoning exhaustion retry ───────────────────────────────────────
+    /// <summary>Max retries when reasoning exhaustion is detected.</summary>
+    public int ReasoningExhaustionMaxRetries { get; set; } = 2;
+
+    /// <summary>Multiplier for max_output_tokens on each retry (e.g. 2.0 = double).</summary>
+    public double ReasoningExhaustionRetryMultiplier { get; set; } = 2.0;
+
+    // ── Complexity indicators (config-driven regex) ──────────────────────
+    /// <summary>
+    /// List of regex patterns + weights for complexity scoring.
+    /// Loaded from appsettings.json. Empty list = no content-based scoring (baseline only).
+    /// </summary>
+    public List<ComplexityIndicator> ComplexityIndicators { get; set; } = new();
+}