Fix JSON parsing error in RPG orchestrator

- Add robust JSON extraction to handle malformed AI responses
- Implement retry logic with exponential backoff for failed API calls
- Improve prompts to ensure JSON-only responses from AI
- Add detailed error analysis for debugging JSON parsing issues
- Update .gitignore to exclude .grok directory containing user data

Fixes the 'Unterminated string in JSON' error that was causing code generation to fail.

Author: NYTEMODEONLY

.gitignore

@@ -9,6 +9,7 @@ yarn-error.log*
 # Environment variables and API keys
 .env
 .grok_code_key
+.grok/
 *.key
 
 # OS generated files

lib/rpg/orchestrator.js

@@ -114,15 +114,21 @@ export class RPGOrchestrator {
    */
   async generateRPGPlan(prompt, context = {}) {
     const existingFiles = context.existingFiles || 'none';
+    const maxRetries = 3;
 
-    const planningPrompt = `
-You are an expert software architect using the RPG (Recursive Planning Graph) methodology for systematic code generation.
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        logger.info(`RPG planning attempt ${attempt}/${maxRetries}`);
+
+        const planningPrompt = `You are an expert software architect using the RPG (Recursive Planning Graph) methodology for systematic code generation.
+
+CRITICAL: Respond with ONLY a valid JSON object. Do NOT include any explanations, comments, or text before or after the JSON. The response must be parseable by JSON.parse().
 
 For the user request: "${prompt}"
 
 ${existingFiles !== 'none' ? `Existing project files: ${JSON.stringify(existingFiles, null, 2)}` : ''}
 
-Create a comprehensive RPG plan following this structure:
+Create a comprehensive RPG plan following this EXACT structure:
 
 {
   "features": ["Feature 1", "Feature 2", "Feature 3"],
@@ -159,51 +165,86 @@ Guidelines:
 - Be specific about technologies and architecture patterns
 - Consider the existing codebase when planning modifications
 
-Respond with valid JSON only.`;
+IMPORTANT: Your response must be ONLY the JSON object, nothing else. Start with { and end with }.${attempt > 1 ? `\n\nPrevious attempts failed due to JSON parsing errors. Please ensure your response is valid JSON.` : ''}`;
 
-    logger.debug('Making RPG planning request', {
-      promptLength: planningPrompt.length,
-      model: this.model,
-    });
+        logger.debug('Making RPG planning request', {
+          promptLength: planningPrompt.length,
+          model: this.model,
+          attempt,
+        });
 
-    const response = await this.client.chat.completions.create({
-      model: this.model,
-      messages: [
-        {
-          role: 'system',
-          content:
-            'You are a precise RPG planner. Respond with valid JSON only. No explanations or markdown.',
-        },
-        { role: 'user', content: planningPrompt },
-      ],
-      max_tokens: 4096,
-      temperature: 0.3,
-    });
+        const response = await this.client.chat.completions.create({
+          model: this.model,
+          messages: [
+            {
+              role: 'system',
+              content:
+                'You are a precise RPG planner. You must respond with ONLY a valid JSON object. Do NOT include any explanations, comments, markdown, or additional text. The response must start with { and end with } and be parseable by JSON.parse(). Failure to follow this instruction will result in parsing errors.',
+            },
+            { role: 'user', content: planningPrompt },
+          ],
+          max_tokens: 4096,
+          temperature: Math.max(0.1, 0.3 - (attempt - 1) * 0.1), // Decrease temperature on retries
+        });
 
-    const rawResponse = response.choices[0].message.content.trim();
-    logger.debug('Received RPG planning response', {
-      responseLength: rawResponse.length,
-    });
+        const rawResponse = response.choices[0].message.content.trim();
+        logger.debug('Received RPG planning response', {
+          responseLength: rawResponse.length,
+          attempt,
+        });
 
-    try {
-      const plan = JSON.parse(rawResponse);
+        // Extract JSON from response - handle cases where AI returns extra text
+        let jsonString = rawResponse;
 
-      // Validate plan structure
-      this.validatePlanStructure(plan);
+        // Try to find JSON object in the response
+        const jsonStart = rawResponse.indexOf('{');
+        const jsonEnd = rawResponse.lastIndexOf('}');
 
-      logger.info('Successfully parsed RPG plan', {
-        features: plan.features?.length || 0,
-        files: Object.keys(plan.files || {}).length,
-        flows: plan.flows?.length || 0,
-        deps: plan.deps?.length || 0,
-      });
+        if (jsonStart !== -1 && jsonEnd !== -1 && jsonEnd > jsonStart) {
+          jsonString = rawResponse.substring(jsonStart, jsonEnd + 1);
+        }
 
-      return { plan, rawResponse };
-    } catch (parseError) {
-      logger.error('Failed to parse RPG planning response', parseError, {
-        rawResponse: rawResponse.substring(0, 500),
-      });
-      throw new Error(`Invalid RPG plan format: ${parseError.message}`);
+        // Clean up common issues
+        jsonString = jsonString
+          .replace(/[\u0000-\u001F\u007F-\u009F]/g, '') // Remove control characters
+          .replace(/,(\s*[}\]])/g, '$1') // Remove trailing commas
+          .trim();
+
+        const plan = JSON.parse(jsonString);
+
+        // Validate plan structure
+        this.validatePlanStructure(plan);
+
+        logger.info('Successfully parsed RPG plan', {
+          features: plan.features?.length || 0,
+          files: Object.keys(plan.files || {}).length,
+          flows: plan.flows?.length || 0,
+          deps: plan.deps?.length || 0,
+          attempt,
+        });
+
+        return { plan, rawResponse };
+
+      } catch (parseError) {
+        logger.error(`RPG planning attempt ${attempt} failed`, parseError, {
+          errorMessage: parseError.message,
+          attempt,
+        });
+
+        // If this is the last attempt, provide detailed error information
+        if (attempt === maxRetries) {
+          const errorDetails = this.analyzeJsonError(
+            parseError.response || 'No response available',
+            parseError
+          );
+          throw new Error(`Invalid RPG plan format after ${maxRetries} attempts: ${parseError.message}. ${errorDetails}`);
+        }
+
+        // Wait before retrying (exponential backoff)
+        const waitTime = Math.pow(2, attempt) * 1000; // 2s, 4s, 8s
+        logger.info(`Retrying RPG planning in ${waitTime}ms...`);
+        await new Promise(resolve => setTimeout(resolve, waitTime));
+      }
     }
   }
 
@@ -480,6 +521,53 @@ Respond with valid JSON only.`;
     }
   }
 
+  /**
+   * Analyze JSON parsing errors to provide helpful debugging information
+   * @param {string} rawResponse - The raw AI response
+   * @param {Error} parseError - The JSON parsing error
+   * @returns {string} Error analysis details
+   */
+  analyzeJsonError(rawResponse, parseError) {
+    const errorMessage = parseError.message;
+    let details = '';
+
+    // Check for common JSON issues
+    if (errorMessage.includes('Unterminated string')) {
+      details = 'The JSON contains an unterminated string literal. Check for missing quotes around string values.';
+    } else if (errorMessage.includes('Unexpected token')) {
+      details = 'The JSON contains unexpected characters. The response may include extra text before or after the JSON.';
+    } else if (errorMessage.includes('Unexpected end of JSON input')) {
+      details = 'The JSON appears to be truncated. The AI response may have been cut off.';
+    } else if (errorMessage.includes('Expected property name')) {
+      details = 'Missing quotes around property names in the JSON object.';
+    }
+
+    // Check if response contains JSON markers
+    const hasJsonStart = rawResponse.includes('{');
+    const hasJsonEnd = rawResponse.includes('}');
+
+    if (!hasJsonStart || !hasJsonEnd) {
+      details += ' No JSON object markers found in response.';
+    } else {
+      const jsonStart = rawResponse.indexOf('{');
+      const jsonEnd = rawResponse.lastIndexOf('}');
+      const jsonLength = jsonEnd - jsonStart + 1;
+      details += ` Found JSON-like content (${jsonLength} chars) in response.`;
+    }
+
+    // Show a snippet around the error position
+    const positionMatch = errorMessage.match(/position (\d+)/);
+    if (positionMatch) {
+      const position = parseInt(positionMatch[1]);
+      const start = Math.max(0, position - 50);
+      const end = Math.min(rawResponse.length, position + 50);
+      const snippet = rawResponse.substring(start, end);
+      details += ` Snippet around error: "...${snippet}..."`;
+    }
+
+    return details;
+  }
+
   // Helper methods
 
   validatePlanStructure(plan) {