DTS-49469: Ai Recommendation parsing issue fix

ai-data-processor/src/main/java/com/publicissapient/kpidashboard/job/recommendationcalculation/parser/BatchRecommendationResponseParser.java

@@ -61,21 +61,24 @@ public class BatchRecommendationResponseParser {
 	 * 
 	 * @param response
 	 *            ChatGenerationResponseDTO from AI Gateway
-	 * @return Optional containing parsed Recommendation, or empty if parsing fails
+	 * @return parsed and validated Recommendation object
 	 * @throws IllegalArgumentException
 	 *             if response is null
+	 * @throws IllegalStateException
+	 *             if response content is empty, JSON extraction fails, or required
+	 *             fields missing
+	 * @throws Exception
+	 *             if JSON parsing fails or other processing errors occur
 	 */
-	public Optional<Recommendation> parseRecommendation(ChatGenerationResponseDTO response) {
+	public Recommendation parseRecommendation(ChatGenerationResponseDTO response) throws Exception {
 		if (response == null) {
 			throw new IllegalArgumentException("AI Gateway response cannot be null");
 		}
 
 		// Validate response content is not null or empty
 		String aiResponse = response.content();
 		if (aiResponse == null || aiResponse.trim().isEmpty()) {
-			log.error("{} AI Gateway returned null or empty response content",
-					JobConstants.LOG_PREFIX_RECOMMENDATION);
-			return Optional.empty();
+			throw new IllegalStateException("AI Gateway returned empty response content");
 		}
 
 		return parseRecommendationContent(aiResponse);
@@ -86,45 +89,83 @@ public Optional<Recommendation> parseRecommendation(ChatGenerationResponseDTO re
 	 * 
 	 * @param aiResponse
 	 *            JSON string from AI Gateway
-	 * @return Optional containing parsed Recommendation, or empty if parsing fails
+	 * @return parsed and validated Recommendation
+	 * @throws IllegalStateException
+	 *             if content is empty, JSON extraction fails, or required fields
+	 *             missing
+	 * @throws Exception
+	 *             if JSON parsing fails
 	 */
-	private Optional<Recommendation> parseRecommendationContent(String aiResponse) {
-		if (StringUtils.isBlank(aiResponse)) {
-			log.error("{} AI response is empty, cannot parse recommendation",
-					JobConstants.LOG_PREFIX_RECOMMENDATION);
-			return Optional.empty();
+	private Recommendation parseRecommendationContent(String aiResponse) throws Exception {
+		String jsonContent = extractJsonContent(aiResponse);
+
+		if (StringUtils.isBlank(jsonContent) || EMPTY_JSON_OBJECT.equals(jsonContent)) {
+			throw new IllegalStateException(
+					"Failed to extract JSON from AI response - malformed markdown or missing JSON content");
 		}
 
-		try {
-			String jsonContent = extractJsonContent(aiResponse);
+		JsonNode rootNode = objectMapper.readTree(jsonContent);
 
-			if (StringUtils.isBlank(jsonContent) || EMPTY_JSON_OBJECT.equals(jsonContent)) {
-				log.error("{} Extracted JSON content is empty or invalid from AI response",
-						JobConstants.LOG_PREFIX_RECOMMENDATION);
-				return Optional.empty();
-			}
-			JsonNode rootNode = objectMapper.readTree(jsonContent);
+		// Check for direct recommendation object with required non-empty fields
+		if (getTextValue(rootNode, TITLE) != null && getTextValue(rootNode, DESCRIPTION) != null) {
+			return parseAndValidateRecommendation(rootNode);
+		}
 
-			// Check for direct recommendation object with required non-empty fields
-			if (hasValidTextField(rootNode, TITLE) && hasValidTextField(rootNode, DESCRIPTION)) {
-				return Optional.of(parseRecommendationNode(rootNode));
-			}
+		// Check for recommendations array
+		JsonNode recommendationsArray = rootNode.get(RECOMMENDATIONS);
+		if (recommendationsArray != null && recommendationsArray.isArray() && !recommendationsArray.isEmpty()) {
+			return parseAndValidateRecommendation(recommendationsArray.get(0));
+		}
 
-			// Check for recommendations array
-			return Optional.ofNullable(rootNode.get(RECOMMENDATIONS)).filter(JsonNode::isArray)
-					.filter(node -> !node.isEmpty()).map(node -> parseRecommendationNode(node.get(0)));
+		// Missing required fields
+		throw new IllegalStateException("AI response missing required fields: title and description must be non-empty");
+	}
 
-		} catch (Exception e) {
-			String preview = StringUtils.abbreviate(aiResponse, 100);
-			log.error("{} Error parsing AI response JSON: {} - Response preview: {}",
-					JobConstants.LOG_PREFIX_RECOMMENDATION, e.getMessage(), preview, e);
-			return Optional.empty();
+	/**
+	 * Parses and validates a recommendation node to ensure data quality.
+	 * 
+	 * @param node
+	 *            the JSON node to parse
+	 * @return validated Recommendation object
+	 * @throws IllegalStateException
+	 *             if required fields are missing or invalid
+	 */
+	private Recommendation parseAndValidateRecommendation(JsonNode node) {
+		Recommendation recommendation = parseRecommendationNode(node);
+
+		// Validate required text fields
+		if (StringUtils.isBlank(recommendation.getTitle())) {
+			throw new IllegalStateException(
+					"AI response missing required field 'title' - recommendation must have non-empty title");
+		}
+
+		if (StringUtils.isBlank(recommendation.getDescription())) {
+			throw new IllegalStateException(
+					"AI response missing required field 'description' - recommendation must have non-empty description");
+		}
+
+		// Validate critical fields
+		if (recommendation.getSeverity() == null) {
+			throw new IllegalStateException(
+					"AI response missing required field 'severity' - cannot determine recommendation priority");
+		}
+
+		if (StringUtils.isBlank(recommendation.getTimeToValue())) {
+			throw new IllegalStateException(
+					"AI response missing required field 'timeToValue' - prompt requires timeline estimate based on severity and complexity");
+		}
+
+		if (recommendation.getActionPlans() == null || recommendation.getActionPlans().isEmpty()) {
+			throw new IllegalStateException(
+					"AI response missing required field 'actionPlans' - recommendation must include actionable steps");
 		}
+
+		return recommendation;
 	}
 
 	/**
 	 * Extracts JSON content from AI response by removing markdown code blocks.
-	 * Handles responses wrapped in ```json``` markdown blocks.
+	 * Handles responses wrapped in ```json``` markdown blocks with fallback logic.
 	 * 
 	 * @param aiResponse
 	 *            the raw AI response string
@@ -135,15 +176,29 @@ private String extractJsonContent(String aiResponse) {
 
 		// Remove markdown code blocks if present
 		if (content.startsWith(MARKDOWN_CODE_FENCE)) {
-			content = StringUtils.substringBetween(content, "\n", MARKDOWN_CODE_FENCE);
-			if (content == null) {
-				return EMPTY_JSON_OBJECT;
+			String extracted = StringUtils.substringBetween(content, "\n", MARKDOWN_CODE_FENCE);
+			if (extracted == null) {
+				log.warn("{} Malformed markdown fence - attempting fallback extraction",
+						JobConstants.LOG_PREFIX_RECOMMENDATION);
+				int firstNewline = content.indexOf('\n');
+				if (firstNewline > 0 && firstNewline < content.length() - 1) {
+					content = content.substring(firstNewline + 1).trim();
+				} else {
+					return EMPTY_JSON_OBJECT;
+				}
+			} else {
+				content = extracted.trim();
 			}
 		}
 
 		// Find and extract JSON object starting from first {
 		int jsonStart = content.indexOf(JSON_START_CHAR);
-		return jsonStart >= 0 ? content.substring(jsonStart) : content;
+		if (jsonStart < 0) {
+			log.warn("{} No JSON object found in response", JobConstants.LOG_PREFIX_RECOMMENDATION);
+			return EMPTY_JSON_OBJECT;
+		}
+
+		return content.substring(jsonStart);
 	}
 
 	/**
@@ -175,37 +230,43 @@ private Optional<Severity> parseSeverity(String severityStr) {
 		try {
 			return Optional.of(Severity.valueOf(severityStr));
 		} catch (IllegalArgumentException e) {
-			log.warn("{} Invalid severity value from AI response: {}. Saving as null.",
+			log.debug("{} Invalid severity value '{}' in AI response - will be validated",
 					JobConstants.LOG_PREFIX_RECOMMENDATION, severityStr);
 			return Optional.empty();
 		}
 	}
 
 	/**
-	 * Parses action plans from JSON array node.
+	 * Parses action plans from JSON array node. Filters out action plans with empty
+	 * or blank title/description to ensure data quality.
+	 * 
+	 * @param actionPlansNode
+	 *            the JSON array node containing action plans
+	 * @return list of valid action plans, or null if none are valid
 	 */
 	private List<ActionPlan> parseActionPlans(JsonNode actionPlansNode) {
 		List<ActionPlan> actionPlans = new ArrayList<>();
-		actionPlansNode.forEach(actionNode -> {
-			ActionPlan action = ActionPlan.builder().title(getTextValue(actionNode, TITLE))
-					.description(getTextValue(actionNode, DESCRIPTION)).build();
-			actionPlans.add(action);
-		});
-		return actionPlans;
-	}
+		int skippedCount = 0;
 
-	/**
-	 * Checks if JSON node has a valid non-empty text field.
-	 * 
-	 * @param node
-	 *            the JSON node to check
-	 * @param fieldName
-	 *            the field name to check
-	 * @return true if field exists and has non-blank text
-	 */
-	private boolean hasValidTextField(JsonNode node, String fieldName) {
-		return Optional.ofNullable(node.get(fieldName)).map(JsonNode::asText).filter(StringUtils::isNotBlank)
-				.isPresent();
+		for (JsonNode actionNode : actionPlansNode) {
+			String title = getTextValue(actionNode, TITLE);
+			String description = getTextValue(actionNode, DESCRIPTION);
+
+			// Only include action plans with both title and description
+			if (StringUtils.isNotBlank(title) && StringUtils.isNotBlank(description)) {
+				ActionPlan action = ActionPlan.builder().title(title).description(description).build();
+				actionPlans.add(action);
+			} else {
+				skippedCount++;
+			}
+		}
+
+		if (skippedCount > 0) {
+			log.debug("{} Filtered out {} action plan(s) with empty content from {} total",
+					JobConstants.LOG_PREFIX_RECOMMENDATION, skippedCount, actionPlansNode.size());
+		}
+
+		return actionPlans.isEmpty() ? null : actionPlans;
 	}
 
 	/**

ai-data-processor/src/main/java/com/publicissapient/kpidashboard/job/recommendationcalculation/processor/ProjectItemProcessor.java

@@ -52,8 +52,9 @@ public class ProjectItemProcessor implements ItemProcessor<ProjectInputDTO, Reco
 	@Override
 	public RecommendationsActionPlan process(@Nonnull ProjectInputDTO projectInputDTO) throws Exception {
 		try {
-			log.debug("{} Starting recommendation calculation for project with nodeId: {}",
-					JobConstants.LOG_PREFIX_RECOMMENDATION, projectInputDTO.nodeId());
+			log.debug("{} Starting recommendation calculation for project: {} (basicProjectConfigId: {})",
+					JobConstants.LOG_PREFIX_RECOMMENDATION, projectInputDTO.name(),
+					projectInputDTO.basicProjectConfigId());
 
 			RecommendationsActionPlan recommendation = recommendationCalculationService
 					.calculateRecommendationsForProject(projectInputDTO);
@@ -63,16 +64,16 @@ public RecommendationsActionPlan process(@Nonnull ProjectInputDTO projectInputDT
 					recommendation.getMetadata().getPersona());
 			return recommendation;
 		} catch (Exception e) {
-			log.error("{} Failed to process project: {} (nodeId: {})",
+			log.error("{} Failed to process project: {} (basicProjectConfigId: {})",
 					JobConstants.LOG_PREFIX_RECOMMENDATION, projectInputDTO.name(),
-					projectInputDTO.nodeId(), e);
+					projectInputDTO.basicProjectConfigId(), e);
 
 			// Save detailed failure trace log with more context
 			String errorMessage = String.format("Processing failed for project %s: %s - %s. Root cause: %s",
 					projectInputDTO.name(), e.getClass().getSimpleName(), e.getMessage(),
 					ExceptionUtils.getRootCauseMessage(e));
 			processorExecutionTraceLogService.upsertTraceLog(JobConstants.JOB_RECOMMENDATION_CALCULATION,
-					projectInputDTO.nodeId(), false, errorMessage);
+					projectInputDTO.basicProjectConfigId(), false, errorMessage);
 
 			// Return null to skip this projectInputDTO
 			return null;

ai-data-processor/src/main/java/com/publicissapient/kpidashboard/job/recommendationcalculation/service/KpiDataExtractionService.java

@@ -64,7 +64,7 @@ public class KpiDataExtractionService {
 	public Map<String, Object> fetchKpiDataForProject(ProjectInputDTO projectInput) {
 		try {
 			log.debug("{} Fetching KPI data for project: {}", JobConstants.LOG_PREFIX_RECOMMENDATION,
-					projectInput.nodeId());
+					projectInput.basicProjectConfigId());
 
 			// Construct KPI requests
 			List<KpiRequest> kpiRequests = constructKpiRequests(projectInput);
@@ -76,9 +76,9 @@ public Map<String, Object> fetchKpiDataForProject(ProjectInputDTO projectInput)
 			if (CollectionUtils.isEmpty(kpiElements)) {
 				log.error(
 						"{} No KPI elements received from KnowHOW API for project: {}. Failing recommendation calculation.",
-						JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.nodeId());
+						JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.basicProjectConfigId());
 				throw new IllegalStateException(
-						"No KPI data received from KnowHOW API for project: " + projectInput.nodeId());
+						"No KPI data received from KnowHOW API for project: " + projectInput.basicProjectConfigId());
 			}
 
 			// Extract and format KPI data
@@ -91,18 +91,18 @@ public Map<String, Object> fetchKpiDataForProject(ProjectInputDTO projectInput)
 			if (!hasData) {
 				log.error(
 						"{} KPI data extraction resulted in empty values for all KPIs for project: {}. Failing recommendation calculation.",
-						JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.nodeId());
+						JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.basicProjectConfigId());
 				throw new IllegalStateException(
-						"No meaningful KPI data available for project: " + projectInput.nodeId());
+						"No meaningful KPI data available for project: " + projectInput.basicProjectConfigId());
 			}
 
 			log.debug("{} Successfully fetched {} KPIs for project: {}", JobConstants.LOG_PREFIX_RECOMMENDATION,
-					kpiData.size(), projectInput.nodeId());
+					kpiData.size(), projectInput.basicProjectConfigId());
 			return kpiData;
 
 		} catch (Exception e) {
 			log.error("{} Error fetching KPI data for project {}: {}", JobConstants.LOG_PREFIX_RECOMMENDATION,
-					projectInput.nodeId(), e.getMessage(), e);
+					projectInput.basicProjectConfigId(), e.getMessage(), e);
 			throw e;
 		}
 	}
@@ -158,14 +158,16 @@ private Map<String, Object> extractKpiData(List<KpiElement> kpiElements) {
 				}
 			} else if (trendValueObj != null) {
 				log.debug("{} Skipping non-list trendValueList for KPI {}: {} (type: {})",
-						JobConstants.LOG_PREFIX_RECOMMENDATION, kpiElement.getKpiId(),
-						kpiElement.getKpiName(), trendValueObj.getClass().getSimpleName());
+						JobConstants.LOG_PREFIX_RECOMMENDATION, kpiElement.getKpiId(), kpiElement.getKpiName(),
+						trendValueObj.getClass().getSimpleName());
 			}
 			kpiDataMap.put(kpiElement.getKpiName(), kpiDataPromptList);
 		});
 
 		return kpiDataMap;
-	}	/**
+	}
+
+	/**
 	 * Checks if DataCountGroup matches filter criteria. Matches if either the main
 	 * filter is in FILTER_LIST, or both filter1 and filter2 are in FILTER_LIST.
 	 * 

ai-data-processor/src/main/java/com/publicissapient/kpidashboard/job/recommendationcalculation/service/RecommendationCalculationService.java

@@ -70,7 +70,8 @@ public class RecommendationCalculationService {
 	 *             if AI response parsing or validation fails or if configuration is
 	 *             invalid
 	 */
-	public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull ProjectInputDTO projectInput) {
+	public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull ProjectInputDTO projectInput)
+			throws Exception {
 		if (CollectionUtils.isNotEmpty(recommendationCalculationConfig.getConfigValidationErrors())) {
 			throw new IllegalStateException(String.format("The following config validation errors occurred: %s",
 					String.join(CommonConstant.COMMA, recommendationCalculationConfig.getConfigValidationErrors())));
@@ -79,7 +80,7 @@ public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull Pro
 		Persona persona = recommendationCalculationConfig.getCalculationConfig().getEnabledPersona();
 
 		log.info("{} Calculating recommendations for project: {} ({}) - Persona: {}",
-				JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.name(), projectInput.nodeId(),
+				JobConstants.LOG_PREFIX_RECOMMENDATION, projectInput.name(), projectInput.basicProjectConfigId(),
 				persona.getDisplayName());
 
 		// Delegate KPI data extraction to specialized service
@@ -90,7 +91,8 @@ public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull Pro
 
 		// Validate prompt was generated successfully
 		if (prompt == null || prompt.trim().isEmpty()) {
-			throw new IllegalStateException("Failed to generate valid prompt for project: " + projectInput.nodeId());
+			throw new IllegalStateException(
+					"Failed to generate valid prompt for project: " + projectInput.basicProjectConfigId());
 		}
 
 		ChatGenerationRequest request = ChatGenerationRequest.builder().prompt(prompt).build();
@@ -99,7 +101,8 @@ public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull Pro
 
 		// Validate AI Gateway returned a response
 		if (response == null) {
-			throw new IllegalStateException("AI Gateway returned null response for project: " + projectInput.nodeId());
+			throw new IllegalStateException(
+					"AI Gateway returned null response for project: " + projectInput.basicProjectConfigId());
 		}
 
 		return buildRecommendationsActionPlan(projectInput, persona, response);
@@ -117,18 +120,18 @@ public RecommendationsActionPlan calculateRecommendationsForProject(@NonNull Pro
 	 * @param response
 	 *            the AI response DTO
 	 * @return complete recommendation action plan with metadata
-	 * @throws IllegalStateException
+	 * @throws Exception
 	 *             if parsing or validation fails
 	 */
 	private RecommendationsActionPlan buildRecommendationsActionPlan(ProjectInputDTO projectInput, Persona persona,
-			ChatGenerationResponseDTO response) {
+			ChatGenerationResponseDTO response) throws Exception {
 
 		Instant now = Instant.now();
 
 		// Parse and validate AI response
-		Recommendation recommendation = recommendationResponseParser.parseRecommendation(response)
-				.orElseThrow(() -> new IllegalStateException(
-						"Failed to parse AI recommendation for project: " + projectInput.nodeId())); // Build metadata
+		Recommendation recommendation = recommendationResponseParser.parseRecommendation(response);
+
+		// Build metadata
 		RecommendationMetadata metadata = RecommendationMetadata.builder()
 				.requestedKpis(recommendationCalculationConfig.getCalculationConfig().getKpiList()).persona(persona)
 				.build();