fix(graph): chart projection, cache refresh, validation and ADR compliance

- Use parent field for compound types (address, money, etc.) in projection
  to avoid clearProjectionPathCollision stripping sibling subfields (fixes
  address.district not appearing on graph X axis)
- Honor Cache-Control: no-cache in withBlobCache for widget refresh
- Validate graph config fields exist before stream; return 400 for
  GRAPH_FIELD_NOT_FOUND
- Strip legacy yAxis when series present (graphStream, graphMetadata)
- GRAPH_FIELD_NOT_FOUND message empathetic, non-technical (ADR-0024)
- validateGraphFieldsExist: use .find() instead of for...of (ADR-0012)

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: silveirado

src/imports/data/api/graphMetadata.ts

@@ -149,14 +149,18 @@ export function enrichGraphConfig(document: string, graphConfig: GraphConfig, la
 		};
 	}
 
-	// Enriquecer yAxis com label traduzido (legado)
-	if (graphConfig.yAxis?.field) {
+	// Enriquecer yAxis com label traduzido (legado — skip when series are present)
+	const hasSeries = Array.isArray(graphConfig.series) && graphConfig.series.length > 0;
+	if (graphConfig.yAxis?.field && !hasSeries) {
 		const fieldMeta = resolveFieldMeta(document, graphConfig.yAxis.field, lang);
 		const shouldUseMetaLabel = isUntranslatedLabel(graphConfig.yAxis.label, graphConfig.yAxis.field);
 		enrichedConfig.yAxis = {
 			...graphConfig.yAxis,
 			label: shouldUseMetaLabel ? fieldMeta.label : graphConfig.yAxis.label,
 		};
+	} else if (hasSeries && graphConfig.yAxis?.field) {
+		// Strip legacy yAxis when series are present to avoid downstream confusion
+		enrichedConfig.yAxis = undefined;
 	}
 
 	// Enriquecer series com labels traduzidos

src/imports/data/api/graphStream.ts

@@ -66,33 +66,22 @@ function extractFieldsFromGraphConfig(document: string, graphConfig: GraphConfig
 		if (field.type === 'lookup' || field.type === 'inheritLookup') {
 			// For lookups, we only need _id - we'll populate the rest later
 			fields.add(`${baseFieldName}._id`);
-		} else if (field.type === 'address') {
-			const addressFields = ['city', 'state', 'country', 'district', 'place', 'number', 'postalCode', 'complement', 'placeType'];
-			addressFields.forEach(af => {
-				fields.add(`${fieldPath}.${af}`);
-			});
-		} else if (field.type === 'money') {
-			fields.add(`${fieldPath}.value`);
-			fields.add(`${fieldPath}.currency`);
-		} else if (field.type === 'personName') {
-			fields.add(`${fieldPath}.full`);
-			fields.add(`${fieldPath}.first`);
-			fields.add(`${fieldPath}.last`);
-		} else if (field.type === 'phone') {
-			fields.add(`${fieldPath}.phoneNumber`);
-			fields.add(`${fieldPath}.countryCode`);
-		} else if (field.type === 'email') {
-			fields.add(`${fieldPath}.address`);
+		} else if (['address', 'money', 'personName', 'phone', 'email'].includes(field.type)) {
+			// Use the parent field name to avoid clearProjectionPathCollision stripping sibling subfields
+			fields.add(baseFieldName);
 		} else {
 			fields.add(fieldPath);
 		}
 	};
 
 	// Extract fields from xAxis, yAxis, categoryField, and series
+	const hasSeries = Array.isArray(graphConfig.series) && graphConfig.series.length > 0;
+
 	if (graphConfig.xAxis?.field) {
 		expandField(graphConfig.xAxis.field);
 	}
-	if (graphConfig.yAxis?.field) {
+	// yAxis is legacy — skip when series are present (series take precedence)
+	if (graphConfig.yAxis?.field && !hasSeries) {
 		expandField(graphConfig.yAxis.field);
 	}
 	if (graphConfig.categoryField) {
@@ -101,9 +90,8 @@ function extractFieldsFromGraphConfig(document: string, graphConfig: GraphConfig
 		// System fields starting with _ are already handled by expandField, but ensure it's explicitly added
 		fields.add(graphConfig.categoryField);
 	}
-	// Extract fields from series
-	if (graphConfig.series) {
-		graphConfig.series.forEach(serie => {
+	if (hasSeries) {
+		graphConfig.series!.forEach(serie => {
 			if (serie.field) {
 				expandField(serie.field);
 			}
@@ -114,7 +102,8 @@ function extractFieldsFromGraphConfig(document: string, graphConfig: GraphConfig
 	// This ensures _createdAt, _updatedAt, etc. are available even if projection filters them
 	// NOTE: System fields that are lookups are handled by expandField above (they get _id suffix)
 	// Here we only add non-lookup system fields (like _createdAt, _updatedAt, _id)
-	const systemFieldsToCheck = [graphConfig.categoryField, graphConfig.xAxis?.field, graphConfig.yAxis?.field];
+	const systemFieldsToCheck: (string | undefined)[] = [graphConfig.categoryField, graphConfig.xAxis?.field];
+	if (!hasSeries && graphConfig.yAxis?.field) systemFieldsToCheck.push(graphConfig.yAxis.field);
 	if (graphConfig.series) {
 		graphConfig.series.forEach(serie => {
 			if (serie.field) systemFieldsToCheck.push(serie.field);
@@ -139,6 +128,38 @@ function extractFieldsFromGraphConfig(document: string, graphConfig: GraphConfig
 	return Array.from(fields);
 }
 
+/**
+ * Validate that every field path used in the graph config exists on the document meta.
+ * Prevents RPC "Field not found" from Python when the document has no such field.
+ */
+function validateGraphFieldsExist(
+	document: string,
+	graphConfig: GraphConfig,
+): { valid: true } | { valid: false; field: string } {
+	const meta = MetaObject.Meta[document];
+	if (meta == null) return { valid: true };
+
+	const fieldPaths: string[] = [];
+	const hasSeries = Array.isArray(graphConfig.series) && graphConfig.series.length > 0;
+	if (graphConfig.xAxis?.field) fieldPaths.push(graphConfig.xAxis.field);
+	// yAxis is legacy — skip validation when series are present (series take precedence)
+	if (graphConfig.yAxis?.field && !hasSeries) fieldPaths.push(graphConfig.yAxis.field);
+	if (graphConfig.categoryField) fieldPaths.push(graphConfig.categoryField);
+	if (hasSeries) {
+		graphConfig.series!.forEach(serie => {
+			if (serie.field) fieldPaths.push(serie.field);
+		});
+	}
+
+	const invalidField = fieldPaths.find(path => {
+		if (!path) return false;
+		const baseFieldName = path.split('.')[0];
+		if (baseFieldName.startsWith('_')) return false;
+		return meta.fields[baseFieldName] == null;
+	});
+	return invalidField != null ? { valid: false, field: invalidField } : { valid: true };
+}
+
 /**
  * Identify lookup fields in the graph config
  */
@@ -177,19 +198,19 @@ function getLookupFieldsInfo(
 		}
 	};
 
-	// Process xAxis, yAxis, categoryField, and series
+	const hasSeries = Array.isArray(graphConfig.series) && graphConfig.series.length > 0;
+
 	if (graphConfig.xAxis?.field) {
 		processField(graphConfig.xAxis.field);
 	}
-	if (graphConfig.yAxis?.field) {
+	if (graphConfig.yAxis?.field && !hasSeries) {
 		processField(graphConfig.yAxis.field);
 	}
 	if (graphConfig.categoryField) {
 		processField(graphConfig.categoryField);
 	}
-	// Process series fields
-	if (graphConfig.series) {
-		graphConfig.series.forEach(serie => {
+	if (hasSeries) {
+		graphConfig.series!.forEach(serie => {
 			if (serie.field) {
 				processField(serie.field);
 			}
@@ -377,6 +398,16 @@ export default async function graphStream({
 		const enrichedConfig = enrichGraphConfig(findParams.document, graphConfig, findParams.lang || 'pt_BR');
 		logger.debug({ lang: findParams.lang || 'pt_BR' }, 'Enriched graph config');
 
+		const fieldValidation = validateGraphFieldsExist(findParams.document, enrichedConfig);
+		if (!fieldValidation.valid) {
+			tracingSpan?.end();
+			const errorMsg = getGraphErrorMessage('GRAPH_FIELD_NOT_FOUND', {
+				field: fieldValidation.field,
+				document: findParams.document,
+			});
+			return errorReturn([{ message: errorMsg.message, code: errorMsg.code, details: errorMsg.details } as KonectyError]);
+		}
+
 		// 1.1 Extract fields from graph config for proper projection
 		const graphFields = extractFieldsFromGraphConfig(findParams.document, enrichedConfig);
 		tracingSpan?.addEvent('Extracted graph fields', { fields: graphFields.join(',') });

src/imports/utils/graphErrors.ts

@@ -57,6 +57,8 @@ const ERROR_MESSAGES: Record<string, string> = {
   GRAPH_CONFIG_AXIS_Y_MISSING: "Y axis not configured. Please configure the Y axis for {{type}} charts.",
   GRAPH_CONFIG_CATEGORY_MISSING: "Category field not configured. Please configure the category field for pie charts.",
   GRAPH_FILTER_INVALID: "Applied filters are invalid. Please check the filters and try again.",
+  GRAPH_FIELD_NOT_FOUND:
+    "The selected field for this chart is no longer available. Please edit the chart and choose a different field.",
   GRAPH_PROCESSING_ERROR: "An error occurred while processing the graph. Please try again or contact support.",
   GRAPH_TIMEOUT: "Graph generation took too long. Please try again with a smaller dataset or more filters.",
   GRAPH_DATA_ERROR: "Could not load data for the graph. Please check your connection and try again.",

src/server/routes/rest/data/dataApi.ts

@@ -803,39 +803,55 @@ export const dataApi: FastifyPluginCallback = (fastify, _, done) => {
 		}
 		const userId = userResult.data._id;
 
-		const cacheResult = await withBlobCache({
-			req: req as unknown as { headers: Record<string, string | string[] | undefined> },
-			reply,
-			userId,
-			document: req.params.document,
-			operation: 'graph',
-			configHash: hashConfig(graphConfig),
-			filter: parsedFilter,
-			cacheTTL,
-			compute: async () => {
-				const result = await graphStream({
-					authTokenId,
-					document: req.params.document,
-					displayName: req.query.displayName,
-					displayType: req.query.displayType,
-					fields: req.query.fields,
-					filter: parsedFilter,
-					sort: req.query.sort,
-					limit: String(limit),
-					start: req.query.start,
-					withDetailFields: req.query.withDetailFields,
-					graphConfig,
-					lang,
-					tracingSpan,
-				});
+		let cacheResult: { blob: string; fromCache: boolean; notModified: boolean };
+		try {
+			cacheResult = await withBlobCache({
+				req: req as unknown as { headers: Record<string, string | string[] | undefined> },
+				reply,
+				userId,
+				document: req.params.document,
+				operation: 'graph',
+				configHash: hashConfig(graphConfig),
+				filter: parsedFilter,
+				cacheTTL,
+				compute: async () => {
+					const result = await graphStream({
+						authTokenId,
+						document: req.params.document,
+						displayName: req.query.displayName,
+						displayType: req.query.displayType,
+						fields: req.query.fields,
+						filter: parsedFilter,
+						sort: req.query.sort,
+						limit: String(limit),
+						start: req.query.start,
+						withDetailFields: req.query.withDetailFields,
+						graphConfig,
+						lang,
+						tracingSpan,
+					});
 
-				if (result.success === false) {
-					throw new Error(JSON.stringify(result));
-				}
+					if (result.success === false) {
+						throw new Error(JSON.stringify(result));
+					}
 
-				return result.svg;
-			},
-		});
+					return result.svg;
+				},
+			});
+		} catch (err) {
+			tracingSpan.end();
+			const errMessage = err instanceof Error ? err.message : String(err);
+			try {
+				const parsed = JSON.parse(errMessage) as { success?: false; errors?: Array<{ code?: string }> };
+				if (parsed.success === false && Array.isArray(parsed.errors)) {
+					const status = parsed.errors[0]?.code === 'GRAPH_FIELD_NOT_FOUND' ? 400 : 500;
+					return reply.status(status).type('application/json').send(parsed);
+				}
+			} catch {
+				// not our JSON
+			}
+			return reply.status(500).type('application/json').send({ success: false, errors: [{ message: errMessage }] });
+		}
 
 		tracingSpan.end();
 
@@ -857,7 +873,7 @@ export const dataApi: FastifyPluginCallback = (fastify, _, done) => {
 	/**
 	 * ADR-0049 / DRY: Shared cache helper for blob-based endpoints (graph SVG, pivot JSON).
 	 * Encapsulates: check cache → ETag/304 → cache miss → compute() → store → HTTP headers.
-	 * Keeps the cache pattern in a single place instead of duplicating across endpoints.
+	 * When client sends Cache-Control: no-cache (e.g. widget refresh), bypasses cache so compute() runs.
 	 */
 	const withBlobCache = async (opts: {
 		req: { headers: Record<string, string | string[] | undefined> };
@@ -873,25 +889,30 @@ export const dataApi: FastifyPluginCallback = (fastify, _, done) => {
 		const { req, reply, userId, document, operation, configHash, filter, cacheTTL, compute } = opts;
 
 		const cacheKey = buildCacheKey(userId, document, operation, configHash, filter);
+		const clientCacheControl = req.headers['cache-control'];
+		const cacheControlStr = Array.isArray(clientCacheControl) ? clientCacheControl[0] : clientCacheControl;
+		const forceRefresh = typeof cacheControlStr === 'string' && cacheControlStr.toLowerCase().includes('no-cache');
+
+		// When client requests force refresh (e.g. widget refresh button), skip cache lookup
+		if (!forceRefresh) {
+			const cached = await getCachedBlob(cacheKey);
+			if (cached != null) {
+				const clientEtag = req.headers['if-none-match'];
+				if (clientEtag != null && clientEtag === cached.etag) {
+					reply.header('ETag', cached.etag);
+					reply.header('Cache-Control', `private, max-age=${cacheTTL}, stale-while-revalidate=${STALE_WHILE_REVALIDATE_SECONDS}`);
+					reply.header('Vary', 'Authorization, Cookie');
+					return { blob: '', fromCache: true, notModified: true };
+				}
 
-		// Check cache
-		const cached = await getCachedBlob(cacheKey);
-		if (cached != null) {
-			const clientEtag = req.headers['if-none-match'];
-			if (clientEtag != null && clientEtag === cached.etag) {
 				reply.header('ETag', cached.etag);
 				reply.header('Cache-Control', `private, max-age=${cacheTTL}, stale-while-revalidate=${STALE_WHILE_REVALIDATE_SECONDS}`);
 				reply.header('Vary', 'Authorization, Cookie');
-				return { blob: '', fromCache: true, notModified: true };
+				return { blob: cached.blob, fromCache: true, notModified: false };
 			}
-
-			reply.header('ETag', cached.etag);
-			reply.header('Cache-Control', `private, max-age=${cacheTTL}, stale-while-revalidate=${STALE_WHILE_REVALIDATE_SECONDS}`);
-			reply.header('Vary', 'Authorization, Cookie');
-			return { blob: cached.blob, fromCache: true, notModified: false };
 		}
 
-		// Cache miss — execute computation
+		// Cache miss or force refresh — execute computation
 		const blob = await compute();
 
 		// Store in cache (skip on force refresh to not pollute cache with forced results)