feat: tighten v2 payload diagnostics

Author: luandro

src/controllers/settingsController.ts

@@ -1,10 +1,11 @@
 import fs from 'fs/promises';
+import os from 'os';
 import path from 'path';
 import { Reader } from 'comapeocat'; // Added Reader import
 import { buildSettingsV1 } from '../services/settingsBuilder';
 import { buildComapeoCatV2 } from '../services/comapeocatBuilder';
 import { ValidationError, ProcessingError } from '../types/errors';
-import type { BuildRequestV2 } from '../types/v2';
+import type { BuildRequestV2, IconInput } from '../types/v2';
 import { config } from '../config/app';
 
 /**
@@ -143,7 +144,7 @@ export async function handleBuildSettingsV2(payload: BuildRequestV2) {
 
     return new Response(cleanupStream, { headers });
   } catch (error) {
-    logV2PayloadForFailure(payload, error);
+    await logV2PayloadForFailure(payload, error);
     throw error;
   }
 }
@@ -183,11 +184,160 @@ function formatWarningsHeader(warnings: string[]) {
   return joined.length > 1024 ? `${joined.slice(0, 1021)}...` : joined;
 }
 
-function logV2PayloadForFailure(payload: BuildRequestV2, error: unknown) {
+const PAYLOAD_LOG_PREFIX = 'comapeo-v2-payload-';
+const SUMMARY_SAMPLE_LIMIT = 8;
+const ISSUE_SAMPLE_LIMIT = 5;
+
+async function logV2PayloadForFailure(payload: BuildRequestV2, error: unknown) {
   try {
-    const serializedPayload = JSON.stringify(payload, null, 2);
-    console.error('[COMAPEO-API][ERROR] /v2 build failed. Payload dump follows:', serializedPayload, error);
+    const summary = buildPayloadSummary(payload);
+    const payloadPath = await persistPayloadSnapshot(payload);
+    console.error(
+      `[COMAPEO-API][ERROR] /v2 build failed. Summary: ${JSON.stringify(summary)}. Full payload saved to: ${payloadPath}`,
+      error
+    );
   } catch (serializationError) {
-    console.error('[COMAPEO-API][ERROR] /v2 build failed and payload serialization threw an error:', serializationError, error);
+    console.error(
+      '[COMAPEO-API][ERROR] /v2 build failed and payload logging also failed:',
+      serializationError,
+      error
+    );
   }
 }
+
+function buildPayloadSummary(payload: BuildRequestV2) {
+  const categories = Array.isArray(payload.categories) ? payload.categories : [];
+  const fields = Array.isArray(payload.fields) ? payload.fields : [];
+  const icons = Array.isArray(payload.icons) ? payload.icons : [];
+  const translationLocales = payload.translations && typeof payload.translations === 'object'
+    ? Object.keys(payload.translations)
+    : [];
+
+  const categoryIds = categories.map((c) => c.id).filter(Boolean);
+  const fieldIds = fields.map((f) => f.id).filter(Boolean);
+  const localesSample = truncateList(translationLocales, SUMMARY_SAMPLE_LIMIT);
+  const categoriesWithoutFields = categories
+    .filter((category) => !Array.isArray(category.fields) && !Array.isArray(category.defaultFieldIds))
+    .map((category) => category.id);
+
+  const iconIds = new Set(icons.map((icon) => icon.id).filter(Boolean));
+  const referencedIconIds = categories
+    .map((category) => {
+      const directIcon = (category as any).icon;
+      if (typeof directIcon === 'string' && directIcon.trim()) {
+        return directIcon.trim();
+      }
+      if (typeof category.iconId === 'string' && category.iconId.trim()) {
+        return category.iconId.trim();
+      }
+      return undefined;
+    })
+    .filter((value): value is string => Boolean(value));
+
+  const missingIconRefs = iconIds.size > 0
+    ? Array.from(new Set(referencedIconIds.filter((ref) => !iconIds.has(ref))))
+    : [];
+
+  const potentialIssues: string[] = [];
+
+  if (categoriesWithoutFields.length > 0) {
+    potentialIssues.push(
+      formatIssue(
+        'Categories without field references',
+        categoriesWithoutFields,
+        ISSUE_SAMPLE_LIMIT
+      )
+    );
+  }
+
+  if (missingIconRefs.length > 0) {
+    potentialIssues.push(
+      formatIssue('Categories referencing missing icon ids', missingIconRefs, ISSUE_SAMPLE_LIMIT)
+    );
+  }
+
+  const iconStats = summarizeIconStats(icons);
+
+  return {
+    metadata: {
+      name: payload.metadata?.name || 'unknown',
+      version: payload.metadata?.version || 'n/a',
+    },
+    counts: {
+      categories: categories.length,
+      fields: fields.length,
+      icons: icons.length,
+      translations: translationLocales.length,
+    },
+    samples: {
+      categoryIds: truncateList(categoryIds, SUMMARY_SAMPLE_LIMIT),
+      fieldIds: truncateList(fieldIds, SUMMARY_SAMPLE_LIMIT),
+      locales: localesSample,
+    },
+    potentialIssues,
+    iconStats,
+  };
+}
+
+function summarizeIconStats(icons: IconInput[]) {
+  if (!icons || icons.length === 0) {
+    return undefined;
+  }
+
+  let inlineCount = 0;
+  let dataUriCount = 0;
+  let remoteCount = 0;
+  let missingSourceCount = 0;
+  let inlineBytesTotal = 0;
+  let inlineBytesMax = 0;
+
+  for (const icon of icons) {
+    if (typeof icon.svgData === 'string') {
+      inlineCount += 1;
+      const size = Buffer.byteLength(icon.svgData, 'utf-8');
+      inlineBytesTotal += size;
+      inlineBytesMax = Math.max(inlineBytesMax, size);
+    } else if (typeof icon.svgUrl === 'string') {
+      if (icon.svgUrl.startsWith('data:image/svg+xml')) {
+        dataUriCount += 1;
+      } else {
+        remoteCount += 1;
+      }
+    } else {
+      missingSourceCount += 1;
+    }
+  }
+
+  const inlineAvg = inlineCount > 0 ? Math.round(inlineBytesTotal / inlineCount) : 0;
+
+  return {
+    inlineCount,
+    dataUriCount,
+    remoteCount,
+    missingSourceCount,
+    inlineBytesTotal,
+    inlineBytesMax,
+    inlineBytesAvg: inlineAvg,
+  };
+}
+
+function formatIssue(label: string, values: string[], limit: number) {
+  const uniqueValues = Array.from(new Set(values));
+  const sample = truncateList(uniqueValues, limit);
+  const excess = uniqueValues.length - sample.length;
+  return excess > 0 ? `${label}: ${sample.join(', ')} (+${excess} more)` : `${label}: ${sample.join(', ')}`;
+}
+
+function truncateList<T>(values: T[], limit: number): T[] {
+  if (!Array.isArray(values) || values.length <= limit) {
+    return values.slice();
+  }
+  return values.slice(0, limit);
+}
+
+async function persistPayloadSnapshot(payload: BuildRequestV2) {
+  const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), PAYLOAD_LOG_PREFIX));
+  const filePath = path.join(tempDir, `payload-${Date.now()}.json`);
+  await fs.writeFile(filePath, JSON.stringify(payload, null, 2), 'utf-8');
+  return filePath;
+}

src/services/comapeocatBuilder.ts

@@ -10,6 +10,7 @@ import { config } from '../config/app';
 import { ProcessingError, ValidationError } from '../types/errors';
 import type {
   BuildRequestV2,
+  CategoryInput,
   ComapeoFieldType,
   FieldInput,
   MappedCategory,
@@ -42,6 +43,12 @@ export async function buildComapeoCatV2(payload: BuildRequestV2): Promise<BuildR
   // Layer 1 enforces size during streaming in app.ts onParse hook
   enforcePayloadSize(payload);
   enforceEntryCap(payload);
+  enforceUniqueIds(payload.fields, 'field');
+  enforceUniqueIds(payload.categories, 'category');
+  if (Array.isArray(payload.icons) && payload.icons.length > 0) {
+    enforceUniqueIds(payload.icons, 'icon');
+  }
+  validateCategoryFieldReferences(payload.categories, payload.fields);
 
   const mapped = await transformPayload(payload);
   const warnings: string[] = [...mapped.warnings];
@@ -845,6 +852,63 @@ function normalizeToNodeStream(streamLike: any) {
   throw new ProcessingError('Writer outputStream is not readable');
 }
 
+function enforceUniqueIds<T extends { id?: string }>(items: T[], entityName: string) {
+  const seen = new Set<string>();
+
+  for (const item of items) {
+    const rawId = typeof item?.id === 'string' ? item.id.trim() : '';
+    if (!rawId) {
+      throw new ValidationError(`${entityName} id must be a non-empty string`);
+    }
+
+    if (seen.has(rawId)) {
+      throw new ValidationError(`Duplicate ${entityName} id detected: ${rawId}`);
+    }
+
+    seen.add(rawId);
+  }
+}
+
+function validateCategoryFieldReferences(categories: CategoryInput[], fields: FieldInput[]) {
+  const fieldIds = new Set(fields.map((field) => field.id).filter((id): id is string => Boolean(id)));
+  const issues: string[] = [];
+
+  for (const category of categories) {
+    const refs = new Set<string>();
+
+    if (Array.isArray(category.fields)) {
+      for (const ref of category.fields) {
+        if (typeof ref === 'string' && ref.trim()) {
+          refs.add(ref.trim());
+        }
+      }
+    }
+
+    if (Array.isArray(category.defaultFieldIds)) {
+      for (const ref of category.defaultFieldIds) {
+        if (typeof ref === 'string' && ref.trim()) {
+          refs.add(ref.trim());
+        }
+      }
+    }
+
+    if (refs.size === 0) {
+      continue;
+    }
+
+    const missing = Array.from(refs).filter((ref) => !fieldIds.has(ref));
+    if (missing.length > 0) {
+      issues.push(`${category.id}: ${missing.join(', ')}`);
+    }
+  }
+
+  if (issues.length > 0) {
+    throw new ValidationError(
+      `Categories reference unknown field ids. Missing references -> ${issues.join(' | ')}`
+    );
+  }
+}
+
 /**
  * Sanitizes a string to be safe for use as a path component.
  * Rejects inputs containing path separators or traversal patterns to prevent attacks.
@@ -889,6 +953,8 @@ export const __test__ = {
   normalizeTags,
   enforcePayloadSize,
   enforceEntryCap,
+  enforceUniqueIds,
+  validateCategoryFieldReferences,
   sanitizePathComponent,
   decodeDataUri,
   normalizeIconId,

src/tests/unit/services/comapeocatBuilder.test.ts

@@ -20,6 +20,30 @@ describe('comapeocatBuilder helpers', () => {
     expect(() => __test__.mapFieldType(badField.type as any)).toThrow(ValidationError);
   });
 
+  it('throws when duplicate field ids are provided', async () => {
+    const payload = createBasePayload();
+    payload.fields.push({ ...payload.fields[0] });
+
+    await expect(buildComapeoCatV2(payload)).rejects.toThrow('Duplicate field id');
+  });
+
+  it('throws when duplicate category ids are provided', async () => {
+    const payload = createBasePayload();
+    payload.categories.push({ ...payload.categories[0] });
+
+    await expect(buildComapeoCatV2(payload)).rejects.toThrow('Duplicate category id');
+  });
+
+  it('throws when duplicate icon ids are provided', async () => {
+    const payload = createBasePayload();
+    payload.icons = [
+      { id: 'icon-1', svgData: '<svg />' },
+      { id: 'icon-1', svgData: '<svg />' },
+    ];
+
+    await expect(buildComapeoCatV2(payload)).rejects.toThrow('Duplicate icon id');
+  });
+
   it('fails when appliesTo is missing', () => {
     expect(() => __test__.mapCategory({ id: 'cat', name: 'Cat' }, 0)).toThrow(ValidationError);
   });
@@ -72,6 +96,13 @@ describe('comapeocatBuilder helpers', () => {
     expect(() => __test__.mapCategory(category, 0)).toThrow('appliesTo must only contain "observation" or "track"');
   });
 
+  it('throws when a category references unknown field ids', async () => {
+    const payload = createBasePayload();
+    payload.categories[0].fields = ['missing-field'];
+
+    await expect(buildComapeoCatV2(payload)).rejects.toThrow('unknown field ids');
+  });
+
   it('throws when appliesTo contains mix of valid and invalid values', () => {
     const category = { id: 'cat', name: 'Cat', appliesTo: ['observation', 'invalid', 'track'] };
     expect(() => __test__.mapCategory(category, 0)).toThrow(ValidationError);