Skip to content

Implement UID sanitization and enhance schema building in content-typ… #828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sayalijoshi27 merged 1 commit into from
Nov 6, 2025
Merged

Implement UID sanitization and enhance schema building in content-typ… #828

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 66 additions & 101 deletions api/src/utils/content-type-creator.utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ interface ContentType {
schema: any[]; // Replace `any` with the specific type if known
}

const RESERVED_UIDS = new Set(['locale', 'publish_details', 'tags']);

function sanitizeUid(uid?: string) {
if (!uid) return uid;
let out = uid?.replace?.(/[^a-zA-Z0-9_]/g, '_').replace?.(/^_+/, '');
if (!/^[a-zA-Z]/.test(out)) out = `field_${out}`;
if (RESERVED_UIDS.has(out)) out = `cm_${out}`; // avoid reserved values
return out.toLowerCase();
}

function extractFieldName(input: string): string {
// Extract text inside parentheses (e.g., "JSON Editor-App")
const match = input.match(/\(([^)]+)\)/);
Expand Down Expand Up @@ -208,6 +218,11 @@ function getLastSegmentNew(str: string, separator: string): string {
}

export function buildSchemaTree(fields: any[], parentUid = '', parentType = ''): any[] {

if (!Array.isArray(fields)) {
console.warn('buildSchemaTree called with invalid fields:', fields);
return [];
}
// Build a lookup map for O(1) access
const fieldMap = new Map<string, any>();
fields.forEach(f => {
Expand Down Expand Up @@ -315,8 +330,8 @@ const saveAppMapper = async ({ marketPlacePath, data, fileName }: any) => {

const convertToSchemaFormate = ({ field, advanced = false, marketPlacePath, keyMapper }: any) => {
// Clean up field UID by removing ALL leading underscores
const cleanedUid = field?.uid?.replace(/^_+/, '') || field?.uid;

const rawUid = field?.uid;
const cleanedUid = sanitizeUid(rawUid);
switch (field?.contentstackFieldType) {
case 'single_line_text': {
return {
Expand Down Expand Up @@ -416,18 +431,18 @@ const convertToSchemaFormate = ({ field, advanced = false, marketPlacePath, keyM

case 'dropdown': {
// 🔧 CONDITIONAL LOGIC: Check if choices have key-value pairs or just values
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
: [{ value: "NF" }];

// Filter out null/undefined choices and ensure they are valid objects
const choices = Array.isArray(rawChoices)
const choices = Array.isArray(rawChoices)
? rawChoices.filter((choice: any) => choice != null && typeof choice === 'object')
: [{ value: "NF" }];
const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&

const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&
choices.some((choice: any) => choice != null && typeof choice === 'object' && choice.key !== undefined && choice.key !== null);

const data = {
"data_type": ['dropdownNumber', 'radioNumber', 'ratingNumber'].includes(field.otherCmsType) ? 'number' : "text",
"display_name": field?.title,
Expand Down Expand Up @@ -456,18 +471,18 @@ const convertToSchemaFormate = ({ field, advanced = false, marketPlacePath, keyM
}
case 'radio': {
// 🔧 CONDITIONAL LOGIC: Check if choices have key-value pairs or just values
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
: [{ value: "NF" }];

// Filter out null/undefined choices and ensure they are valid objects
const choices = Array.isArray(rawChoices)
const choices = Array.isArray(rawChoices)
? rawChoices.filter((choice: any) => choice != null && typeof choice === 'object')
: [{ value: "NF" }];
const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&

const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&
choices.some((choice: any) => choice != null && typeof choice === 'object' && choice.key !== undefined && choice.key !== null);

const data = {
"data_type": ['dropdownNumber', 'radioNumber', 'ratingNumber'].includes(field.otherCmsType) ? 'number' : "text",
"display_name": field?.title,
Expand Down Expand Up @@ -495,18 +510,18 @@ const convertToSchemaFormate = ({ field, advanced = false, marketPlacePath, keyM
}
case 'checkbox': {
// 🔧 CONDITIONAL LOGIC: Check if choices have key-value pairs or just values
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
const rawChoices = Array.isArray(field?.advanced?.options) && field?.advanced?.options?.length > 0
? field?.advanced?.options
: [{ value: "NF" }];

// Filter out null/undefined choices and ensure they are valid objects
const choices = Array.isArray(rawChoices)
const choices = Array.isArray(rawChoices)
? rawChoices.filter((choice: any) => choice != null && typeof choice === 'object')
: [{ value: "NF" }];
const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&

const hasKeyValuePairs = Array.isArray(choices) && choices.length > 0 &&
choices.some((choice: any) => choice != null && typeof choice === 'object' && choice.key !== undefined && choice.key !== null);

const data = {
"data_type": "text",
"display_name": field?.title,
Expand Down Expand Up @@ -803,7 +818,7 @@ const convertToSchemaFormate = ({ field, advanced = false, marketPlacePath, keyM
"non_localizable": field.advanced?.nonLocalizable ?? false,
}
} else {
console.info('Contnet Type Filed', field?.contentstackField)
console.info('Content Type Field', field?.contentstackField)
}
}
}
Expand Down Expand Up @@ -866,22 +881,22 @@ const writeGlobalField = async (schema: any, globalSave: string) => {
return;
}
}

// 🔧 FIX: Check for duplicates before adding
if (!schema || typeof schema !== 'object') {
console.error("🚀 ~ writeGlobalField ~ Invalid schema provided");
return;
}

if (!schema.uid) {
console.error("🚀 ~ writeGlobalField ~ Schema missing uid");
return;
}

if (!Array.isArray(globalfields)) {
globalfields = [];
}

const existingIndex = globalfields.findIndex((gf: any) => gf != null && gf.uid === schema.uid);
if (existingIndex !== -1 && existingIndex < globalfields.length) {
// Replace existing global field instead of duplicating
Expand All @@ -896,7 +911,7 @@ const writeGlobalField = async (schema: any, globalSave: string) => {
console.error("🚀 ~ writeGlobalField ~ Cannot push schema: invalid schema or globalfields array");
}
}

try {
await fs.promises.writeFile(filePath, JSON.stringify(globalfields, null, 2));
} catch (writeErr) {
Expand Down Expand Up @@ -971,90 +986,40 @@ const mergeTwoCts = async (ct: any, mergeCts: any) => {
export const contenTypeMaker = async ({ contentType, destinationStackId, projectId, newStack, keyMapper, region, user_id }: any) => {
const marketPlacePath = path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, destinationStackId);
const srcFunc = 'contenTypeMaker';

let ct: ContentType = {
title: contentType?.contentstackTitle,
uid: contentType?.contentstackUid,
schema: []
}
};

let currentCt: any = {};
if (Object?.keys?.(keyMapper)?.length &&
keyMapper?.[contentType?.contentstackUid] !== "" &&
keyMapper?.[contentType?.contentstackUid] !== undefined) {
currentCt = await existingCtMapper({ keyMapper, contentTypeUid: contentType?.contentstackUid, projectId, region, user_id });
}
const ctData: any = buildSchemaTree(contentType?.fieldMapping);
ctData?.forEach((item: any) => {
if (item?.contentstackFieldType === 'group') {
const group: Group = {
"data_type": "group",
"display_name": item?.contentstackField,
"field_metadata": {},
"schema": [],
"uid": item?.contentstackFieldUid,
"multiple": false,
"mandatory": false,
"unique": false
}
// 🔧 FIX: Track processed fields to prevent duplicates within groups
const processedFieldUIDs = new Set();

item?.schema?.forEach((element: any) => {
const fieldUID = extractValue(element?.contentstackFieldUid, item?.contentstackFieldUid, '.');

// Skip if this field UID was already processed in this group
if (processedFieldUIDs.has(fieldUID)) {
return;
}
processedFieldUIDs.add(fieldUID);

const field: any = {
...element,
uid: fieldUID,
title: extractValue(element?.contentstackField, item?.contentstackField, ' >')?.trim(),
}
const schema: any = convertToSchemaFormate({
field,
advanced: element?.advanced ?? false, // 🔧 FIX: Pass advanced from element data
marketPlacePath,
keyMapper
});
if (typeof schema === 'object' && Array.isArray(group?.schema) && element?.isDeleted === false) {
group.schema.push(schema);
}
})

// 🔧 FIX: Only add group if it has schema and doesn't already exist
if (group?.schema && Array.isArray(group.schema) && group.schema.length > 0 && group?.uid) {
if (!ct?.schema || !Array.isArray(ct.schema)) {
ct.schema = [];
}
const existingGroupIndex = ct.schema.findIndex((g: any) => g != null && g.uid === group.uid);
if (existingGroupIndex !== -1 && existingGroupIndex < ct.schema.length) {
ct.schema[existingGroupIndex] = group;
} else {
ct.schema.push(group);
}
}
} else {
const dt: any = convertToSchemaFormate({
field: {
...item,
title: item?.contentstackField,
uid: item?.contentstackFieldUid
},
advanced: item?.advanced ?? false, // 🔧 FIX: Pass advanced from item data
marketPlacePath,
keyMapper
});
if (dt && item?.isDeleted === false) {
ct?.schema?.push(dt);
}

// Safe: ensures we never pass undefined to the builder
const ctData: any[] = buildSchemaTree(contentType?.fieldMapping || []);

// Use the deep converter that properly handles groups & modular blocks
for (const item of ctData) {
if (item?.isDeleted === true) continue;

const fieldSchema = buildFieldSchema(item, marketPlacePath, '');
if (fieldSchema) {
ct?.schema.push(fieldSchema);
}
})
}

// dedupe by uid to avoid dup nodes after merges
ct.schema = removeDuplicateFields(ct.schema || []);

if (currentCt?.uid) {
ct = await mergeTwoCts(ct, currentCt);
}
if (ct?.uid && ct?.schema?.length) {
if (ct?.uid && Array.isArray(ct?.schema) && ct?.schema.length) {
if (contentType?.type === 'global_field') {
const globalSave = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, GLOBAL_FIELDS_DIR_NAME);
const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {});
Expand All @@ -1067,6 +1032,6 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project
await saveContent(ct, contentSave);
}
} else {
console.info(contentType?.contentstackUid, 'missing')
console.info(contentType?.contentstackUid, 'missing');
}
}
};