diff --git a/api/src/services/contentful.service.ts b/api/src/services/contentful.service.ts index 68b548c63..124b9b847 100644 --- a/api/src/services/contentful.service.ts +++ b/api/src/services/contentful.service.ts @@ -10,7 +10,7 @@ import { JSDOM } from "jsdom"; import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer'; -import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js"; +import { CHUNK_SIZE, LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../constants/index.js"; import { Locale } from "../models/types.js"; import jsonRTE from "./contentful/jsonRTE.js"; import { getAllLocales, getLogMessage } from "../utils/index.js"; @@ -86,6 +86,18 @@ function makeChunks(assetData: any) { return chunks; } +const mapLocales = ({ masterLocale, locale, locales }: any) => { + if (locales?.masterLocale?.[masterLocale ?? ''] === locale) { + return Object?.keys(locales?.masterLocale)?.[0] + } + for (const [key, value] of Object?.entries?.(locales) ?? {}) { + if (typeof value !== 'object' && value === locale) { + return key; + } + } + return locale.toLowerCase(); +} + const transformCloudinaryObject = (input: any) => { const result: any = []; if (!Array.isArray(input)) { @@ -122,7 +134,7 @@ const transformCloudinaryObject = (input: any) => { id: uuidv4(), folder: "", cs_metadata: { - config_label: "config" + config_label: "default_multi_config_key" } }); } @@ -702,17 +714,6 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string, } }; -const mapLocales = ({ masterLocale, locale, locales }: any) => { - if (locales?.masterLocale?.[masterLocale ?? ''] === locale) { - return Object?.keys(locales?.masterLocale)?.[0] - } - for (const [key, value] of Object?.entries?.(locales) ?? {}) { - if (typeof value !== 'object' && value === locale) { - return key; - } - } - return locale.toLowerCase(); -} /** * Creates and processes entries from a given package file and saves them to the destination stack directory. @@ -740,7 +741,7 @@ const mapLocales = ({ masterLocale, locale, locales }: any) => { * * @throws Will log errors encountered during file reading, processing, or writing of entries. */ -const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise => { +const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string, project: any): Promise => { const srcFunc = 'createEntry'; try { const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME); @@ -749,7 +750,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje const data = await fs.promises.readFile(packagePath, "utf8"); const entries = JSON.parse(data)?.entries; const content = JSON.parse(data)?.contentTypes; - + const LocaleMapper = { masterLocale: project?.master_locale ?? LOCALE_MAPPER?.masterLocale, ...project?.locales ?? {} }; if (entries && entries.length > 0) { const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? []; const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME); @@ -799,13 +800,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje }); const pathName = getDisplayName(name, displayField); locales.forEach((locale) => { - const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER }); + const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LocaleMapper }); const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId) .filter((env: any) => env?.name === environment_id) ?.map((env: any) => ({ environment: env?.uid, version: 1, - locale: locale?.toLowerCase?.(), + locale: localeCode, })) : []; const title = fields?.[pathName]?.[locale] || ""; const urlTitle = title @@ -816,7 +817,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje title: title?.trim?.() === "" ? (urlTitle || id) : title, uid: id, url: `/${name?.toLowerCase?.()}/${urlTitle}`, - locale: locale?.toLowerCase?.(), + locale: localeCode, publish_details: publishDetails, }; // Format object keys to snake_case @@ -843,11 +844,12 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje for await (const [localeKey, localeValues] of Object.entries( values as { [key: string]: any } )) { + const localeCode = mapLocales({ masterLocale: master_locale, locale: localeKey, locales: LocaleMapper }); const chunks = makeChunks(localeValues); for (const [entryKey, entryValue] of Object.entries(localeValues)) { const message = getLogMessage( srcFunc, - `Entry title "${(entryValue as { title: string })?.title}"(${ctName}) in the ${localeKey} locale has been successfully transformed.`, + `Entry title "${(entryValue as { title: string })?.title}"(${ctName}) in the ${localeCode} locale has been successfully transformed.`, {} ); await customLogger(projectId, destination_stack_id, "info", message); @@ -856,7 +858,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje let chunkIndex = 1; const filePath = path.join( entriesSave, - ctName, localeKey.toLowerCase() + ctName, localeCode ); for await (const [chunkId, chunkData] of Object.entries(chunks)) { refs[chunkIndex++] = `${chunkId}-entries.json`; @@ -874,7 +876,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje await customLogger(projectId, destination_stack_id, 'info', message); } } catch (err) { - console.info("🚀 ~ createEntry ~ err:", err) + console.error("🚀 ~ createEntry ~ err:", err) const message = getLogMessage( srcFunc, `Error encountered while creating entries.`, @@ -885,6 +887,10 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje } }; +function getKeyByValue(obj: Record, targetValue: string): string | undefined { + return Object.entries(obj).find(([_, value]) => value === targetValue)?.[0]; +} + /** * Processes and creates locale configurations from a given package file and saves them to the destination stack directory. * @@ -909,7 +915,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje * * @throws Will log errors encountered during file reading, processing, or writing of locale configurations. */ -const createLocale = async (packagePath: string, destination_stack_id: string, projectId: string) => { +const createLocale = async (packagePath: string, destination_stack_id: string, projectId: string, project: any) => { const srcFunc = 'createLocale'; const localeSave = path.join(DATA, destination_stack_id, LOCALE_DIR_NAME); const globalFieldSave = path.join(DATA, destination_stack_id, GLOBAL_FIELDS_DIR_NAME); @@ -933,13 +939,14 @@ const createLocale = async (packagePath: string, destination_stack_id: string, p ) await customLogger(projectId, destination_stack_id, 'error', message); } - - await Promise.all(locales.map(async (localeData: any) => { - const title = localeData.sys.id; + const fallbackMapLocales: any = { ...project?.master_locale ?? {}, ...project?.locales ?? {} } + await Promise?.all(locales?.map?.(async (localeData: any) => { + const currentMapLocale = getKeyByValue?.(fallbackMapLocales, localeData?.code) ?? `${localeData.code.toLowerCase()}`; + const title = localeData?.sys?.id; const newLocale: Locale = { - code: `${localeData.code.toLowerCase()}`, - name: localeCodes?.[localeData.code.toLowerCase()] || "English - United States", - fallback_locale: "", + code: currentMapLocale, + name: localeCodes?.[currentMapLocale] || "English - United States", + fallback_locale: getKeyByValue(fallbackMapLocales, localeData?.fallbackCode) ?? '', uid: `${title}`, }; @@ -947,16 +954,15 @@ const createLocale = async (packagePath: string, destination_stack_id: string, p msLocale[title] = newLocale; const message = getLogMessage( srcFunc, - `Master Locale ${newLocale.code} has been successfully transformed.`, + `Master Locale ${newLocale?.code} has been successfully transformed.`, {} ) await customLogger(projectId, destination_stack_id, 'info', message); } else { - newLocale.name = `${localeData.name}`; allLocales[title] = newLocale; const message = getLogMessage( srcFunc, - `Locale ${newLocale.code} has been successfully transformed.`, + `Locale ${newLocale?.code} has been successfully transformed.`, {} ) await customLogger(projectId, destination_stack_id, 'info', message); diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index bc11378d1..726f650d2 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -254,12 +254,12 @@ const startTestMigration = async (req: Request): Promise => { break; } case CMS.CONTENTFUL: { - await contentfulService?.createLocale(file_path, project?.current_test_stack_id, projectId); + await contentfulService?.createLocale(file_path, project?.current_test_stack_id, projectId, project); await contentfulService?.createRefrence(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createWebhooks(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.current_test_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.current_test_stack_id, projectId, true); - await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); + await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project); await contentfulService?.createVersionFile(project?.current_test_stack_id, projectId); break; } @@ -326,18 +326,16 @@ const startMigration = async (req: Request): Promise => { await wordpressService?.extractPosts(packagePath, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale) await wordpressService?.extractGlobalFields(project?.destination_stack_id, projectId) await wordpressService?.createVersionFile(project?.destination_stack_id, projectId); - - } break; } case CMS.CONTENTFUL: { - await contentfulService?.createLocale(file_path, project?.destination_stack_id, projectId); + await contentfulService?.createLocale(file_path, project?.destination_stack_id, projectId, project); await contentfulService?.createRefrence(file_path, project?.destination_stack_id, projectId); await contentfulService?.createWebhooks(file_path, project?.destination_stack_id, projectId); await contentfulService?.createEnvironment(file_path, project?.destination_stack_id, projectId); await contentfulService?.createAssets(file_path, project?.destination_stack_id, projectId); - await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale); + await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale, project); await contentfulService?.createVersionFile(project?.destination_stack_id, projectId); break; } @@ -478,8 +476,8 @@ export const updateLocaleMapper = async (req: Request) => { console.error(`project.json not found at ${projectFilePath}`); throw new Error(`project.json not found.`); } - // Find the project with the specified projectId + await ProjectModelLowdb.read(); const project: any = ProjectModelLowdb?.chain?.get?.("projects")?.find?.({ id: projectId })?.value(); if (project) { const index = ProjectModelLowdb?.chain?.get("projects")?.findIndex?.({ id: projectId })?.value(); diff --git a/upload-api/migration-contentful/libs/contentTypeMapper.js b/upload-api/migration-contentful/libs/contentTypeMapper.js index c83982788..42966b228 100644 --- a/upload-api/migration-contentful/libs/contentTypeMapper.js +++ b/upload-api/migration-contentful/libs/contentTypeMapper.js @@ -129,12 +129,18 @@ const createFieldObject = (item, contentstackFieldType, backupFieldType, referen */ const createDropdownOrRadioFieldObject = (item, fieldType) => { let choices = []; - if (!item?.validations?.length) { - choices.push({ value: 'value', key: 'key' }); + if (item?.items?.validations?.length) { + item?.items?.validations?.forEach?.((valid) => { + valid.in?.forEach((value) => choices.push({ value: ["Symbol", "Text", "Array"].includes(item?.items?.type) ? `${value}` : value, key: `${value}` })); + }) } else { - item.validations.forEach((valid) => { - valid.in?.forEach((value) => choices.push({ value: ["Symbol", "Text", "Array"].includes(item.type) ? `${value}` : value, key: `${value}` })); - }); + if (!item?.validations?.length) { + choices.push({ value: 'value', key: 'key' }); + } else { + item.validations.forEach((valid) => { + valid.in?.forEach((value) => choices.push({ value: ["Symbol", "Text", "Array"].includes(item.type) ? `${value}` : value, key: `${value}` })); + }); + } } return { ...createFieldObject(item, fieldType, fieldType), @@ -336,6 +342,7 @@ const contentTypeMapper = (data) => { break; } } + break; case 'Boolean': acc.push(createFieldObject(item, 'boolean', 'boolean')); diff --git a/upload-api/migration-contentful/libs/extractLocale.js b/upload-api/migration-contentful/libs/extractLocale.js index 7f702694d..65c4b84f4 100644 --- a/upload-api/migration-contentful/libs/extractLocale.js +++ b/upload-api/migration-contentful/libs/extractLocale.js @@ -21,7 +21,7 @@ const extractLocale = async (jsonFilePath) => { if (Array?.isArray?.(jsonData?.locales)) { jsonData?.locales?.forEach?.(locale => { if (locale?.code) { - uniqueLanguages.add(locale?.code?.toLowerCase?.()); // Normalize to lowercase + uniqueLanguages.add(locale?.code); // Normalize to lowercase } }); } diff --git a/upload-api/migration-sitecore/libs/convert.js b/upload-api/migration-sitecore/libs/convert.js index a5a780afb..c5555e179 100644 --- a/upload-api/migration-sitecore/libs/convert.js +++ b/upload-api/migration-sitecore/libs/convert.js @@ -12,7 +12,7 @@ const read = require("fs-readdir-recursive"); const helper = require("../utils/helper"); const { MIGRATION_DATA_CONFIG } = require("../constants/index"); const config = { - "data": "./"+MIGRATION_DATA_CONFIG.DATA, + "data": "./" + MIGRATION_DATA_CONFIG.DATA, "backup": "./backupMigrationData", "xml_filename": "", "sitecore_folder": "", @@ -30,12 +30,21 @@ function ExtractFiles(sitecore_folder) { for (let i = 0; i < xml_folder.length; i++) { if (xml_folder?.[i]?.endsWith?.("xml")) { const xml_data = path?.join?.(sitecore_folder, xml_folder?.[i]); - const json_data = xml_data.replace('xml', ''); - parseString(helper.readXMLFile(xml_data), { explicitArray: false }, function (err, result) { + const jsonFilePath = xml_data?.replace?.('xml', ''); + parseString(helper?.readXMLFile?.(xml_data), { explicitArray: false }, function (err, result) { if (err) { console.error("failed to parse xml: ", err); } else { - const filePath = path.join(json_data, config?.json_filename); + const filePath = path.join(jsonFilePath, config?.json_filename); + try { + const jsonFileArray = read?.(jsonFilePath)?.filter?.((fileExt) => fileExt?.includes?.('.json')) ?? []; + for (const ext of jsonFileArray) { + const absolutePath = path?.resolve?.(path?.join?.(jsonFilePath, ext)); + fs?.unlinkSync?.(absolutePath); + } + } catch (error) { + console.error("Error deleting file:", error); + } fs.writeFileSync(filePath, JSON.stringify(result, null, 4), "utf8"); } }) diff --git a/upload-api/migration-sitecore/libs/extractLocales.js b/upload-api/migration-sitecore/libs/extractLocales.js index 539634fff..ee34f3b0b 100644 --- a/upload-api/migration-sitecore/libs/extractLocales.js +++ b/upload-api/migration-sitecore/libs/extractLocales.js @@ -13,7 +13,7 @@ const extractLocales = (dir) => { if (item.isDirectory()) { extractLocales?.(fullPath); // Proper recursion - } else if (item?.isFile() && item?.name === "data.json.json") { + } else if (item?.isFile() && item?.name === "data.json") { try { const rawData = fs?.readFileSync?.(fullPath, "utf8"); const jsonData = JSON?.parse?.(rawData); diff --git a/upload-api/src/controllers/sitecore/index.ts b/upload-api/src/controllers/sitecore/index.ts index db27a298b..bb5369458 100644 --- a/upload-api/src/controllers/sitecore/index.ts +++ b/upload-api/src/controllers/sitecore/index.ts @@ -17,12 +17,8 @@ const { const createSitecoreMapper = async (filePath: string = "", projectId: string | string[], app_token: string | string[], affix: string | string[], config: object) => { try { const newPath = path.join(filePath, 'items'); - - const localeData = await extractLocales(path.join(filePath, 'items','master','sitecore','content')); - console.log("Fetched Locales: ", localeData); - - await ExtractFiles(newPath); + const localeData = await extractLocales(path.join(filePath, 'items', 'master', 'sitecore', 'content')); await ExtractConfiguration(newPath); await contentTypes(newPath, affix, config); const infoMap = await reference(); @@ -61,7 +57,7 @@ const createSitecoreMapper = async (filePath: string = "", projectId: string | s message: HTTP_TEXTS?.MAPPER_SAVED, }); } - + const mapperConfig = { method: 'post', maxBodyLength: Infinity, @@ -71,12 +67,12 @@ const createSitecoreMapper = async (filePath: string = "", projectId: string | s 'Content-Type': 'application/json' }, data: { - locale:Array.from(localeData) + locale: Array.from(localeData) }, }; const mapRes = await axios.request(mapperConfig) - if(mapRes?.status==200){ + if (mapRes?.status == 200) { logger.info('Legacy CMS', { status: HTTP_CODES?.OK, message: HTTP_TEXTS?.LOCALE_SAVED, diff --git a/upload-api/src/validators/sitecore/index.ts b/upload-api/src/validators/sitecore/index.ts index 551c50e6f..80b18c069 100644 --- a/upload-api/src/validators/sitecore/index.ts +++ b/upload-api/src/validators/sitecore/index.ts @@ -39,7 +39,7 @@ async function sitecoreValidator({ data }: props) { blob = await Promise.all(blob); mediaLibrary = await Promise.all(mediaLibrary); - if (templates?.length > 0 && content?.length > 0 && blob?.length > 0 && mediaLibrary?.length > 0) { + if (templates?.length > 0 && content?.length > 0 && blob?.length > 0 && mediaLibrary?.length > 0) { return true; } return false;