Merge pull request #504 from contentstack/dev

Merge pull request #503 from contentstack/feature/CMG-485-entries

fixed test migartion

Author: RohitKini

.github/workflows/repo-sync.yml

@@ -10,7 +10,7 @@ import { JSDOM } from "jsdom";
 import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer';
 
 
-import { CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js";
+import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js";
 import { Locale } from "../models/types.js";
 import jsonRTE from "./contentful/jsonRTE.js";
 import { getAllLocales, getLogMessage } from "../utils/index.js";
@@ -234,7 +234,7 @@ const processField = (
         return assetsData;
       } else {
         const id = lang_value?.sys?.id;
-        if (id in assetId) {
+        if (assetId && id in assetId) {
           return assetId?.[id]
         }
         return null;
@@ -253,7 +253,7 @@ const processField = (
         return refs;
       }
       const id = lang_value?.sys?.id;
-      if (id in entryId) return [[entryId?.[id]]];
+      if (entryId && id in entryId) return [[entryId?.[id]]];
       return null;
     }
 
@@ -333,6 +333,7 @@ const processField = (
   }
 };
 
+
 // Helper function to clean up brackets in non-numeric lang_value
 const cleanBrackets = (lang_value: any) => {
   const myJSON = JSON.stringify(lang_value);
@@ -436,11 +437,11 @@ const saveAsset = async (
     const publishDetails: { environment: any; version: number; locale: any }[] =
       [];
     const assetsSave = path.join(DATA, destination_stack_id, ASSETS_DIR_NAME);
-    const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME);
+    const environmentsId = await readFile(path.join(DATA, destination_stack_id, ENVIRONMENTS_DIR_NAME), ENVIRONMENTS_FILE_NAME) ?? {};
     const localeId = await readFile(path.join(DATA, destination_stack_id, LOCALE_DIR_NAME), LOCALE_CF_LANGUAGE);
 
     if (assets.fields.file && assets.fields.title) {
-      Object.values(environmentsId).forEach((env: any) => {
+      Object?.values?.(environmentsId)?.length && Object.values(environmentsId).forEach((env: any) => {
         if (env?.name === assets?.sys?.environment?.sys?.id) {
           Object.values(localeId).forEach((locale: any) => {
             publishDetails.push({
@@ -696,6 +697,18 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string,
   }
 };
 
+const mapLocales = ({ masterLocale, locale, locales }: any) => {
+  if (locales?.masterLocale?.[masterLocale ?? ''] === locale) {
+    return Object?.keys(locales?.masterLocale)?.[0]
+  }
+  for (const [key, value] of Object?.entries?.(locales) ?? {}) {
+    if (typeof value !== 'object' && value === locale) {
+      return key;
+    }
+  }
+  return locale.toLowerCase();
+}
+
 /**
  * Creates and processes entries from a given package file and saves them to the destination stack directory.
  *
@@ -722,7 +735,8 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string,
  *
  * @throws Will log errors encountered during file reading, processing, or writing of entries.
  */
-const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any): Promise<void> => {
+const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise<void> => {
+  console.info("🚀 ~ createEntry ~ master_locale:", master_locale)
   const srcFunc = 'createEntry';
   try {
     const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME);
@@ -735,7 +749,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
     if (entries && entries.length > 0) {
       const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? [];
       const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME);
-      const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME);
+      const environmentsId = await readFile(environmentSave, ENVIRONMENTS_FILE_NAME) ?? {};
       const displayField: { [key: string]: any } = {}
       content.map((item: any) => {
         displayField[item.name.toLowerCase().replace(/[^a-z0-9]+/g, "_")] =
@@ -781,13 +795,15 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
             });
             const pathName = getDisplayName(name, displayField);
             locales.forEach((locale) => {
-              const publishDetails = Object?.values?.(environmentsId)
+              const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER });
+              console.info("🚀 ~ locales.forEach ~ localeCode:", localeCode);
+              const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId)
                 .filter((env: any) => env?.name === environment_id)
                 ?.map((env: any) => ({
                   environment: env?.uid,
                   version: 1,
-                  locale: locale.toLowerCase(),
-                }));
+                  locale: locale?.toLowerCase(),
+                })) : [];
 
               const title = entryData[name][locale][id][pathName] || "";
               const urlTitle = title
@@ -814,21 +830,18 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
               );
             });
           });
-
           return entryData;
         },
         {}
       );
-      const writePromises = [];
-
-      for (const [newKey, values] of Object.entries(result)) {
+      for await (const [newKey, values] of Object.entries(result)) {
         const currentCT = contentTypes?.find((ct: any) => ct?.otherCmsUid === newKey);
         const ctName = currentCT?.contentstackUid in mapperKeys ?
           mapperKeys?.[currentCT?.contentstackUid] : (currentCT?.contentstackUid ?? newKey.replace(/([A-Z])/g, "_$1").toLowerCase());
-        for (const [localeKey, localeValues] of Object.entries(
+        for await (const [localeKey, localeValues] of Object.entries(
           values as { [key: string]: any }
         )) {
-          const chunks = await makeChunks(localeValues);
+          const chunks = makeChunks(localeValues);
           for (const [entryKey, entryValue] of Object.entries(localeValues)) {
             const message = getLogMessage(
               srcFunc,
@@ -843,14 +856,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
             entriesSave,
             ctName, localeKey.toLowerCase()
           );
-          for (const [chunkId, chunkData] of Object.entries(chunks)) {
+          for await (const [chunkId, chunkData] of Object.entries(chunks)) {
             refs[chunkIndex++] = `${chunkId}-entries.json`;
-            writePromises.push(writeFile(filePath, `${chunkId}-entries.json`, chunkData))
+            await writeFile(filePath, `${chunkId}-entries.json`, chunkData)
           }
-          writePromises.push(writeFile(filePath, ENTRIES_MASTER_FILE, refs));
+          await writeFile(filePath, ENTRIES_MASTER_FILE, refs);
         }
       }
-      await Promise.all(writePromises);
     } else {
       const message = getLogMessage(
         srcFunc,

api/src/services/contentful.service.ts

@@ -250,7 +250,7 @@ function parseBlockReference(obj: any, lang?: LangType, destination_stack_id?: S
 
 function parseInlineReference(obj: any, lang?: LangType, destination_stack_id?: StackId): any {
   const entryId: { [key: string]: any } = destination_stack_id && readFile(path.join(process.cwd(), DATA, destination_stack_id, RTE_REFERENCES_DIR_NAME, RTE_REFERENCES_FILE_NAME));
-  const entry = Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]);
+  const entry = entryId && Object.entries(entryId).find(([arrayKey, arrayValue]) => arrayKey === lang && arrayValue[obj.data.target.sys.id]);
 
   if (entry) {
     const [arrayKey, arrayValue] = entry;
@@ -314,6 +314,7 @@ function parseBlockAsset(obj: any, lang?: LangType, destination_stack_id?: Stack
   };
 }
 
+
 function parseBlockquote(obj: any): any {
   const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
   return {

api/src/services/contentful/jsonRTE.ts

@@ -25,6 +25,7 @@ import { extensionService } from "./extension.service.js";
 
 
 
+
 /**
  * Creates a test stack.
  *
@@ -259,7 +260,7 @@ const startTestMigration = async (req: Request): Promise<any> => {
         await contentfulService?.createWebhooks(file_path, project?.current_test_stack_id, projectId);
         await contentfulService?.createEnvironment(file_path, project?.current_test_stack_id, projectId);
         await contentfulService?.createAssets(file_path, project?.current_test_stack_id, projectId, true);
-        await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys);
+        await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale);
         await contentfulService?.createVersionFile(project?.current_test_stack_id, projectId);
         break;
       }
@@ -299,8 +300,8 @@ const startMigration = async (req: Request): Promise<any> => {
     await customLogger(projectId, project?.destination_stack_id, 'info', message);
     await setLogFilePath(loggerPath);
     const contentTypes = await fieldAttacher({ orgId, projectId, destinationStackId: project?.destination_stack_id, region, user_id });
-    await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.current_test_stack_id, region, userId: user_id });
-    await extensionService?.createExtension({ destinationStackId: project?.current_test_stack_id });
+    await marketPlaceAppService?.createAppManifest({ orgId, destinationStackId: project?.destination_stack_id, region, userId: user_id });
+    await extensionService?.createExtension({ destinationStackId: project?.destination_stack_id });
     switch (cms) {
       case CMS.SITECORE_V8:
       case CMS.SITECORE_V9:
@@ -337,7 +338,7 @@ const startMigration = async (req: Request): Promise<any> => {
         await contentfulService?.createWebhooks(file_path, project?.destination_stack_id, projectId);
         await contentfulService?.createEnvironment(file_path, project?.destination_stack_id, projectId);
         await contentfulService?.createAssets(file_path, project?.destination_stack_id, projectId);
-        await contentfulService?.createEntry(file_path, project?.current_test_stack_id, projectId, contentTypes, project?.mapperKeys);
+        await contentfulService?.createEntry(file_path, project?.destination_stack_id, projectId, contentTypes, project?.mapperKeys, project?.stackDetails?.master_locale);
         await contentfulService?.createVersionFile(project?.destination_stack_id, projectId);
         break;
       }

api/src/services/migration.service.ts

@@ -9,7 +9,6 @@ import { orgService } from './org.service.js';
 import { getLogMessage } from '../utils/index.js';
 import customLogger from '../utils/custom-logger.utils.js';
 
-
 const append = "a";
 
 const baseDirName = MIGRATION_DATA_CONFIG.DATA

api/src/services/sitecore.service.ts

@@ -10,11 +10,12 @@ import customLogger from "../utils/custom-logger.utils.js";
 import { getLogMessage } from "../utils/index.js";
 import { Advanced } from "../models/FieldMapper.js";
 
-
 const { JSDOM } = jsdom;
+
 const virtualConsole = new jsdom.VirtualConsole();
 // Get the current file's path
 const __filename = fileURLToPath(import.meta.url);
+
 // Get the current directory
 const __dirname = path.dirname(__filename);
 

api/src/services/wordpress.service.ts

@@ -748,7 +748,7 @@ export const contenTypeMaker = async ({ contentType, destinationStackId, project
   if (currentCt?.uid) {
     ct = await mergeTwoCts(ct, currentCt);
   }
-  if (ct?.uid) {
+  if (ct?.uid && ct?.schema?.length) {
     if (contentType?.type === 'global_field') {
       const globalSave = path.join(MIGRATION_DATA_CONFIG.DATA, destinationStackId, GLOBAL_FIELDS_DIR_NAME);
       const message = getLogMessage(srcFunc, `Global Field ${ct?.uid} has been successfully Transformed.`, {});

api/src/utils/content-type-creator.utils.ts

@@ -8,7 +8,6 @@ const {
   ENTRIES_DIR_NAME,
   ASSETS_DIR_NAME,
   ASSETS_SCHEMA_FILE,
-  CONTENT_TYPES_DIR_NAME,
   CONTENT_TYPES_SCHEMA_FILE,
   ENTRIES_MASTER_FILE,
   GLOBAL_FIELDS_DIR_NAME,
@@ -240,22 +239,24 @@ const sortGlobalField = async (baseDir: string, finalData: any) => {
   }
 }
 
-const sortContentType = async (baseDir: string, finalData: any) => {
-  const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME);
-  const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME);
-  const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8'));
-  await sortGlobalField(baseDir, finalData);
-  const contentTypes: any = [];
-  for await (const ct of finalData) {
-    const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType);
-    await lookForReference(findCtData, finalData);
-    contentTypes?.push(findCtData);
-  }
-  await deleteFolderAsync(contentTypePath);
-  for await (const ctItem of contentTypes) {
-    await saveContent(ctItem, contentSave);
-  }
-}
+//this code can be used in feature
+
+// const sortContentType = async (baseDir: string, finalData: any) => {
+//   const contentTypePath: string = path.join(process.cwd(), baseDir, CONTENT_TYPES_DIR_NAME);
+//   const contentSave = path.join(baseDir, CONTENT_TYPES_DIR_NAME);
+//   const ctData = await JSON.parse(await fs.promises.readFile(path.join(contentTypePath, CONTENT_TYPES_SCHEMA_FILE), 'utf8'));
+//   await sortGlobalField(baseDir, finalData);
+//   const contentTypes: any = [];
+//   for await (const ct of finalData) {
+//     const findCtData = ctData?.find((ele: any) => ele?.uid === ct?.contentType);
+//     await lookForReference(findCtData, finalData);
+//     contentTypes?.push(findCtData);
+//   }
+//   await deleteFolderAsync(contentTypePath);
+//   for await (const ctItem of contentTypes) {
+//     await saveContent(ctItem, contentSave);
+//   }
+// }
 
 
 
@@ -275,7 +276,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => {
         const pathParts = normalizedPath.split(path.sep);
         const ct = pathParts?.[0]; // First directory
         const locale = pathParts?.[1]; // Second directory
-        allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length, entryData, filePath, locale })
+        allData?.push({ contentType: ct, count: Object?.keys?.(entryData)?.length ?? 0, entryData, filePath, locale })
       }
     }
   }
@@ -292,7 +293,7 @@ export const testFolderCreator = async ({ destinationStackId }: any) => {
     finalData?.push({ contentType: et?.contentType, entryObj, locale: et?.locale });
   });
   await sortAssets(baseDir);
-  await sortContentType(baseDir, finalData);
+  // await sortContentType(baseDir, finalData);
   await deleteFolderAsync(entryDelete);
   for await (const entry of finalData) {
     const fileMeta = { "1": `${entry?.locale}.json` };