Merge branch 'dev' of https://github.com/contentstack/migration-v2-node-server into feature/content-mapper

Author: sayalijoshi27

api/src/models/FieldMapper.ts

@@ -28,6 +28,7 @@ interface FieldMapper {
   field_mapper: {
     id: string;
     projectId: string;
+    contentTypeId: string;
     uid: string;
     otherCmsField: string;
     otherCmsType: string;

api/src/services/contentMapper.service.ts

@@ -36,7 +36,21 @@ const putTestData = async (req: Request) => {
   const contentTypes = req.body.contentTypes;
 
   try {
-    await FieldMapperModel.read();
+
+     /*
+  this code snippet is iterating over an array called contentTypes and 
+  transforming each element by adding a unique identifier (id) if it doesn't already exist. 
+  The transformed elements are then stored in the contentType variable, 
+  and the generated id values are pushed into the contentIds array.
+  */
+  await ContentTypesMapperModelLowdb.read();
+  const contentIds: any[] = [];
+  const contentType = contentTypes.map((item: any) => {
+    const id = item?.id?.replace(/[{}]/g, "")?.toLowerCase() || uuidv4();
+    item.id = id;
+    contentIds.push(id);
+    return { ...item, id, projectId };
+  });
 
   /*
   this code snippet iterates over an array of contentTypes and performs 
@@ -47,36 +61,24 @@ const putTestData = async (req: Request) => {
   It then updates the field_mapper property of a data object using the FieldMapperModel.update() function. 
   Finally, it updates the fieldMapping property of each type in the contentTypes array with the fieldIds array.
   */
+  await FieldMapperModel.read();
+
   contentTypes.map((type: any, index: any) => {
     const fieldIds: string[] = [];
     const fields = type?.fieldMapping?.filter((field: any) => field)?.map?.((field: any) => {
       const id = field?.id ? field?.id?.replace(/[{}]/g, "")?.toLowerCase() : uuidv4();
       field.id = id;
       fieldIds.push(id);
-      return { id, projectId, isDeleted: false, ...field };
+      return { id, projectId, contentTypeId: type?.id, isDeleted: false, ...field };
     });
 
     FieldMapperModel.update((data: any) => {
       data.field_mapper = [...(data?.field_mapper ?? []), ...(fields ?? [])];
     });
-    contentTypes[index].fieldMapping = fieldIds;
+    contentType[index].fieldMapping = fieldIds;
   });
 
-  await ContentTypesMapperModelLowdb.read();
-  const contentIds: any[] = [];
 
-  /*
-  this code snippet is iterating over an array called contentTypes and 
-  transforming each element by adding a unique identifier (id) if it doesn't already exist. 
-  The transformed elements are then stored in the contentType variable, 
-  and the generated id values are pushed into the contentIds array.
-  */
-  const contentType = contentTypes.map((item: any) => {
-    const id = item?.id?.replace(/[{}]/g, "")?.toLowerCase() || uuidv4();
-    item.id = id;
-    contentIds.push(id);
-    return { ...item, id, projectId };
-  });
 
   await ContentTypesMapperModelLowdb.update((data: any) => {
     data.ContentTypesMappers = [
@@ -248,7 +250,7 @@ const getFieldMapping = async (req: Request) => {
     const fieldData = contentType?.fieldMapping?.map?.((fields: any) => {
       const fieldMapper = FieldMapperModel.chain
         .get("field_mapper")
-        .find({ id: fields, projectId: projectId })
+        .find({ id: fields, projectId: projectId, contentTypeId: contentTypeId })
         .value();
 
       return fieldMapper;
@@ -616,7 +618,7 @@ const updateContentType = async (req: Request) => {
       await FieldMapperModel.read();
       fieldMapping.forEach((field: any) => {
         const fieldIndex = FieldMapperModel.data.field_mapper.findIndex(
-          (f: any) => f?.id === field?.id
+          (f: any) => f?.id === field?.id && f?.contentTypeId === field?.contentTypeId
         );
         if (fieldIndex > -1 && field?.contentstackFieldType !== "") {
           FieldMapperModel.update((data: any) => {

api/src/services/contentful.service.ts

@@ -939,16 +939,16 @@ const createLocale = async (packagePath: string, destination_stack_id: string, p
     }
     const fallbackMapLocales: any = { ...project?.master_locale ?? {}, ...project?.locales ?? {} }
     await Promise?.all(locales?.map?.(async (localeData: any) => {
-      const currentMapLocale = getKeyByValue?.(fallbackMapLocales, localeData?.code) ?? `${localeData.code.toLowerCase()}`;
+      const currentMapLocale = getKeyByValue?.(fallbackMapLocales, localeData?.code) ?? `${localeData?.code?.toLowerCase?.()}`;
       const title = localeData?.sys?.id;
       const newLocale: Locale = {
         code: currentMapLocale,
         name: localeCodes?.[currentMapLocale] || "English - United States",
         fallback_locale: getKeyByValue(fallbackMapLocales, localeData?.fallbackCode) ?? '',
         uid: `${title}`,
       };
-
-      if (localeData.default === true) {
+      const masterLocaleCode = getKeyByValue(project?.master_locale, localeData?.code);
+      if (masterLocaleCode !== undefined) {
         msLocale[title] = newLocale;
         const message = getLogMessage(
           srcFunc,
@@ -957,16 +957,26 @@ const createLocale = async (packagePath: string, destination_stack_id: string, p
         )
         await customLogger(projectId, destination_stack_id, 'info', message);
       } else {
-        allLocales[title] = newLocale;
-        const message = getLogMessage(
-          srcFunc,
-          `Locale ${newLocale?.code} has been successfully transformed.`,
-          {}
-        )
-        await customLogger(projectId, destination_stack_id, 'info', message);
+        if (project?.locales?.[localeData?.code]) {
+          allLocales[title] = newLocale;
+          const message = getLogMessage(
+            srcFunc,
+            `Locale ${newLocale?.code} has been successfully transformed.`,
+            {}
+          )
+          await customLogger(projectId, destination_stack_id, 'info', message);
+        }
       }
       localeList[title] = newLocale;
     }));
+    const masterLocaleData = Object?.values(msLocale)?.[0];
+    if (masterLocaleData) {
+      for (const [key, value] of Object.entries(allLocales) ?? {}) {
+        if (value?.code === masterLocaleData?.fallback_locale) {
+          allLocales[key].fallback_locale = masterLocaleData?.code
+        }
+      }
+    }
     await writeFile(localeSave, LOCALE_FILE_NAME, allLocales)
     await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale)
     await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList)

api/src/services/wordpress.service.ts

@@ -5,7 +5,7 @@ import axios from "axios";
 //import _ from "lodash";
 import { MIGRATION_DATA_CONFIG } from "../constants/index.js";
 import jsdom from "jsdom";
-import { htmlToJson } from "@contentstack/json-rte-serializer";
+import { htmlToJson, jsonToHtml } from "@contentstack/json-rte-serializer";
 import customLogger from "../utils/custom-logger.utils.js";
 import { getLogMessage } from "../utils/index.js";
 import { Advanced } from "../models/FieldMapper.js";
@@ -108,20 +108,28 @@ let assetData: Record<string, any> | any = {};
 let blog_base_url = "";
 
 //helper function to convert entries with content type
-function mapContentTypeToEntry(contentType: any, data: any) {
-  return contentType?.fieldMapping?.reduce((acc: { [key: string]: any }, field: FieldMapping) => {
+async function mapContentTypeToEntry(contentType: any, data: any) {
+  const result: { [key: string]: any } = {};
+
+  for (const field of contentType?.fieldMapping || []) {
     const fieldValue = data?.[field?.uid];
     let formattedValue;
 
     switch (field?.contentstackFieldType) {
       case "single_line_text":
       case "text":
-      case 'html':
         formattedValue = fieldValue;
         break;
-      case "json":
-        formattedValue = convertHtmlToJson(fieldValue);
+      case "html":
+        formattedValue =
+          fieldValue && typeof fieldValue === "object"
+            ? await convertJsonToHtml(fieldValue)
+            : fieldValue;
         break;
+      case "json":
+        
+            formattedValue = await convertHtmlToJson(fieldValue);    
+    break;
       case "reference":
         formattedValue = getParent(data,data[field.uid]);
         break;
@@ -132,10 +140,12 @@ function mapContentTypeToEntry(contentType: any, data: any) {
       formattedValue = Array.isArray(formattedValue) ? formattedValue : [formattedValue];
     }
 
-    acc[field.contentstackFieldUid] = formattedValue;
-    return acc;
-  }, {});
+    result[field?.contentstackFieldUid] = formattedValue;
+  }
+
+  return result;
 }
+
 // helper functions
 async function writeFileAsync(filePath: string, data: any, tabSpaces: number) {
   filePath = path.resolve(filePath);
@@ -596,7 +606,7 @@ async function getAllreference(affix: string, packagePath: string, destinationSt
       ...processReferenceData(referenceTerms, "terms", "wp:term_slug", terms)
     );
     referenceArray.push(
-      ...processReferenceData(referenceTags, "tags", "wp:tag_slug", tag)
+      ...processReferenceData(referenceTags, "tag", "wp:tag_slug", tag)
     );
 
     if (referenceArray.length > 0) {
@@ -776,7 +786,7 @@ async function saveAuthors(authorDetails: any[], destinationStackId: string, pro
       authordata[customId] = {
         ...authordata[customId],
         uid: customId,
-        ...mapContentTypeToEntry(contentType, authordataEntry),
+        ...( await mapContentTypeToEntry(contentType, authordataEntry)),
       };
       authordata[customId].publish_details = [];
 
@@ -1473,7 +1483,7 @@ async function saveTerms(termsDetails: any[], destinationStackId: string, projec
       `${master_locale}.json`
     );
     const termsdata = termsDetails.reduce(
-      (acc: { [key: string]: any }, data) => {
+      async (acc: { [key: string]: any }, data) => {
 
         const { id } = data;
          const uid = `terms_${id}`;
@@ -1485,7 +1495,7 @@ async function saveTerms(termsDetails: any[], destinationStackId: string, projec
         acc[customId] = {
           ...acc[customId],
           uid: customId,
-          ...mapContentTypeToEntry(contentType, data), // Pass individual term object
+          ...(await mapContentTypeToEntry(contentType, data)), // Pass individual term object
         };
         acc[customId].publish_details = [];
         return acc;
@@ -1598,7 +1608,7 @@ async function saveTags(tagDetails: any[], destinationStackId: string, projectId
       tagsFolderPath,
       `${master_locale}.json`
     );
-    const tagdata = tagDetails.reduce((acc: { [key: string]: any }, data) => {
+    const tagdata = tagDetails.reduce(async (acc: { [key: string]: any }, data) => {
       const { id } = data;
       const uid = `tags_${id}`;
       //const title = `Tags - ${id}`;
@@ -1609,7 +1619,7 @@ async function saveTags(tagDetails: any[], destinationStackId: string, projectId
       acc[customId]={
         ...acc[customId],
         uid:customId,
-        ...mapContentTypeToEntry(contenttype,data),
+        ...( await mapContentTypeToEntry(contenttype,data)),
       };
       acc[customId].publish_details = [];
 
@@ -1715,11 +1725,22 @@ async function startingDirCategories(affix: string, ct: string, master_locale:st
 }
 
 const convertHtmlToJson = (htmlString: any) => {
-  const dom = new JSDOM(htmlString?.replace(/&amp;/g, "&"));
-  const htmlDoc = dom.window.document.querySelector("body");
-  return htmlToJson(htmlDoc);
+  if(typeof htmlString === 'string'){
+    const dom =  new JSDOM(htmlString?.replace(/&amp;/g, "&"));
+    const htmlDoc = dom.window.document.querySelector("body");
+    const jsonValue = htmlToJson(htmlDoc)
+    return jsonValue;
+
+  }
+  else return htmlString;
 };
 
+const convertJsonToHtml = async (json: any) => {
+  const htmlValue = await jsonToHtml(json);
+  return htmlValue;
+
+}
+
 function getParent(data: any,id: string) {
   const parentId: any = fs.readFileSync(
     path.join(referencesFolder, MIGRATION_DATA_CONFIG.REFERENCES_FILE_NAME),
@@ -1745,7 +1766,7 @@ async function saveCategories(categoryDetails: any[], destinationStackId:string,
   const srcFunc = 'saveCategories';
   try {
     const categorydata = categoryDetails.reduce(
-      (acc: { [key: string]: any }, data) => {
+      async (acc: { [key: string]: any }, data) => {
         const uid = `category_${data["id"]}`;
 
         const customId = uid
@@ -1754,7 +1775,7 @@ async function saveCategories(categoryDetails: any[], destinationStackId:string,
         acc[customId]={
           ...acc[customId],
           uid:customId,
-          ...mapContentTypeToEntry(contenttype,data),
+          ...(await mapContentTypeToEntry(contenttype,data)),
         }
         acc[customId].publish_details = [];
 
@@ -2057,23 +2078,18 @@ async function processChunkData(
             data,
             postdata
           );
-          const formattedPosts = Object?.entries(formatted_posts)?.reduce(
-            (acc: { [key: string]: any }, data:any) => {
-             
-              const customId = idCorrector(data["uid"])
-      
-              // Accumulate category data
-              acc[customId]={
-                ...acc[customId],
-                uid: customId,
-                ...mapContentTypeToEntry(contenttype,data),
-              };
-              acc[customId].publish_details = [];
-      
-              return acc;
-            },
-            {}
-          );
+          const formattedPosts: any = {};
+         
+          for (const [key, value] of Object.entries(formatted_posts as {[key: string]: any})) {
+            const customId = idCorrector(value?.uid);
+    
+            formattedPosts[customId] = {
+              ...formattedPosts[customId],
+              uid: customId,
+              ...(await mapContentTypeToEntry(contenttype, value)),
+            };
+            formattedPosts[customId].publish_details = [];
+          }
           Object.assign(postdata,formattedPosts);
 
           // await writeFileAsync(
@@ -2095,6 +2111,7 @@ async function processChunkData(
     if (isLastChunk && allSuccess) {
       console.info("last data");
     }
+    console.info("postData ---> ", postdata)
     return postdata
   } catch (error) {
     console.error(error);
@@ -2105,8 +2122,8 @@ async function processChunkData(
 
 async function extractPosts( packagePath: string, destinationStackId: string, projectId: string,contentTypes:any, keyMapper:any, master_locale: string) {
   const srcFunc = "extractPosts";
-  const ct:any = keyMapper?.["categories"];
-  const contenttype = contentTypes?.find((item:any)=> item?.otherCmsUid === 'categories');
+  const ct:any = keyMapper?.["posts"];
+  const contenttype = contentTypes?.find((item:any)=> item?.otherCmsUid === 'posts');
 
   try {
     await startingDirPosts(ct, master_locale);

ui/src/components/AdvancePropertise/index.tsx

@@ -576,7 +576,7 @@ const AdvancePropertise = (props: SchemaProps) => {
                       isMulti={true}
                       onChange={(selectedOptions: ContentTypeOption[]) => {
                         setCTValue(selectedOptions);
-                        const embedObject = selectedOptions.map((item: optionsType) => item.label); // Update the state with the selected options
+                        const embedObject = selectedOptions.map((item: optionsType) => item?.value); // Update the state with the selected options
                         props?.updateFieldSettings(
                           props?.rowId,
                           {

ui/src/components/ContentMapper/index.tsx

@@ -671,6 +671,7 @@ const ContentMapper = forwardRef(({handleStepChange}: contentMapperProps, ref: R
       const validTableData = data?.fieldMapping?.filter((field: FieldMapType) => field?.otherCmsType !== undefined);
 
       setTableData(validTableData || []);
+      setSelectedEntries(validTableData);
       setTotalCounts(validTableData?.length);
       setInitialRowSelectedData(validTableData?.filter((item: FieldMapType) => !item?.isDeleted))
       setIsLoading(false);