refactor:added pre-processing for xmldata and site_url for asset if incomplete asset url

Author: AishDani

api/src/services/wordpress.service.ts

@@ -237,9 +237,17 @@ async function startingDirAssests(destinationStackId: string) {
   }
 }
 
-async function saveAsset(assets: any, retryCount: number, affix: string, destinationStackId: string, projectId: string) {
+function toCheckUrl(url : string, baseSiteUrl: string) {
+
+  const validPattern = /^(https?:\/\/|www\.)/;
+  return validPattern.test(url)
+    ? url
+    : `${baseSiteUrl}${url.replace(/^\/+/, "")}`;
+}
+
+async function saveAsset(assets: any, retryCount: number, affix: string, destinationStackId: string, projectId: string, baseSiteUrl:string) {
   const srcFunc = 'saveAsset';
-  const url = encodeURI(assets["wp:attachment_url"]);
+  const url = encodeURI(toCheckUrl(assets["wp:attachment_url"],baseSiteUrl));
   const name = url.split("/").pop() || "";
 
   let description =
@@ -355,7 +363,7 @@ async function saveAsset(assets: any, retryCount: number, affix: string, destina
    await writeFileAsync(failedJSONFilePath, failedJSON, 4);
 
     if (retryCount === 0) {
-      return saveAsset(assets, 1, affix, destinationStackId, projectId);
+      return saveAsset(assets, 1, affix, destinationStackId, projectId, baseSiteUrl);
     } else {
       const message = getLogMessage(
         srcFunc,
@@ -369,7 +377,7 @@ async function saveAsset(assets: any, retryCount: number, affix: string, destina
   }
 }
 
-async function getAsset(attachments: any[], affix: string, destinationStackId: string, projectId: string) {
+async function getAsset(attachments: any[], affix: string, destinationStackId: string, projectId: string, baseSiteUrl:string) {
   const BATCH_SIZE = 5; // 5 promises at a time
   const results = [];
 
@@ -378,7 +386,7 @@ async function getAsset(attachments: any[], affix: string, destinationStackId: s
 
     const batchResults = await Promise.allSettled(
       batch.map((data) => {
-        saveAsset(data, 0, affix, destinationStackId, projectId)
+        saveAsset(data, 0, affix, destinationStackId, projectId, baseSiteUrl)
       })
     );
     results.push(...batchResults);
@@ -402,6 +410,9 @@ async function getAllAssets(
     await startingDirAssests(destinationStackId);
     const alldata: any = await fs.promises.readFile(packagePath, "utf8");
     const alldataParsed = JSON.parse(alldata);
+    const baseSiteUrl =
+    alldataParsed?.rss?.channel?.["wp:base_site_url"] ||
+    alldataParsed?.channel?.["wp:base_site_url"];
     const assets: Asset[] =
       alldataParsed?.rss?.channel?.item ?? alldataParsed?.channel?.item;
     // await writeFileAsync(path.join(assetsSave, MIGRATION_DATA_CONFIG.ASSETS_FILE_NAME), assets, 4);
@@ -420,7 +431,7 @@ async function getAllAssets(
       ({ "wp:post_type": postType }) => postType === "attachment"
     );
     if (attachments.length > 0) {
-      await getAsset(attachments, affix, destinationStackId, projectId);
+      await getAsset(attachments, affix, destinationStackId, projectId,baseSiteUrl);
     }
     return;
   } catch (error) {

upload-api/src/helper/index.ts

@@ -79,15 +79,30 @@ const saveJson = async (jsonContent: string, fileName: string) => {
   }
 };
 
+const cleanXml = (xml: string): string => {
+  return xml
+    .replace(/<!--[\s\S]*?-->/g, '') 
+    .replace(/<!DOCTYPE[^>]*>/g, '') 
+    .trim();
+};
+
+
+
 // parse xml to json
 const parseXmlToJson = async (xml: any) => {
   try {
+   
+    const xmldata = cleanXml(xml)
+
     const parser = new xml2js.Parser({
       attrkey: "attributes",
       charkey: "text",
       explicitArray: false,
+      trim: true,  
+      normalize: true, 
+      normalizeTags: true, 
     });
-    const data = await parser.parseStringPromise(xml);
+    const data = await parser.parseStringPromise(xmldata);
     return data
   } catch (err) {
     console.error(err);

upload-api/src/services/fileProcessing.ts

@@ -4,7 +4,7 @@ import JSZip from 'jszip';
 import validator from '../validators';
 import config from '../config/index';
 import logger from '../utils/logger.js';
-
+import * as Cheerio from 'cheerio';
 
 const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: string, name :string) => {
   if (fileExt === 'zip') {
@@ -36,7 +36,9 @@ const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: st
     }
   } else if (fileExt === 'xml') {
     if (await validator({ data: zipBuffer, type: cmsType, extension: fileExt }) ) {
-      const parsedJson = await parseXmlToJson(zipBuffer);
+      const $ = Cheerio.load(zipBuffer, { xmlMode: true });
+      const fixedXml = $.xml(); 
+      const parsedJson = await parseXmlToJson(fixedXml);
       const isSaved = await saveJson(parsedJson,"data.json");
       if (isSaved) {
         logger.info('Validation success:', {