Merge branch 'dev' of github.com:contentstack/migration-v2-node-server into bugfix/cmg-616

Author: AishDani

.talismanrc

@@ -1,7 +1,13 @@
-fileignoreconfig:
-- filename: .github/workflows/secrets-scan.yml
-  ignore_detectors:
-    - filecontent
-- filename: remove-broken-imports.js
-  checksum: d9d3ca95b2f4df855c8811c73b5714e80b31e5e84b46affa0cb514dcfcc145bf
+ignoreconfig:
+  - filename: .github/workflows/secrets-scan.yml
+    ignore_detectors:
+      - filecontent
+
+  - filename: remove-broken-imports.js
+    checksum: d9d3ca95b2f4df855c8811c73b5714e80b31e5e84b46affa0cb514dcfcc145bf
+
+  - filename: ui/package-lock.json
+    ignore_detectors:
+      - Base64Detector
+
 version: "1.0"

api/package.json

@@ -48,7 +48,7 @@
     "p-limit": "^6.2.0",
     "path-to-regexp": "^8.2.0",
     "router": "^2.0.0",
-    "shelljs": "^0.8.5",
+    "shelljs": "^0.9.0",
     "socket.io": "^4.7.5",
     "uuid": "^9.0.1",
     "winston": "^3.11.0"

api/src/constants/index.ts

@@ -177,7 +177,7 @@ export const LOCALE_MAPPER: any = {
   masterLocale: {
     "en-us": "en",
   },
-  locales:{fr: "fr-fr",}
+  locales: { fr: "fr-fr", }
 };
 export const CHUNK_SIZE = 1048576;
 
@@ -272,3 +272,13 @@ export const MIGRATION_DATA_CONFIG = {
 
   EXPORT_INFO_FILE: "export-info.json",
 };
+
+export const GET_AUDIT_DATA = {
+  MIGRATION: "migration-v2",
+  API_DIR: "api",
+  MIGRATION_DATA_DIR: "migration-data",
+  LOGS_DIR: "logs",
+  AUDIT_DIR: "audit",
+  AUDIT_REPORT: "audit-report",
+  FILTERALL: "all",
+}

api/src/controllers/migration.controller.ts

@@ -49,10 +49,13 @@ const deleteTestStack = async (req: Request, res: Response): Promise<void> => {
   const resp = await migrationService.deleteTestStack(req);
   res.status(200).json(resp);
 };
-
+const getAuditData = async (req: Request, res: Response): Promise<void> => {
+  const resp = await migrationService.getAuditData(req);
+  res.status(resp?.status).json(resp);
+};
 const getLogs = async (req: Request, res: Response): Promise<void> => {
   const resp = await migrationService.getLogs(req);
-  res.status(200).json(resp);
+  res.status(resp?.status).json(resp);
 };
 
 const saveLocales = async (req: Request, res: Response): Promise<void> => {
@@ -72,5 +75,6 @@ export const migrationController = {
   startMigration,
   getLogs,
   saveLocales,
-  saveMappedLocales
+  saveMappedLocales,
+  getAuditData
 };

api/src/routes/migration.routes.ts

@@ -20,7 +20,10 @@ router.post(
   "/test-stack/:orgId/:projectId",
   asyncRouter(migrationController.startTestMigration)
 );
-
+router.get(
+  "/get_audit_data/:orgId/:projectId/:stackId/:moduleName/:skip/:limit/:startIndex/:stopIndex/:searchText/:filter",
+  asyncRouter(migrationController.getAuditData)
+)
 /**
  * Route for deleting a test stack.
  * @route POST /test-stack/:projectId

api/src/services/migration.service.ts

@@ -7,6 +7,7 @@ import https from "../utils/https.utils.js";
 import { LoginServiceType } from "../models/types.js";
 import getAuthtoken from "../utils/auth.utils.js";
 import logger from "../utils/logger.js";
+import { GET_AUDIT_DATA } from "../constants/index.js";
 import {
   HTTP_TEXTS,
   HTTP_CODES,
@@ -118,12 +119,10 @@ const createTestStack = async (req: Request): Promise<LoginServiceType> => {
     }
     return {
       data: {
-        data: res.data,
-        url: `${
-          config.CS_URL[token_payload?.region as keyof typeof config.CS_URL]
-        }/stack/${res.data.stack.api_key}/dashboard`,
+        data: res?.data,
+        url: `${config?.CS_URL[token_payload?.region as keyof typeof config.CS_URL]}/stack/${res?.data?.stack?.api_key}/dashboard`,
       },
-      status: res.status,
+      status: res?.status,
     };
   } catch (error: any) {
     logger.error(
@@ -135,13 +134,158 @@ const createTestStack = async (req: Request): Promise<LoginServiceType> => {
       )
     );
 
+    throw new ExceptionFunction(
+      error?.message || HTTP_TEXTS?.INTERNAL_ERROR,
+      error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR
+    );
+  }
+};
+
+const getAuditData = async (req: Request): Promise<any> => {
+  const projectId = path?.basename(req?.params?.projectId);
+  const stackId = path?.basename(req?.params?.stackId);
+  const moduleName = path.basename(req?.params?.moduleName);
+  const limit = parseInt(req?.params?.limit);
+  const startIndex = parseInt(req?.params?.startIndex);
+  const stopIndex = startIndex + limit;
+  const searchText = req?.params?.searchText;
+  const filter = req?.params?.filter;
+  const srcFunc = "getAuditData";
+
+  if (projectId?.includes('..') || stackId?.includes('..') || moduleName?.includes('..')) {
+    throw new BadRequestError("Invalid projectId, stackId, or moduleName");
+  }
+
+  try {
+    const mainPath = process?.cwd()?.split?.(GET_AUDIT_DATA?.MIGRATION)?.[0];
+    const logsDir = path.join(mainPath, GET_AUDIT_DATA?.MIGRATION, GET_AUDIT_DATA?.API_DIR, GET_AUDIT_DATA?.MIGRATION_DATA_DIR);
+
+    const stackFolders = fs.readdirSync(logsDir);
+
+    const stackFolder = stackFolders?.find(folder => folder?.startsWith?.(stackId));
+    if (!stackFolder) {
+      throw new BadRequestError("Migration data not found for this stack");
+    }
+
+    const auditLogPath = path?.resolve(logsDir, stackFolder, GET_AUDIT_DATA?.LOGS_DIR, GET_AUDIT_DATA?.AUDIT_DIR, GET_AUDIT_DATA?.AUDIT_REPORT);
+    if (!fs.existsSync(auditLogPath)) {
+      throw new BadRequestError("Audit log path not found");
+    }
+
+
+    // Read and parse the JSON file for the module
+    const filePath = path?.resolve(auditLogPath, `${moduleName}.json`);
+    let fileData;
+    if (fs?.existsSync(filePath)) {
+      const fileContent = await fsPromises.readFile(filePath, 'utf8');
+      try {
+        if (typeof fileContent === 'string') {
+          fileData = JSON?.parse(fileContent);
+        }
+      } catch (error) {
+        logger.error(`Error parsing JSON from file ${filePath}:`, error);
+        throw new BadRequestError('Invalid JSON format in audit file');
+      }
+    }
+
+    if (!fileData) {
+      throw new BadRequestError(`No audit data found for module: ${moduleName}`);
+    }
+    let transformedData = transformAndFlattenData(fileData);
+    if (filter != GET_AUDIT_DATA?.FILTERALL) {
+      const filters = filter?.split("-");
+      moduleName === 'Entries_Select_feild' ? transformedData = transformedData?.filter((log) => {
+        return filters?.some((filter) => {
+          return (
+            log?.display_type?.toLowerCase()?.includes(filter?.toLowerCase())
+          );
+        });
+      }) : transformedData = transformedData?.filter((log) => {
+        return filters?.some((filter) => {
+          return (
+            log?.data_type?.toLowerCase()?.includes(filter?.toLowerCase())
+          );
+        });
+      });
+
+    }
+    if (searchText && searchText !== null && searchText !== "null") {
+      transformedData = transformedData?.filter((item: any) => {
+        return Object?.values(item)?.some(value =>
+          value &&
+          typeof value === 'string' &&
+          value?.toLowerCase?.()?.includes(searchText?.toLowerCase())
+        );
+      });
+    }
+    const paginatedData = transformedData?.slice?.(startIndex, stopIndex);
+
+    return {
+      data: paginatedData,
+      totalCount: transformedData?.length,
+      status: HTTP_CODES?.OK
+    };
+
+  } catch (error: any) {
+    logger.error(
+      getLogMessage(
+        srcFunc,
+        `Error getting audit log data for module: ${moduleName}`,
+        error
+      )
+    );
     throw new ExceptionFunction(
       error?.message || HTTP_TEXTS.INTERNAL_ERROR,
       error?.statusCode || error?.status || HTTP_CODES.SERVER_ERROR
     );
   }
 };
+/**
+ * Transforms and flattens nested data structure into an array of items
+ * with sequential tuid values
+ */
+const transformAndFlattenData = (data: any): Array<{ [key: string]: any, id: number }> => {
+  try {
+    const flattenedItems: Array<{ [key: string]: any }> = [];
 
+    // Handle the data based on its structure
+    if (Array.isArray(data)) {
+      // If data is already an array, use it directly
+      data?.forEach((item, index) => {
+        flattenedItems?.push({
+          ...item ?? {},
+          uid: item?.uid || `item-${index}`
+        });
+      });
+    } else if (typeof data === 'object' && data !== null) {
+      Object?.entries?.(data)?.forEach(([key, value]) => {
+        if (Array.isArray(value)) {
+          value?.forEach((item, index) => {
+            flattenedItems?.push({
+              ...item ?? {},
+              parentKey: key,
+              uid: item?.uid || `${key}-${index}`
+            });
+          });
+        } else if (typeof value === 'object' && value !== null) {
+          flattenedItems?.push({
+            ...value,
+            key,
+            uid: (value as any)?.uid || key
+          });
+        }
+      });
+    }
+
+    return flattenedItems?.map((item, index) => ({
+      ...item ?? {},
+      id: index + 1
+    }));
+  } catch (error) {
+    console.error('Error transforming data:', error);
+    return [];
+  }
+};
 /**
  * Deletes a test stack.
  * @param req - The request object.
@@ -794,7 +938,6 @@ const startMigration = async (req: Request): Promise<any> => {
   }
 };
 
-
 const getLogs = async (req: Request): Promise<any> => {
   const projectId = req?.params?.projectId ? path?.basename(req.params.projectId) : "";
   const stackId = req?.params?.stackId ? path?.basename(req.params.stackId) : "";
@@ -803,9 +946,7 @@ const getLogs = async (req: Request): Promise<any> => {
   const stopIndex = startIndex + limit;
   const searchText = req?.params?.searchText ?? null;
   const filter = req?.params?.filter ?? "all";
-
   const srcFunc = "getLogs";
-
   if (
     !projectId ||
     !stackId ||
@@ -814,40 +955,44 @@ const getLogs = async (req: Request): Promise<any> => {
   ) {
     throw new BadRequestError("Invalid projectId or stackId");
   }
-
   try {
     const mainPath = process?.cwd()?.split("migration-v2")?.[0];
     if (!mainPath) {
       throw new BadRequestError("Invalid application path");
     }
-
     const logsDir = path?.join(mainPath, "migration-v2", "api", "logs");
     const loggerPath = path?.join(logsDir, projectId, `${stackId}.log`);
     const absolutePath = path?.resolve(loggerPath);
-
     if (!absolutePath?.startsWith(logsDir)) {
       throw new BadRequestError("Access to this file is not allowed.");
     }
-
     if (fs.existsSync(absolutePath)) {
+      let index = 0;
       const logs = await fs.promises.readFile(absolutePath, "utf8");
       let logEntries = logs
         ?.split("\n")
         ?.map((line) => {
           try {
-            return line ? JSON?.parse(line) : null;
+            const parsedLine = JSON?.parse(line)
+            parsedLine['id'] = index;
+            ++index;
+            return parsedLine ? parsedLine : null;
           } catch (error) {
             return null;
           }
         })
         ?.filter?.((entry) => entry !== null);
-
       if (!logEntries?.length) {
         return { logs: [], total: 0 };
       }
-
+      const filterOptions = Array?.from(new Set(logEntries?.map((log) => log?.level)));
+      const auditStartIndex = logEntries?.findIndex?.(log => log?.message?.includes("Starting audit process"));
+      const auditEndIndex = logEntries?.findIndex?.(log => log?.message?.includes("Audit process completed"));
+      logEntries = [
+        ...logEntries.slice(0, auditStartIndex),
+        ...logEntries.slice(auditEndIndex + 1)
+      ]
       logEntries = logEntries?.slice?.(1, logEntries?.length - 2);
-
       if (filter !== "all") {
         const filters = filter?.split("-") ?? [];
         logEntries = logEntries?.filter((log) => {
@@ -858,17 +1003,17 @@ const getLogs = async (req: Request): Promise<any> => {
           });
         });
       }
-
       if (searchText && searchText !== "null") {
         logEntries = logEntries?.filter?.((log) =>
           matchesSearchText(log, searchText)
         );
       }
-
       const paginatedLogs = logEntries?.slice?.(startIndex, stopIndex) ?? [];
       return {
         logs: paginatedLogs,
         total: logEntries?.length ?? 0,
+        filterOptions: filterOptions,
+        status: HTTP_CODES?.OK
       };
     } else {
       logger.error(getLogMessage(srcFunc, HTTP_TEXTS.LOGS_NOT_FOUND));
@@ -983,4 +1128,5 @@ export const migrationService = {
   getLogs,
   createSourceLocales,
   updateLocaleMapper,
+  getAuditData
 };