From 027ccabbace7fa58c7828607d45c51605c809162 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Mon, 18 Aug 2025 03:21:37 +0530 Subject: [PATCH 1/3] refactor: enhance Dockerfiles and entrypoint scripts, update dependencies, and improve error handling in migration services --- .dockerignore | 9 +++- api/Dockerfile | 13 +++-- api/package.json | 2 +- api/src/services/migration.service.ts | 31 ++++++----- api/src/services/sitecore.service.ts | 25 ++++++--- docker-compose.yml | 9 +++- ui/.dockerignore | 8 ++- ui/Dockerfile | 14 +++-- upload-api/Dockerfile | 26 +++++---- upload-api/docker-entrypoint.sh | 7 +++ upload-api/src/controllers/sitecore/index.ts | 57 +++++++++++--------- upload-api/src/helper/index.ts | 19 ++++++- upload-api/src/routes/index.ts | 11 ++-- upload-api/src/services/fileProcessing.ts | 5 +- 14 files changed, 163 insertions(+), 73 deletions(-) create mode 100644 upload-api/docker-entrypoint.sh diff --git a/.dockerignore b/.dockerignore index 5171c5408..582f87502 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,9 @@ node_modules -npm-debug.log \ No newline at end of file +npm-debug.log +build/ +*.log +.env +.git +.gitignore +Dockerfile +.dockerignore \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index a08a28b86..fe7653ba5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,13 +1,20 @@ -FROM --platform=linux/amd64 node:24.1.0-alpine3.22 +FROM node:24.4.1-alpine WORKDIR /usr/src/app COPY package*.json ./ -RUN npm install +RUN npm install \ + && addgroup -S nodeapp \ + && adduser -S nodeapp -G nodeapp COPY . . +# If your app writes to a build or other directory, create and set permissions here +RUN mkdir -p /usr/src/app/build && chown -R nodeapp:nodeapp /usr/src/app + +USER nodeapp + EXPOSE 5001 -CMD [ "npm","run", "dev"] \ No newline at end of file +CMD [ "npm", "run", "dev" ] \ No newline at end of file diff --git a/api/package.json b/api/package.json index f4c8e9ded..dc864e161 100644 --- a/api/package.json +++ b/api/package.json @@ -25,7 +25,7 @@ }, "homepage": "https://github.com/contentstack/migration-v2.git#readme", "dependencies": { - "@contentstack/cli": "^1.34.0", + "@contentstack/cli": "1.41.0", "@contentstack/cli-utilities": "^1.12.0", "@contentstack/json-rte-serializer": "^2.0.7", "@contentstack/marketplace-sdk": "^1.2.4", diff --git a/api/src/services/migration.service.ts b/api/src/services/migration.service.ts index 471c99ac6..bb07b5775 100644 --- a/api/src/services/migration.service.ts +++ b/api/src/services/migration.service.ts @@ -327,20 +327,20 @@ const startTestMigration = async (req: Request): Promise => { destinationStackId: project?.current_test_stack_id, }); await taxonomyService?.createTaxonomy({ - orgId, + orgId, projectId, - stackId:project?.destination_stack_id, + stackId: project?.destination_stack_id, current_test_stack_id: project?.current_test_stack_id, region, userId: user_id, }); await globalFieldServie?.createGlobalField({ region, - user_id, - stackId:project?.destination_stack_id, + user_id, + stackId: project?.destination_stack_id, current_test_stack_id: project?.current_test_stack_id, }); - + switch (cms) { case CMS.SITECORE_V8: case CMS.SITECORE_V9: @@ -361,6 +361,9 @@ const startTestMigration = async (req: Request): Promise => { projectId, project ); + await siteCoreService?.createEnvironment( + project?.current_test_stack_id + ) await siteCoreService?.createVersionFile( project?.current_test_stack_id ); @@ -563,17 +566,17 @@ const startMigration = async (req: Request): Promise => { destinationStackId: project?.destination_stack_id, }); await taxonomyService?.createTaxonomy({ - orgId, - projectId, - stackId:project?.destination_stack_id, - current_test_stack_id: project?.destination_stack_id, - region, - userId: user_id, + orgId, + projectId, + stackId: project?.destination_stack_id, + current_test_stack_id: project?.destination_stack_id, + region, + userId: user_id, }); await globalFieldServie?.createGlobalField({ region, - user_id, - stackId:project?.destination_stack_id, + user_id, + stackId: project?.destination_stack_id, current_test_stack_id: project?.destination_stack_id, }); switch (cms) { @@ -732,7 +735,7 @@ const getAuditData = async (req: Request): Promise => { if (!safeEntriesSelectFieldPath.startsWith(auditLogPath)) { throw new BadRequestError('Access to this file is not allowed.'); } - + const fileContent = await fsPromises?.readFile(safeEntriesSelectFieldPath, 'utf8'); try { if (typeof fileContent === 'string') { diff --git a/api/src/services/sitecore.service.ts b/api/src/services/sitecore.service.ts index fb1b2538c..1d2452121 100644 --- a/api/src/services/sitecore.service.ts +++ b/api/src/services/sitecore.service.ts @@ -16,6 +16,7 @@ import { getSafePath } from '../utils/sanitize-path.utils.js'; const append = 'a'; const baseDirName = MIGRATION_DATA_CONFIG.DATA; const { + ENVIRONMENTS_DIR_NAME, ENTRIES_DIR_NAME, LOCALE_DIR_NAME, LOCALE_MASTER_LOCALE, @@ -24,6 +25,7 @@ const { ASSETS_DIR_NAME, ASSETS_FILE_NAME, ASSETS_SCHEMA_FILE, + ENVIRONMENTS_FILE_NAME } = MIGRATION_DATA_CONFIG; const idCorrector = ({ id }: any) => { @@ -159,7 +161,7 @@ const createAssets = async ({ const blobPath: any = path.join(packagePath, 'blob', 'master'); const assetsPath = read(blobPath); if (assetsPath?.length) { - const isIdPresent = assetsPath?.find((ast) =>{ + const isIdPresent = assetsPath?.find((ast) => { return ast?.includes(metaData?.id) } ); @@ -307,8 +309,7 @@ const createEntry = async ({ for await (const ctType of contentTypes) { const message = getLogMessage( srcFunc, - `Transforming entries of Content Type ${ - keyMapper?.[ctType?.contentstackUid] ?? ctType?.contentstackUid + `Transforming entries of Content Type ${keyMapper?.[ctType?.contentstackUid] ?? ctType?.contentstackUid } has begun.`, {} ); @@ -407,7 +408,7 @@ const createEntry = async ({ ); const mapperCt: string = keyMapper?.[ctType?.contentstackUid] !== '' && - keyMapper?.[ctType?.contentstackUid] !== undefined + keyMapper?.[ctType?.contentstackUid] !== undefined ? keyMapper?.[ctType?.contentstackUid] : ctType?.contentstackUid; const fileMeta = { '1': `${newLocale}.json` }; @@ -422,8 +423,7 @@ const createEntry = async ({ } else { const message = getLogMessage( srcFunc, - `No entries found for the content type ${ - keyMapper?.[ctType?.contentstackUid] ?? ctType?.contentstackUid + `No entries found for the content type ${keyMapper?.[ctType?.contentstackUid] ?? ctType?.contentstackUid }.`, {} ); @@ -530,9 +530,22 @@ const createVersionFile = async (destinationStackId: string) => { ); }; +const createEnvironment = async (destinationStackId: string) => { + const baseDir = path.join(baseDirName, destinationStackId); + const environmentSave = path.join(baseDir, ENVIRONMENTS_DIR_NAME); + const environmentFile = path.join(environmentSave, ENVIRONMENTS_FILE_NAME); + + // Ensure the directory exists + await fs.promises.mkdir(environmentSave, { recursive: true }); + + // Write an empty environments file (or replace {} with your actual data) + await fs.promises.writeFile(environmentFile, JSON.stringify({}), 'utf8'); +} + export const siteCoreService = { createEntry, createAssets, createLocale, createVersionFile, + createEnvironment }; diff --git a/docker-compose.yml b/docker-compose.yml index e368831a8..c223cf244 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,10 @@ services: - "5001:5001" restart: always volumes: + - ${CMS_DATA_PATH}:${CONTAINER_PATH} - shared_data:/app/extracted_files + security_opt: + - no-new-privileges:true upload-api: container_name: migration-upload-api @@ -20,6 +23,8 @@ services: volumes: - ${CMS_DATA_PATH}:${CONTAINER_PATH} - shared_data:/app/extracted_files + security_opt: + - no-new-privileges:true ui: container_name: migration-ui @@ -28,6 +33,8 @@ services: ports: - "3000:3000" restart: always + security_opt: + - no-new-privileges:true volumes: - shared_data: \ No newline at end of file + shared_data: diff --git a/ui/.dockerignore b/ui/.dockerignore index af54d5fa7..f510896bd 100644 --- a/ui/.dockerignore +++ b/ui/.dockerignore @@ -1,3 +1,9 @@ node_modules npm-debug.log -build \ No newline at end of file +Dockerfile +build/ +*.log +.git +.gitignore +.dockerignore +.env \ No newline at end of file diff --git a/ui/Dockerfile b/ui/Dockerfile index db1e5218c..d495642e1 100644 --- a/ui/Dockerfile +++ b/ui/Dockerfile @@ -1,13 +1,21 @@ -FROM --platform=linux/amd64 node:22-alpine +FROM node:22-alpine WORKDIR /usr/src/app COPY package*.json ./ -RUN apk update && apk upgrade && npm install +RUN apk update && apk upgrade \ + && npm install \ + && addgroup -S nodeapp \ + && adduser -S nodeapp -G nodeapp COPY . . +# Ensure permissions for nodeapp user +RUN chown -R nodeapp:nodeapp /usr/src/app + +USER nodeapp + EXPOSE 3000 -CMD [ "npm","run", "start"] \ No newline at end of file +CMD [ "npm", "run", "start" ] \ No newline at end of file diff --git a/upload-api/Dockerfile b/upload-api/Dockerfile index 3890502cf..85e07c3a9 100644 --- a/upload-api/Dockerfile +++ b/upload-api/Dockerfile @@ -1,20 +1,26 @@ -# Use an official Node.js runtime as a base image -FROM --platform=linux/amd64 node:24.1.0-alpine3.22 +FROM node:24.4.1-alpine -# Set the working directory in the container WORKDIR /app -# Copy package.json and package-lock.json to the working directory COPY package*.json ./ -# Install application dependencies -RUN npm install +RUN npm install \ + && addgroup -S nodeapp \ + && adduser -S nodeapp -G nodeapp -# Copy the application code to the container COPY . . -# Expose the port your app will run on +RUN mkdir -p /app/build \ + && mkdir -p /app/extracted_files \ + && chown -R nodeapp:nodeapp /app + +COPY docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +USER nodeapp + EXPOSE 4002 -# Define the command to run your application -CMD ["npm", "run", "start"] \ No newline at end of file +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["npm", "run", "start"] \ No newline at end of file diff --git a/upload-api/docker-entrypoint.sh b/upload-api/docker-entrypoint.sh new file mode 100644 index 000000000..df0525efb --- /dev/null +++ b/upload-api/docker-entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e + +# Fix permissions for extracted_files volume at runtime +chown -R nodeapp:nodeapp /app/extracted_files + +exec "$@" \ No newline at end of file diff --git a/upload-api/src/controllers/sitecore/index.ts b/upload-api/src/controllers/sitecore/index.ts index e487dae86..fd6166971 100644 --- a/upload-api/src/controllers/sitecore/index.ts +++ b/upload-api/src/controllers/sitecore/index.ts @@ -22,32 +22,48 @@ interface RequestParams { endpoint?: string; } -const createLocaleSource = async ({ app_token, localeData, projectId }: { app_token: string | string[], localeData: any, projectId: string | string[] }) => { +const createLocaleSource = async ({ + app_token, + localeData, + projectId, +}: { + app_token: string | string[]; + localeData: any; + projectId: string | string[]; +}) => { const mapperConfig = { method: 'post', maxBodyLength: Infinity, url: `${process.env.NODE_BACKEND_API}/v2/migration/localeMapper/${projectId}`, headers: { app_token, - 'Content-Type': 'application/json' + 'Content-Type': 'application/json', }, data: { - locale: Array?.from?.(localeData) ?? [] + locale: Array.isArray(localeData) ? localeData : Array.from(localeData ?? []), }, }; - const mapRes = await axios?.request?.(mapperConfig); - if (mapRes?.status == 200) { - logger.info('Legacy CMS', { - status: HTTP_CODES?.OK, - message: HTTP_TEXTS?.LOCALE_SAVED, - }); - } else { - logger.warn('Legacy CMS error:', { - status: HTTP_CODES?.UNAUTHORIZED, - message: HTTP_TEXTS?.LOCALE_FAILED, + + try { + const mapRes = await axios.request(mapperConfig); + if (mapRes?.status === 200) { + logger.info('Legacy CMS', { + status: HTTP_CODES?.OK, + message: HTTP_TEXTS?.LOCALE_SAVED, + }); + } else { + logger.warn('Legacy CMS error:', { + status: mapRes?.status, + message: HTTP_TEXTS?.LOCALE_FAILED, + }); + } + } catch (error: any) { + logger.warn('Legacy CMS error:', { + status: error?.response?.status || HTTP_CODES?.UNAUTHORIZED, + message: error?.response?.data?.message || HTTP_TEXTS?.LOCALE_FAILED, }); } -} +}; /** @@ -104,6 +120,7 @@ const sendRequestWithRetry = async (params: RequestParams): Promise { try { const newPath = path.join(filePath, 'items'); + console.log("🚀 ~ createSitecoreMapper ~ newPath:", newPath) await ExtractFiles(newPath); const localeData = await extractLocales(path.join(filePath, 'items', 'master', 'sitecore', 'content')); await createLocaleSource?.({ app_token, localeData, projectId }); @@ -127,18 +144,6 @@ const createSitecoreMapper = async (filePath: string = "", projectId: string | s fieldMapping.contentTypes.push(element); } } - // const config = { - // method: 'post', - // maxBodyLength: Infinity, - // url: `${process.env.NODE_BACKEND_API}/v2/mapper/createDummyData/${projectId}`, - // headers: { - // app_token, - // 'Content-Type': 'application/json' - // }, - // data: JSON.stringify(fieldMapping), - // }; - - // const { data } = await axios.request(config); const { data } = await sendRequestWithRetry({ payload: fieldMapping, projectId, diff --git a/upload-api/src/helper/index.ts b/upload-api/src/helper/index.ts index a7ea4b79a..ee4783347 100644 --- a/upload-api/src/helper/index.ts +++ b/upload-api/src/helper/index.ts @@ -14,10 +14,22 @@ const getFileName = (params: { Key: string }) => { return obj; }; +/** + * Splits a file path and returns the first folder or file name. + * Example: "umesh/items/master/sitecore/content" => "umesh" + */ +function getFirstNameFromFilename(filename: string): string { + if (!filename) return ''; + // Split by both Unix and Windows separators + const parts = filename.split(/[\\/]/); + return parts[0] || ''; +} + const saveZip = async (zip: any, name: string) => { try { const newMainFolderName = name; const keys = Object.keys(zip.files); + let filePathSaved = undefined; for await (const filename of keys) { const file = zip.files[filename]; @@ -29,6 +41,9 @@ const saveZip = async (zip: any, name: string) => { !filename.startsWith(newMainFolderName + '/') ) { newFilePath = path.join(newMainFolderName, filename); + if (!filename?.includes?.(MACOSX_FOLDER)) { + filePathSaved = getFirstNameFromFilename(filename); + } } const filePath = path.join(__dirname, '..', '..', 'extracted_files', newFilePath); @@ -40,14 +55,14 @@ const saveZip = async (zip: any, name: string) => { } } - return true; + return { isSaved: true, filePath: filePathSaved }; } catch (err: any) { console.error(err); logger.info('Zipfile error:', { status: HTTP_CODES?.SERVER_ERROR, message: HTTP_TEXTS?.ZIP_FILE_SAVE, }); - return false; + return { isSaved: false, filePath: undefined }; } }; diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index 8ef1cb7bb..dbdfb5d48 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -165,7 +165,10 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, name); res.status(data?.status || 200).json(data); if (data?.status === 200) { - const filePath = path.join(__dirname, '..', '..', 'extracted_files', name); + let filePath = path.join(__dirname, '..', '..', 'extracted_files', name); + if (data?.file !== undefined) { + filePath = path.join(__dirname, '..', '..', 'extracted_files', name, data?.file); + } createMapper(filePath, projectId, app_token, affix, config); } }); @@ -217,8 +220,10 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, fileName); res.json(data); res.send('file valited sucessfully.'); - const filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName); - console.log("🚀 ~ bodyStream.on ~ filePath:", filePath) + let filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName); + if (data?.file !== undefined) { + filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName, data?.file); + } createMapper(filePath, projectId, app_token, affix, config); }); } diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index 0e538a4e0..025ef3b66 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -17,7 +17,7 @@ const handleFileProcessing = async ( await zip.loadAsync(zipBuffer); if (await validator({ data: zip, type: cmsType, extension: fileExt })) { const isSaved = await saveZip(zip, name); - if (isSaved) { + if (isSaved?.isSaved) { logger.info('Validation success:', { status: HTTP_CODES?.OK, message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL @@ -25,7 +25,8 @@ const handleFileProcessing = async ( return { status: HTTP_CODES?.OK, message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL, - file_details: config + file_details: config, + file: isSaved?.filePath }; } } else { From e551b710b5a0187e58b1e9fe7ebcae980c679c8b Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Mon, 18 Aug 2025 03:24:28 +0530 Subject: [PATCH 2/3] feat: add sitecore.service.ts to fileignoreconfig with checksum --- .talismanrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.talismanrc b/.talismanrc index a522a9226..90a2da83c 100644 --- a/.talismanrc +++ b/.talismanrc @@ -88,4 +88,6 @@ fileignoreconfig: checksum: faac1367b4d80e022d4b7a165f9d2ac8bfd428f1a91af849064992ae2b3d6b1f - filename: ui/src/components/DestinationStack/Actions/LoadLanguageMapper.tsx checksum: 988d93b93768ef0909305590ab0b89456333b9937ec12828edf23163f8640dfc + - filename: api/src/services/sitecore.service.ts + checksum: ceaed6417b1b6fd2581f547cbc7e871900ecbae5557496b26f025781f323ba82 version: "1.0" \ No newline at end of file From b367c1750c171d5d71776a8f468c592dc7fd8193 Mon Sep 17 00:00:00 2001 From: umeshmore45 Date: Mon, 18 Aug 2025 03:27:59 +0530 Subject: [PATCH 3/3] fix: correct typo in success message for file validation --- upload-api/src/controllers/sitecore/index.ts | 3 +-- upload-api/src/routes/index.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/upload-api/src/controllers/sitecore/index.ts b/upload-api/src/controllers/sitecore/index.ts index fd6166971..817507fd9 100644 --- a/upload-api/src/controllers/sitecore/index.ts +++ b/upload-api/src/controllers/sitecore/index.ts @@ -40,7 +40,7 @@ const createLocaleSource = async ({ 'Content-Type': 'application/json', }, data: { - locale: Array.isArray(localeData) ? localeData : Array.from(localeData ?? []), + locale: Array.isArray(localeData) ? localeData : (localeData ? [localeData] : []), }, }; @@ -120,7 +120,6 @@ const sendRequestWithRetry = async (params: RequestParams): Promise { try { const newPath = path.join(filePath, 'items'); - console.log("🚀 ~ createSitecoreMapper ~ newPath:", newPath) await ExtractFiles(newPath); const localeData = await extractLocales(path.join(filePath, 'items', 'master', 'sitecore', 'content')); await createLocaleSource?.({ app_token, localeData, projectId }); diff --git a/upload-api/src/routes/index.ts b/upload-api/src/routes/index.ts index dbdfb5d48..0b87309e6 100644 --- a/upload-api/src/routes/index.ts +++ b/upload-api/src/routes/index.ts @@ -219,7 +219,7 @@ router.get('/validator', express.json(), fileOperationLimiter, async function (r const data = await handleFileProcessing(fileExt, zipBuffer, cmsType, fileName); res.json(data); - res.send('file valited sucessfully.'); + res.send('file validated successfully.'); let filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName); if (data?.file !== undefined) { filePath = path.join(__dirname, '..', '..', 'extracted_files', fileName, data?.file);