diff --git a/.github/workflows/sast-scan.yml b/.github/workflows/sast-scan.yml index 3b9521a55..67b0e5b18 100644 --- a/.github/workflows/sast-scan.yml +++ b/.github/workflows/sast-scan.yml @@ -8,4 +8,4 @@ jobs: steps: - uses: actions/checkout@v2 - name: Semgrep Scan - run: docker run -v /var/run/docker.sock:/var/run/docker.sock -v "${PWD}:/src" returntocorp/semgrep semgrep scan --config auto \ No newline at end of file + run: docker run -v /var/run/docker.sock:/var/run/docker.sock -v "${PWD}:/src" returntocorp/semgrep semgrep scan --config auto diff --git a/README.md b/README.md index 3f079c37e..fc71234fd 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ # Migration v2 + ## Overview + The migration-v2 project is designed to facilitate the migration of assets and related functionalities. This project includes multiple components such as API, UI, and upload API. ## Installation + Clone the repository: + ``` git clone https://github.com/contentstack/migration-v2.git cd migration-v2 @@ -16,6 +20,7 @@ Check for readme.md files and install dependencies for folders This is the migration V2's node server. ### Environment Variables + The following environment variables are used in this project: - `APP_TOKEN_KEY`: The token key for the application. Default is `MIGRATION_V2`. @@ -27,14 +32,15 @@ Make sure to set these variables in a `.env` file at the root of your project. 2. To run the production server, create a `./production.env` file and add environment variables as per `./example.env` ### To start the server -Run `npm run dev` +Run `npm run dev` ## Migration UI Migration UI is a web application designed to facilitate the migration of content. It provides a user-friendly interface for managing and migrating content efficiently. ### Features + - User-friendly interface for content migration - Integration with Contentstack - State management with Redux Toolkit @@ -43,14 +49,15 @@ Migration UI is a web application designed to facilitate the migration of conten ### Installation 1. Navigate to the project directory: - ```sh - cd ui - ``` + + ```sh + cd ui + ``` 2. Install the dependencies: - ```sh - npm install - ``` + ```sh + npm install + ``` ### Environment Variables @@ -66,7 +73,9 @@ The following environment variables are used in this project: Make sure to set these variables in a `.env` file at the root of your ui project. ### Usage + Start the development server: + ```sh npm start ``` @@ -74,6 +83,7 @@ npm start Open your browser and navigate to http://localhost:3000. ### Dependencies + - @contentstack/json-rte-serializer: ^2.0.5 - @contentstack/venus-components: ^2.2.4 - @reduxjs/toolkit: ^2.2.5 @@ -85,15 +95,21 @@ Open your browser and navigate to http://localhost:3000. - @types/react-redux: ^7.1.33 ## Migration v2 upload-api + ### Overview + The migration-v2 upload-api project is designed to facilitate the migration of different CMS to Contentstack functionalities. ### Installation + Navigate to the project directory: + ``` cd migration-v2/upload-api ``` + Install dependencies: + ``` npm install ``` @@ -108,6 +124,7 @@ The following environment variables are used in this project: Make sure to set these variables in a `.env` file at the root of your project. ### Configuration + Please refer sample.config.json and provide this config in index.ts of config folder of upload-api/src The following configuration is used in this project: @@ -116,6 +133,7 @@ The following configuration is used in this project: - `isLocalPath`: A flag to indicate if the path is local. Default is `true`. #### AWS Data + - `awsRegion`: The AWS region. Default is `us-east-2`. - `awsAccessKeyId`: The AWS access key ID. - `awsSecretAccessKey`: The AWS secret access key. @@ -124,22 +142,29 @@ The following configuration is used in this project: - `buketKey`: The key for the AWS S3 bucket. Default is `project/package 45.zip`. #### Local Path + - `localPath`: The local path to the extracted files. for example `/upload-api/extracted_files/package 45.zip`. ## Cli + Navigate to the project directory - 1. Install pnpm - Since pnpm is required, you need to install it globally if it's not already installed: - ``` - npm install -g pnpm - ``` - 2. Run the Setup Script - Now, run the main setup script specified in the package.json: - ``` - npm run setup-repo - ``` + +1. Install pnpm + Since pnpm is required, you need to install it globally if it's not already installed: + +``` +npm install -g pnpm +``` + +2. Run the Setup Script + Now, run the main setup script specified in the package.json: + +``` +npm run setup-repo +``` ## Scripts + - `npm start`: Starts the main server by running index.js. - `npm run api`: Navigates to the api directory and runs the development server. - `npm run upload`: Navigates to the upload-api directory and starts the upload API server. @@ -149,22 +174,27 @@ Navigate to the project directory - `npm test`: Displays an error message indicating that no tests are specified. ## Repository + - Type: git - URL: https://github.com/contentstack/migration-v2.git ## Bugs and Issues + - URL: https://github.com/contentstack/migration-v2/issues ## Homepage + - URL: https://github.com/contentstack/migration-v2#readme ## DevDependencies + - husky: ^4.3.8 - prettier: ^2.4.1 - rimraf: ^3.0.2 - validate-branch-name: ^1.3.0 ## Husky Configuration + ``` "husky": { "hooks": {} @@ -172,6 +202,7 @@ Navigate to the project directory ``` - Branch Name Validation + ``` "validate-branch-name": { "pattern": "^(feature|bugfix|hotfix)/[a-z0-9-]{5,30}$", @@ -180,10 +211,13 @@ Navigate to the project directory ``` ## License + This project is licensed under the ISC License. ## Author + The author information is not specified. ## Contact + For further assistance, please contact the project maintainer through the issues page on GitHub. diff --git a/api/src/constants/app/index.json b/api/src/constants/app/index.json index 6340cbaac..8fc7dfa76 100644 --- a/api/src/constants/app/index.json +++ b/api/src/constants/app/index.json @@ -3665,4 +3665,4 @@ } ], "count": 11 -} \ No newline at end of file +} diff --git a/api/src/constants/index.ts b/api/src/constants/index.ts index 8e311ed26..2ccc9ae4d 100644 --- a/api/src/constants/index.ts +++ b/api/src/constants/index.ts @@ -1,11 +1,11 @@ export const CS_REGIONS = ["NA", "EU", "AZURE_NA", "AZURE_EU", "GCP_NA"]; export const DEVURLS: any = { - 'NA': 'developerhub-api.contentstack.com', - 'EU': 'eu-developerhub-api.contentstack.com', - 'AZURE_NA': 'azure-na-developerhub-api.contentstack.com', - 'AZURE_EU': 'azure-eu-developerhub-api.contentstack.com', - 'GCP_NA': 'gcp-na-developerhub-api.contentstack.com', -} + NA: "developerhub-api.contentstack.com", + EU: "eu-developerhub-api.contentstack.com", + AZURE_NA: "azure-na-developerhub-api.contentstack.com", + AZURE_EU: "azure-eu-developerhub-api.contentstack.com", + GCP_NA: "gcp-na-developerhub-api.contentstack.com", +}; export const CMS = { CONTENTFUL: "contentful", SITECORE_V8: "sitecore v8", @@ -13,7 +13,7 @@ export const CMS = { SITECORE_V10: "sitecore v10", WORDPRESS: "wordpress", AEM: "aem", -} +}; export const MODULES = [ "Project", "Migration", @@ -58,7 +58,8 @@ export const HTTP_TEXTS = { "Project's Fileformat confirmation updated successfully", CMS_UPDATED: "Project's migration cms updated successfully", STACK_UPDATED: "Project's migration stack details updated successfully", - CONTENT_MAPPER_UPDATED: "Project's migration content mapping updated successfully", + CONTENT_MAPPER_UPDATED: + "Project's migration content mapping updated successfully", FILE_FORMAT_UPDATED: "Project's migration file format updated successfully", DESTINATION_STACK_UPDATED: "Project's migration destination stack updated successfully", @@ -95,14 +96,11 @@ export const HTTP_TEXTS = { "Sorry, the requested content mapper id does not exists.", ADMIN_LOGIN_ERROR: "Sorry, You Don't have admin access in any of the Organisation", - PROJECT_DELETE: - "Project Deleted Successfully", - PROJECT_REVERT: - "Project Reverted Successfully", - LOGS_NOT_FOUND: - "Sorry, no logs found for requested stack migration.", + PROJECT_DELETE: "Project Deleted Successfully", + PROJECT_REVERT: "Project Reverted Successfully", + LOGS_NOT_FOUND: "Sorry, no logs found for requested stack migration.", MIGRATION_EXECUTION_KEY_UPDATED: - "Project's migration execution key updated successfully" + "Project's migration execution key updated successfully", }; export const HTTP_RESPONSE_HEADERS = { @@ -177,18 +175,28 @@ export const CONTENT_TYPE_STATUS = { export const LOCALE_MAPPER: any = { //not more than one locale mapping in master locale masterLocale: { - 'en-us': 'en' + "en-us": "en", }, - 'fr': 'fr-fr' -} + fr: "fr-fr", +}; export const CHUNK_SIZE = 1048576; -export const LIST_EXTENSION_UID = 'bltc44e51cc9f4b0d80'; +export const LIST_EXTENSION_UID = "bltc44e51cc9f4b0d80"; export const KEYTOREMOVE = [ - "update", "fetch", "delete", "oauth", "hosting", "install", "reinstall", - "upgrade", "getRequests", "authorize", "authorization", "listInstallations" -] + "update", + "fetch", + "delete", + "oauth", + "hosting", + "install", + "reinstall", + "upgrade", + "getRequests", + "authorize", + "authorization", + "listInstallations", +]; export const MIGRATION_DATA_CONFIG = { DATA: "./cmsMigrationData", @@ -262,5 +270,5 @@ export const MIGRATION_DATA_CONFIG = { GLOBAL_FIELDS_DIR_NAME: "global_fields", GLOBAL_FIELDS_FILE_NAME: "globalfields.json", - EXPORT_INFO_FILE: "export-info.json" -} \ No newline at end of file + EXPORT_INFO_FILE: "export-info.json", +}; diff --git a/index.js b/index.js index 8993572b9..9ac3496f4 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,5 @@ -const fs = require('fs'); -const path = require('path'); +const fs = require("fs"); +const path = require("path"); const apiEnvContent = ` APP_TOKEN_KEY=MIGRATION_V2 @@ -21,19 +21,18 @@ NODE_BACKEND_API =http://localhost:5001 `; const envFilePaths = { - API: path.join(__dirname, 'api', 'production.env'), - UI: path.join(__dirname, 'ui', '.env.local'), - 'Upload-API': path.join(__dirname, 'upload-api', '.env'), + API: path.join(__dirname, "api", "production.env"), + UI: path.join(__dirname, "ui", ".env.local"), + "Upload-API": path.join(__dirname, "upload-api", ".env"), }; // Define the contents for each file in an array const envContents = { API: apiEnvContent, UI: uiEnvContent, - 'Upload-API': uploadAPIEnvContent, + "Upload-API": uploadAPIEnvContent, }; - // Function to create env files const createEnvFiles = () => { // Loop through each key in the envFilePaths object diff --git a/package.json b/package.json index e1feb8a51..305fec691 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,9 @@ "xml2js": "^0.5.0" }, "husky": { - "hooks": {} + "hooks": { + "pre-commit": "npx validate-branch-name" + } }, "validate-branch-name": { "pattern": "^(feature|bugfix|hotfix)/[a-z0-9-]{5,30}$", diff --git a/ui/README.md b/ui/README.md index 8b1378917..e69de29bb 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1 +0,0 @@ - diff --git a/ui/package-lock.json b/ui/package-lock.json index aa1bea4b4..84ea7ab82 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -17,7 +17,7 @@ "@types/react": "^18.2.28", "@types/react-dom": "^18.2.13", "@types/react-redux": "^7.1.33", - "axios": "^1.7.8", + "axios": "^1.8.2", "bootstrap": "5.1.3", "chokidar": "^3.6.0", "final-form": "^4.20.10", @@ -6905,9 +6905,9 @@ } }, "node_modules/axios": { - "version": "1.7.8", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz", - "integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.8.3.tgz", + "integrity": "sha512-iP4DebzoNlP/YN2dpwCgb8zoCmhtkajzS48JvwmkSkXvPI3DHc7m+XYL5tGnSlJtR6nImXZmdCuN5aP8dh1d8A==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", diff --git a/ui/package.json b/ui/package.json index f27b27110..78c2115da 100644 --- a/ui/package.json +++ b/ui/package.json @@ -12,7 +12,7 @@ "@types/react": "^18.2.28", "@types/react-dom": "^18.2.13", "@types/react-redux": "^7.1.33", - "axios": "^1.7.8", + "axios": "^1.8.2", "bootstrap": "5.1.3", "chokidar": "^3.6.0", "final-form": "^4.20.10", diff --git a/ui/public/index.html b/ui/public/index.html index f96d1d670..5fc754fb5 100644 --- a/ui/public/index.html +++ b/ui/public/index.html @@ -32,7 +32,10 @@ href="https://fonts.googleapis.com/css2?family=Montserrat:wght@100..900&display=swap" rel="stylesheet" /> - + ', ''); @@ -107,7 +107,7 @@ export const setDataInLocalStorage = (key: string, data: string) => { }; // utitlity function to retrieve state from sessionStorage -export const getStateFromLocalStorage = (key : string) => { +export const getStateFromLocalStorage = (key: string) => { const state = sessionStorage.getItem(key); return state ? JSON.parse(state) : null; }; diff --git a/ui/src/utilities/projectDBMapper.ts b/ui/src/utilities/projectDBMapper.ts index cd77ade06..0e46c44b0 100644 --- a/ui/src/utilities/projectDBMapper.ts +++ b/ui/src/utilities/projectDBMapper.ts @@ -1,46 +1,46 @@ export function createObject(projectData: any) { - const object = { - legacy_cms: { - selectedCms: { - cms_id: projectData?.legacy_cms?.cms_id, - allowed_file_formats: projectData?.legacy_cms?.allowed_file_formats, + const object = { + legacy_cms: { + selectedCms: { + cms_id: projectData?.legacy_cms?.cms_id, + allowed_file_formats: projectData?.legacy_cms?.allowed_file_formats + }, + affix: projectData?.legacy_cms?.affix, + file_format: projectData?.legacy_cms?.file_format, + uploadedFile: { + file_details: { + localPath: projectData?.legacy_cms?.file_path, + awsData: { + awsRegion: projectData?.legacy_cms?.awsDetails?.awsRegion, + bucketName: projectData?.legacy_cms?.awsDetails?.bucketName, + buketKey: projectData?.legacy_cms?.awsDetails?.buketKey }, - affix: projectData?.legacy_cms?.affix, - file_format: projectData?.legacy_cms?.file_format, - uploadedFile: { - file_details: { - localPath: projectData?.legacy_cms?.file_path, - awsData: { - awsRegion: projectData?.legacy_cms?.awsDetails?.awsRegion, - bucketName: projectData?.legacy_cms?.awsDetails?.bucketName, - buketKey: projectData?.legacy_cms?.awsDetails?.buketKey - }, - isLocalPath: projectData?.legacy_cms?.is_localPath - }, - isValidated: projectData?.legacy_cms?.is_fileValid - }, + isLocalPath: projectData?.legacy_cms?.is_localPath }, - destination_stack: { - selectedOrg: { - value: projectData?.org_id, - label: projectData?.org_name - }, - selectedStack: { - value: projectData?.destination_stack_id, - label: projectData?.destination_stack_name, - master_locale: projectData?.destination_stack_master_locale - }, - stackArray:{ - value: projectData?.destination_stack_id, - label: projectData?.destination_stack_name, - master_locale: projectData?.destination_stack_master_locale, - created_at: projectData?.destination_stack_created_at, - } - }, - content_mapping: projectData?.content_mapping, - stackDetails: projectData?.stackDetails, - mapperKeys: projectData?.mapperKeys, - }; + isValidated: projectData?.legacy_cms?.is_fileValid + } + }, + destination_stack: { + selectedOrg: { + value: projectData?.org_id, + label: projectData?.org_name + }, + selectedStack: { + value: projectData?.destination_stack_id, + label: projectData?.destination_stack_name, + master_locale: projectData?.destination_stack_master_locale + }, + stackArray: { + value: projectData?.destination_stack_id, + label: projectData?.destination_stack_name, + master_locale: projectData?.destination_stack_master_locale, + created_at: projectData?.destination_stack_created_at + } + }, + content_mapping: projectData?.content_mapping, + stackDetails: projectData?.stackDetails, + mapperKeys: projectData?.mapperKeys + }; - return object; -} \ No newline at end of file + return object; +} diff --git a/ui/tsconfig.json b/ui/tsconfig.json index bc3afcbe1..3efb9fd7b 100644 --- a/ui/tsconfig.json +++ b/ui/tsconfig.json @@ -18,7 +18,7 @@ "baseUrl": "./", "paths": { "*": ["src/types/*"], - "react": [ "./node_modules/@types/react" ] + "react": ["./node_modules/@types/react"] }, "typeRoots": ["./node_modules/@types", "./src/types"] }, diff --git a/upload-api/migration-contentful/config/index.json b/upload-api/migration-contentful/config/index.json index c2a6e4b1a..81b16389e 100644 --- a/upload-api/migration-contentful/config/index.json +++ b/upload-api/migration-contentful/config/index.json @@ -4,8 +4,4 @@ "contentful": "contentfulSchema" }, "fileName": "en-us.json" - - - - -} \ No newline at end of file +} diff --git a/upload-api/migration-contentful/index.js b/upload-api/migration-contentful/index.js index 6da326239..045af75ec 100644 --- a/upload-api/migration-contentful/index.js +++ b/upload-api/migration-contentful/index.js @@ -2,11 +2,10 @@ const extractContentTypes = require('./libs/extractContentTypes'); const createInitialMapper = require('./libs/createInitialMapper'); -const extractLocale = require('./libs/extractLocale') +const extractLocale = require('./libs/extractLocale'); module.exports = { extractContentTypes, createInitialMapper, extractLocale }; - diff --git a/upload-api/migration-contentful/libs/createInitialMapper.js b/upload-api/migration-contentful/libs/createInitialMapper.js index 77b6a986b..ff5ed67f7 100644 --- a/upload-api/migration-contentful/libs/createInitialMapper.js +++ b/upload-api/migration-contentful/libs/createInitialMapper.js @@ -44,9 +44,9 @@ const uidCorrector = (uid, prefix) => { /** * Creates an initial mapping for content types by processing files in a specified directory. - * + * * @returns {Promise<{ contentTypes: object[] }>} A promise that resolves to an object containing an array of content type objects. - * + * * @description * This function performs the following steps: * 1. Reads all files in a specified directory containing data about content types. @@ -56,9 +56,9 @@ const uidCorrector = (uid, prefix) => { * 5. The content type fields are further enriched by mapping the fields from the data using a helper function `contentTypeMapper`. * 6. After processing all the files, the content type objects are returned as an array. * 7. The function handles errors and logs them to the console if any occur during the process. - * + * * The function also deletes a folder at the end of the process (using `deleteFolderSync`), which may be used for cleanup purposes. - * + * * // Outputs: an array of content type objects, each containing metadata and field mappings. */ const createInitialMapper = async () => { @@ -110,9 +110,7 @@ const createInitialMapper = async () => { } ]; const dataArray = data.filter((item) => item.id !== 'title' && item.id !== 'url'); - const contentstackFields = [...uidTitle, ...contentTypeMapper(dataArray)].filter( - Boolean - ); + const contentstackFields = [...uidTitle, ...contentTypeMapper(dataArray)].filter(Boolean); contentTypeObject.fieldMapping = contentstackFields; initialMapper.push(contentTypeObject); diff --git a/upload-api/migration-contentful/libs/extractLocale.js b/upload-api/migration-contentful/libs/extractLocale.js index 65c4b84f4..473b2e68a 100644 --- a/upload-api/migration-contentful/libs/extractLocale.js +++ b/upload-api/migration-contentful/libs/extractLocale.js @@ -1,36 +1,36 @@ -"use strict"; +'use strict'; /* eslint-disable @typescript-eslint/no-var-requires */ /** * External module dependencies. */ -const fs = require("fs"); +const fs = require('fs'); /** * @description - * Function to retrieve the unique source locales from the legacy CMS + * Function to retrieve the unique source locales from the legacy CMS * @param {*} jsonFilePath - Local file path of the exported data * @returns {Array} - Array of unique locales used in the exported data */ const extractLocale = async (jsonFilePath) => { - try { - const rawData = fs?.readFileSync?.(jsonFilePath, "utf8"); - const jsonData = JSON?.parse?.(rawData); + try { + const rawData = fs?.readFileSync?.(jsonFilePath, 'utf8'); + const jsonData = JSON?.parse?.(rawData); - // Extract unique language codes from locales array - const uniqueLanguages = new Set(); - if (Array?.isArray?.(jsonData?.locales)) { - jsonData?.locales?.forEach?.(locale => { - if (locale?.code) { - uniqueLanguages.add(locale?.code); // Normalize to lowercase - } - }); + // Extract unique language codes from locales array + const uniqueLanguages = new Set(); + if (Array?.isArray?.(jsonData?.locales)) { + jsonData?.locales?.forEach?.((locale) => { + if (locale?.code) { + uniqueLanguages.add(locale?.code); // Normalize to lowercase } - - return [...uniqueLanguages]; // Convert Set to array for output - } catch (error) { - console.error(`Error reading JSON file:`, error.message); - return []; + }); } + + return [...uniqueLanguages]; // Convert Set to array for output + } catch (error) { + console.error(`Error reading JSON file:`, error.message); + return []; + } }; -module.exports = extractLocale \ No newline at end of file +module.exports = extractLocale; diff --git a/upload-api/migration-contentful/utils/apps/appDetails.json b/upload-api/migration-contentful/utils/apps/appDetails.json index 2d7619fd0..19c7a0722 100644 --- a/upload-api/migration-contentful/utils/apps/appDetails.json +++ b/upload-api/migration-contentful/utils/apps/appDetails.json @@ -11290,4 +11290,4 @@ } ] } -} \ No newline at end of file +} diff --git a/upload-api/migration-contentful/utils/helper.js b/upload-api/migration-contentful/utils/helper.js index aa4b57289..c5a259012 100755 --- a/upload-api/migration-contentful/utils/helper.js +++ b/upload-api/migration-contentful/utils/helper.js @@ -6,7 +6,7 @@ const mkdirp = require('mkdirp'); const path = require('path'); const fs = require('fs'); - const readFile = function (filePath, parse) { +const readFile = function (filePath, parse) { parse = typeof parse == 'undefined' ? true : parse; filePath = path.resolve(filePath); let data; @@ -14,18 +14,18 @@ const fs = require('fs'); return data; }; - const writeFile = function (filePath, data) { +const writeFile = function (filePath, data) { filePath = path.resolve(filePath); data = typeof data == 'object' ? JSON.stringify(data) : data || '{}'; fs.writeFileSync(filePath, data, 'utf-8'); }; - const appendFile = function (filePath, data) { +const appendFile = function (filePath, data) { filePath = path.resolve(filePath); fs.appendFileSync(filePath, data); }; - const makeDirectory = function () { +const makeDirectory = function () { for (let key in arguments) { let dirname = path.resolve(arguments[key]); if (!fs.existsSync(dirname)) mkdirp.sync(dirname); diff --git a/upload-api/migration-sitecore/constants/index.js b/upload-api/migration-sitecore/constants/index.js index 3e5e04fc1..25722bf5d 100644 --- a/upload-api/migration-sitecore/constants/index.js +++ b/upload-api/migration-sitecore/constants/index.js @@ -45,6 +45,6 @@ const MIGRATION_DATA_CONFIG = { - module.exports = { - MIGRATION_DATA_CONFIG - }; \ No newline at end of file +module.exports = { + MIGRATION_DATA_CONFIG +}; diff --git a/upload-api/migration-sitecore/index.d.ts b/upload-api/migration-sitecore/index.d.ts index b621b558c..d304c0d33 100644 --- a/upload-api/migration-sitecore/index.d.ts +++ b/upload-api/migration-sitecore/index.d.ts @@ -1,4 +1,4 @@ // eslint-disable-next-line @typescript-eslint/no-var-requires -import contentTypes from "./libs/contenttypes.js"; +import contentTypes from './libs/contenttypes.js'; -export default contentTypes; \ No newline at end of file +export default contentTypes; diff --git a/upload-api/migration-sitecore/index.js b/upload-api/migration-sitecore/index.js index 10c85354b..ce84e6dae 100644 --- a/upload-api/migration-sitecore/index.js +++ b/upload-api/migration-sitecore/index.js @@ -1,19 +1,19 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ - -const contentTypes = require('./libs/contenttypes.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires -const ExtractConfiguration = require('./libs/configuration.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires -const reference = require('./libs/reference.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires -const ExtractFiles = require('./libs/convert.js'); -// eslint-disable-next-line @typescript-eslint/no-var-requires -const extractLocales = require('./libs/extractLocales.js'); - -module.exports = { - contentTypes, - ExtractConfiguration, - reference, - ExtractFiles, - extractLocales -}; +/* eslint-disable @typescript-eslint/no-var-requires */ + +const contentTypes = require('./libs/contenttypes.js'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const ExtractConfiguration = require('./libs/configuration.js'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const reference = require('./libs/reference.js'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const ExtractFiles = require('./libs/convert.js'); +// eslint-disable-next-line @typescript-eslint/no-var-requires +const extractLocales = require('./libs/extractLocales.js'); + +module.exports = { + contentTypes, + ExtractConfiguration, + reference, + ExtractFiles, + extractLocales +}; diff --git a/upload-api/migration-sitecore/libs/contenttypes.js b/upload-api/migration-sitecore/libs/contenttypes.js index 234819878..3fa2e954b 100644 --- a/upload-api/migration-sitecore/libs/contenttypes.js +++ b/upload-api/migration-sitecore/libs/contenttypes.js @@ -1,687 +1,689 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ -const path = require('path'); -const _ = require('lodash'); -const read = require('fs-readdir-recursive'); -const helper = require('../utils/helper'); -const restrictedUid = require('../utils'); -const { MIGRATION_DATA_CONFIG } = require('../constants/index'); - -const extraField = 'title'; -const configChecker = path?.join('content', 'Common', 'Configuration'); -const append = 'a'; -let config = {}; - -const { - DATA_MAPPER_DIR, - DATA_MAPPER_CONFIG_FILE, - DATA_MAPPER_CONFIG_TREE_FILE, - CONTENT_TYPES_DIR_NAME -} = MIGRATION_DATA_CONFIG; - -function isKeyPresent(keyToFind, timeZones) { - return timeZones?.some?.((timeZone) => Object?.keys?.(timeZone)?.includes?.(keyToFind)); -} - -const createTemplate = ({ components }) => { - components.item.$.field = components?.item?.fields?.field; - return components?.item?.$; -}; - -function startsWithNumber(str) { - return /^\d/.test(str); -} - -const uidCorrector = ({ uid }) => { - if (startsWithNumber(uid)) { - return `${append}_${_.replace(uid, new RegExp('[ -]', 'g'), '_')?.toLowerCase()}`?.replace?.( - '$', - '' - ); - } - const newUid = _.replace(uid, new RegExp('[ -]', 'g'), '_')?.toLowerCase(); - return newUid?.replace?.('$', ''); -}; - -const templatesComponents = ({ path: newPath }) => { - const fields = []; - for (let i = 0; i < newPath?.length; i++) { - const allFields = []; - const allPaths = read(newPath?.[i]?.pth); - for (let j = 0; j < allPaths?.length; j++) { - if (allPaths?.[j]?.endsWith('data.json')) { - const innerField = []; - const components = helper.readFile(path?.join?.(newPath?.[i]?.pth, allPaths?.[j])); - const data = components?.item?.$ ?? {}; - components?.item?.fields?.field.forEach((item) => { - if (item?.$?.key === 'type' || item?.$?.key === 'source' || item?.$?.key === extraField) { - innerField.push({ - content: item.content, - ...item.$ - }); - } - }); - if (innerField?.length) { - data.fields = innerField; - allFields.push(data); - } - } - } - fields?.push({ meta: newPath?.[i]?.obj?.item?.$, schema: allFields }); - } - return fields; -}; - -const templateStandardValues = ({ components }) => { - const standardValues = []; - const data = components?.item?.$ ?? {}; - components?.item?.fields?.field.forEach((item) => { - if (!item?.$?.key.includes('__')) { - standardValues.push({ - content: item.content, - ...item.$ - }); - } - }); - data.fields = standardValues; - return data; -}; - -const contentTypeKeyMapper = ({ template, contentType, contentTypeKey = 'contentTypeKey' }) => { - let keyMapper = {}; - const keys = helper.readFile( - path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR, `${contentTypeKey}.json`) - ); - if (keys) { - keyMapper = keys; - } - keyMapper[template?.id] = contentType?.uid; - helper.writeFile( - path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR), - JSON.stringify(keyMapper, null, 4), - contentTypeKey, - (err) => { - if (err) throw err; - } - ); -}; - -const ContentTypeSchema = ({ - type, - name, - uid, - default_value = '', - id, - choices = [], - sourLet, - sitecoreKey, - affix -}) => { - const isPresent = restrictedUid?.find((item) => item === uid); - if (isPresent) { - uid = `${affix}_${uid}`; - } - switch (type) { - case 'Single-Line Text': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'single_line_text', - backupFieldType: 'single_line_text', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - case 'Checkbox': { - default_value = default_value === '1' ? true : false; - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'boolean', - backupFieldType: 'boolean', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - case 'Rich Text': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'json', - backupFieldType: 'json', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - - case 'Droplist': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'dropdown', - backupFieldType: 'dropdown', - backupFieldUid: uid, - advanced: { - options: choices, - default_value: default_value !== '' ? default_value : null, - Multiple: false - } - }; - } - case 'Image': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'file', - backupFieldType: 'file', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - case 'General Link': - case 'Internal Link': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'link', - backupFieldType: 'link', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - - case 'Multi-Line Text': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'multi_line_text', - backupFieldType: 'multi_line_text', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - - case 'Integer': - case 'Number': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'number', - backupFieldType: 'number', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - - case 'Date': - case 'Time': { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'isodate', - backupFieldType: 'isodate', - backupFieldUid: uid, - advanced: { default_value: default_value !== '' ? default_value : null } - }; - } - - case 'Grouped Droplist': { - if (choices?.length) { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'dropdown', - backupFieldType: 'dropdown', - backupFieldUid: uid, - advanced: { - options: choices, - default_value: default_value !== '' ? default_value : null, - Multiple: false - } - }; - } - break; - } - case 'Treelist': { - if (sourLet?.key !== 'source') { - return { - id: id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'reference', - backupFieldUid: uid, - backupFieldType: 'reference' - }; - } - break; - } - default: { - return { - id, - uid: sitecoreKey, - otherCmsField: name, - otherCmsType: type, - contentstackField: name, - contentstackFieldUid: uid, - contentstackFieldType: 'reference', - backupFieldUid: uid, - backupFieldType: 'reference' - }; - } - } -}; - -function makeUnique({ data }) { - const newData = data; - let tempMapping = {}; - if (newData?.[0]?.key) { - newData?.forEach((choice) => { - if (choice?.key) { - if (!tempMapping?.[choice?.value]) { - tempMapping[choice?.value] = []; - } - tempMapping[choice?.value].push(choice?.key); - } - }); - const result = Object?.entries(tempMapping).map(([value, keys]) => { - return { - key: keys?.join('/'), - value: value - }; - }); - const newValue = []; - result?.forEach((item) => { - if (item?.key === undefined) { - newValue?.push({ ...item, key: item?.value }); - } else { - newValue?.push(item); - } - }); - return newValue; - } else { - let uniqueValues = []; - const result = data?.filter((item) => { - if (uniqueValues?.includes(item?.value)) { - return false; - } else { - uniqueValues?.push(item?.value); - return true; - } - }); - const newValue = []; - result?.forEach((item) => { - if (item?.key === undefined) { - newValue?.push({ ...item, key: item?.value }); - } else { - newValue?.push(item); - } - }); - return newValue; - } -} - -const groupFlat = (data, item) => { - const flat = []; - if (data?.data_type === 'group' && data?.schema?.[0] !== undefined) { - const group = { - uid: item?.meta?.key, - otherCmsField: item?.meta?.name, - otherCmsType: 'Group', - contentstackField: item?.meta?.name, - contentstackFieldUid: uidCorrector({ uid: item?.meta?.key }), - contentstackFieldType: 'group', - backupFieldType: 'group' - }; - flat?.push(group); - data?.schema?.forEach((element) => { - const obj = { - ...element, - uid: `${item?.meta?.key}.${element?.uid}`, - otherCmsField: `${item?.meta?.name} > ${element?.otherCmsField}`, - contentstackField: `${item?.meta?.name} > ${element?.contentstackField}`, - contentstackFieldUid: `${uidCorrector({ uid: item?.meta?.key })}.${element?.contentstackFieldUid}`, - backupFieldUid: `${uidCorrector({ uid: item?.meta?.key })}.${element?.contentstackFieldUid}` - }; - flat?.push(obj); - }); - } - return flat; -}; - -const contentTypeMapper = ({ - components, - standardValues, - content_type, - basePath, - sitecore_folder, - affix -}) => { - const source = helper.readFile( - path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR, DATA_MAPPER_CONFIG_FILE) - ); - const sourceTree = helper.readFile( - path.join( - process.cwd(), - MIGRATION_DATA_CONFIG.DATA, - DATA_MAPPER_DIR, - DATA_MAPPER_CONFIG_TREE_FILE - ) - ); - let mainSchema = []; - components?.forEach((item) => { - if (item?.schema?.length) { - const groupSchema = { - data_type: 'group', - display_name: item?.meta?.name, - field_metadata: {}, - schema: [], - uid: uidCorrector({ uid: item?.meta?.key }), - multiple: true, - mandatory: false, - unique: false - }; - for (let i = 0; i < item?.schema?.length; i++) { - const field = item?.schema?.[i]; - const appendStandardValues = standardValues?.fields?.find( - (item) => item?.key === field?.key - ); - if (appendStandardValues) { - field?.fields?.forEach((item) => { - if (item?.content === appendStandardValues?.type) { - item.standardValues = appendStandardValues; - } - }); - } - let compType = {}; - let sourLet = {}; - let sourceType = []; - let advanced = false; - let name = field?.name; - field?.fields?.forEach((item) => { - if (item?.key === 'type') { - compType = item; - } - if (item?.key === 'source') { - sourLet = item; - if (compType?.content === 'Droplink') { - if (sourceTree) { - if (item?.content?.includes(configChecker)) { - sourceType = makeUnique({ data: sourceTree?.[item?.content] }); - compType.content = 'Droplist'; - if (isKeyPresent('key', sourceType)) { - advanced = true; - } - } else { - console.log( - '🚀 ~ file: contenttypes.js:305 ~ field?.fields?.forEach ~ item?.content:', - item?.content - ); - } - } else { - console.log( - '🚀 ~ file: contenttypes.js:371 ~ field?.fields?.forEach ~ compType:', - compType?.standardValues?.key - ); - } - } else { - if (source) { - if (item?.content?.includes('datasource=')) { - const gUid = item?.content?.split('}')?.[0]?.replace('datasource={', ''); - if (gUid) { - const dataSourcePaths = read( - path?.join?.(sitecore_folder, 'master', 'sitecore', 'content', 'Common') - ); - let isDataSourcePresent = dataSourcePaths?.find((sur) => - sur?.includes(`{${gUid}}`) - ); - isDataSourcePresent = isDataSourcePresent?.split(`{${gUid}}`)?.[0]; - if (isDataSourcePresent) { - const optionsPath = read( - path?.join?.( - sitecore_folder, - 'master', - 'sitecore', - 'content', - 'Common', - isDataSourcePresent - ) - ); - const refName = []; - optionsPath?.forEach((newPath) => { - if (newPath?.endsWith('data.json')) { - const data = helper.readFile( - path?.join?.( - sitecore_folder, - 'master', - 'sitecore', - 'content', - 'Common', - isDataSourcePresent, - newPath - ) - ); - if (data?.item?.$?.template) { - refName.push(data?.item?.$?.template); - } - } - }); - if (refName?.length) { - const unique = [...new Set(refName)]; - contentTypeKeyMapper({ - template: { id: content_type?.contentstackUid }, - contentType: { uid: { name, uid: field?.key, unique } }, - contentTypeKey: 'treeListRef' - }); - } - } - } - } else { - sourceType = makeUnique({ data: source?.[item?.content] }); - if (isKeyPresent('key', sourceType)) { - advanced = true; - } - } - } - } - } - if (item?.key === extraField) { - if (item?.content && item?.content !== '') { - name = item?.content; - } - } - }); - if (compType?.content !== 'Droptree') { - groupSchema?.schema?.push( - ContentTypeSchema({ - name, - uid: uidCorrector({ uid: field?.key }), - type: compType?.content, - default_value: compType?.standardValues?.content, - id: field?.id, - choices: sourceType?.slice(0, config?.plan?.dropdown?.optionLimit - 2 ?? 98), - advanced, - sourLet, - sitecoreKey: field?.key, - isFromMapper: true, - affix - }) - ); - } - } - mainSchema?.push(...groupFlat(groupSchema, item)); - } - const isUrlfound = mainSchema?.find( - (rt) => rt?.contentstackFieldUid?.toLowerCase?.() === 'url' - ); - if (isUrlfound === undefined) { - mainSchema?.unshift({ - uid: 'url', - otherCmsField: 'url', - otherCmsType: 'text', - contentstackField: 'Url', - contentstackFieldUid: 'url', - contentstackFieldType: 'url', - backupFieldType: 'url', - backupFieldUid: 'url' - }); - } - const isPresent = mainSchema?.find( - (item) => item?.contentstackFieldUid?.toLowerCase?.() === 'title' - ); - if (isPresent === undefined) { - mainSchema.unshift({ - uid: 'title', - otherCmsField: 'title', - otherCmsType: 'text', - contentstackField: 'Title', - contentstackFieldUid: 'title', - contentstackFieldType: 'text', - backupFieldType: 'text', - backupFieldUid: 'title' - }); - } - }); - return mainSchema; -}; - -const contentTypeMaker = ({ template, basePath, sitecore_folder, affix }) => { - const content_type = { - id: template?.id, - status: 1, - otherCmsTitle: template?.name, - otherCmsUid: template?.key, - isUpdated: false, - updateAt: '', - contentstackTitle: template?.name, - contentstackUid: uidCorrector({ uid: template?.key }) - }; - template?.field?.forEach((item) => { - if (item?.$?.key === '__base template' && item?.$?.type === 'tree list') { - contentTypeKeyMapper({ - template, - contentType: { uid: { ...item?.$, content: item?.content } }, - contentTypeKey: 'base' - }); - } - }); - content_type.fieldMapping = contentTypeMapper({ - components: template?.components, - standardValues: template?.standardValues, - content_type, - basePath, - sitecore_folder, - affix - }); - return content_type; -}; - -function findExactPath(path, searchTerm) { - return path?.endsWith(searchTerm); -} - -function singleContentTypeCreate({ templatePaths, globalPath, sitecore_folder, affix }) { - const newPath = read(templatePaths); - const templatesComponentsPath = []; - let templatesStandaedValuePath = {}; - let templatesMetaDataPath = {}; - for (let i = 0; i < newPath?.length; i++) { - if (findExactPath(newPath?.[i], 'data.json')) { - const data = helper?.readFile(path?.join?.(templatePaths, newPath?.[i])); - if (data?.item?.$?.template === 'template section') { - templatesComponentsPath?.push({ - pth: path?.join?.(templatePaths, newPath?.[i] ?? '')?.split('/{')?.[0], - obj: data - }); - } else if (data?.item?.$?.template === 'template') { - templatesMetaDataPath = data; - } else if (data?.item?.$?.key?.includes?.('standard values')) { - templatesStandaedValuePath = data; - } - } - } - const template = createTemplate({ components: templatesMetaDataPath }); - template.components = templatesComponents({ - path: templatesComponentsPath, - basePath: templatePaths - }); - template.standardValues = templateStandardValues({ components: templatesStandaedValuePath }); - const contentType = contentTypeMaker({ template, basePath: globalPath, sitecore_folder, affix }); - if (contentType?.fieldMapping?.length) { - helper?.writeFile( - path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, CONTENT_TYPES_DIR_NAME), - JSON.stringify(contentType, null, 4), - contentType?.contentstackUid, - (err) => { - if (err) throw err; - } - ); - contentTypeKeyMapper({ template, contentType: { uid: contentType?.contentstackUid } }); - } - return true; -} - -function ExtractContentTypes(sitecore_folder, affix, configData) { - config = configData; - const folder = read(sitecore_folder); - const templatePaths = []; - for (let i = 0; i < folder?.length; i++) { - if (folder?.[i]?.includes('templates') && folder?.[i]?.endsWith('data.json')) { - const data = helper?.readFile(path?.join?.(sitecore_folder, folder?.[i])); - if (data?.item?.$?.template === 'template') { - templatePaths?.push(path?.join?.(sitecore_folder, folder?.[i])?.split('/{')?.[0]); - } - } - } - if (templatePaths?.length) { - const unique = [...new Set(templatePaths)]; - unique?.forEach((item) => { - singleContentTypeCreate({ templatePaths: item, globalPath: folder, sitecore_folder, affix }); - }); - } else { - throw { message: 'Templates Not Found.' }; - } -} - -module.exports = ExtractContentTypes; +/* eslint-disable @typescript-eslint/no-var-requires */ +const path = require('path'); +const _ = require('lodash'); +const read = require('fs-readdir-recursive'); +const helper = require('../utils/helper'); +const restrictedUid = require('../utils'); +const { MIGRATION_DATA_CONFIG } = require('../constants/index'); + +const extraField = 'title'; +const configChecker = path?.join('content', 'Common', 'Configuration'); +const append = 'a'; +let config = {}; + +const { + DATA_MAPPER_DIR, + DATA_MAPPER_CONFIG_FILE, + DATA_MAPPER_CONFIG_TREE_FILE, + CONTENT_TYPES_DIR_NAME +} = MIGRATION_DATA_CONFIG; + +function isKeyPresent(keyToFind, timeZones) { + return timeZones?.some?.((timeZone) => Object?.keys?.(timeZone)?.includes?.(keyToFind)); +} + +const createTemplate = ({ components }) => { + components.item.$.field = components?.item?.fields?.field; + return components?.item?.$; +}; + +function startsWithNumber(str) { + return /^\d/.test(str); +} + +const uidCorrector = ({ uid }) => { + if (startsWithNumber(uid)) { + return `${append}_${_.replace(uid, new RegExp('[ -]', 'g'), '_')?.toLowerCase()}`?.replace?.( + '$', + '' + ); + } + const newUid = _.replace(uid, new RegExp('[ -]', 'g'), '_')?.toLowerCase(); + return newUid?.replace?.('$', ''); +}; + +const templatesComponents = ({ path: newPath }) => { + const fields = []; + for (let i = 0; i < newPath?.length; i++) { + const allFields = []; + const allPaths = read(newPath?.[i]?.pth); + for (let j = 0; j < allPaths?.length; j++) { + if (allPaths?.[j]?.endsWith('data.json')) { + const innerField = []; + const components = helper.readFile(path?.join?.(newPath?.[i]?.pth, allPaths?.[j])); + const data = components?.item?.$ ?? {}; + components?.item?.fields?.field.forEach((item) => { + if (item?.$?.key === 'type' || item?.$?.key === 'source' || item?.$?.key === extraField) { + innerField.push({ + content: item.content, + ...item.$ + }); + } + }); + if (innerField?.length) { + data.fields = innerField; + allFields.push(data); + } + } + } + fields?.push({ meta: newPath?.[i]?.obj?.item?.$, schema: allFields }); + } + return fields; +}; + +const templateStandardValues = ({ components }) => { + const standardValues = []; + const data = components?.item?.$ ?? {}; + components?.item?.fields?.field.forEach((item) => { + if (!item?.$?.key.includes('__')) { + standardValues.push({ + content: item.content, + ...item.$ + }); + } + }); + data.fields = standardValues; + return data; +}; + +const contentTypeKeyMapper = ({ template, contentType, contentTypeKey = 'contentTypeKey' }) => { + let keyMapper = {}; + const keys = helper.readFile( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR, `${contentTypeKey}.json`) + ); + if (keys) { + keyMapper = keys; + } + keyMapper[template?.id] = contentType?.uid; + helper.writeFile( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR), + JSON.stringify(keyMapper, null, 4), + contentTypeKey, + (err) => { + if (err) throw err; + } + ); +}; + +const ContentTypeSchema = ({ + type, + name, + uid, + default_value = '', + id, + choices = [], + sourLet, + sitecoreKey, + affix +}) => { + const isPresent = restrictedUid?.find((item) => item === uid); + if (isPresent) { + uid = `${affix}_${uid}`; + } + switch (type) { + case 'Single-Line Text': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'single_line_text', + backupFieldType: 'single_line_text', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + case 'Checkbox': { + default_value = default_value === '1' ? true : false; + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'boolean', + backupFieldType: 'boolean', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + case 'Rich Text': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'json', + backupFieldType: 'json', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + + case 'Droplist': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'dropdown', + backupFieldType: 'dropdown', + backupFieldUid: uid, + advanced: { + options: choices, + default_value: default_value !== '' ? default_value : null, + Multiple: false + } + }; + } + case 'Image': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'file', + backupFieldType: 'file', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + case 'General Link': + case 'Internal Link': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'link', + backupFieldType: 'link', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + + case 'Multi-Line Text': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'multi_line_text', + backupFieldType: 'multi_line_text', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + + case 'Integer': + case 'Number': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'number', + backupFieldType: 'number', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + + case 'Date': + case 'Time': { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'isodate', + backupFieldType: 'isodate', + backupFieldUid: uid, + advanced: { default_value: default_value !== '' ? default_value : null } + }; + } + + case 'Grouped Droplist': { + if (choices?.length) { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'dropdown', + backupFieldType: 'dropdown', + backupFieldUid: uid, + advanced: { + options: choices, + default_value: default_value !== '' ? default_value : null, + Multiple: false + } + }; + } + break; + } + case 'Treelist': { + if (sourLet?.key !== 'source') { + return { + id: id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'reference', + backupFieldUid: uid, + backupFieldType: 'reference' + }; + } + break; + } + default: { + return { + id, + uid: sitecoreKey, + otherCmsField: name, + otherCmsType: type, + contentstackField: name, + contentstackFieldUid: uid, + contentstackFieldType: 'reference', + backupFieldUid: uid, + backupFieldType: 'reference' + }; + } + } +}; + +function makeUnique({ data }) { + const newData = data; + let tempMapping = {}; + if (newData?.[0]?.key) { + newData?.forEach((choice) => { + if (choice?.key) { + if (!tempMapping?.[choice?.value]) { + tempMapping[choice?.value] = []; + } + tempMapping[choice?.value].push(choice?.key); + } + }); + const result = Object?.entries(tempMapping).map(([value, keys]) => { + return { + key: keys?.join('/'), + value: value + }; + }); + const newValue = []; + result?.forEach((item) => { + if (item?.key === undefined) { + newValue?.push({ ...item, key: item?.value }); + } else { + newValue?.push(item); + } + }); + return newValue; + } else { + let uniqueValues = []; + const result = data?.filter((item) => { + if (uniqueValues?.includes(item?.value)) { + return false; + } else { + uniqueValues?.push(item?.value); + return true; + } + }); + const newValue = []; + result?.forEach((item) => { + if (item?.key === undefined) { + newValue?.push({ ...item, key: item?.value }); + } else { + newValue?.push(item); + } + }); + return newValue; + } +} + +const groupFlat = (data, item) => { + const flat = []; + if (data?.data_type === 'group' && data?.schema?.[0] !== undefined) { + const group = { + uid: item?.meta?.key, + otherCmsField: item?.meta?.name, + otherCmsType: 'Group', + contentstackField: item?.meta?.name, + contentstackFieldUid: uidCorrector({ uid: item?.meta?.key }), + contentstackFieldType: 'group', + backupFieldType: 'group' + }; + flat?.push(group); + data?.schema?.forEach((element) => { + const obj = { + ...element, + uid: `${item?.meta?.key}.${element?.uid}`, + otherCmsField: `${item?.meta?.name} > ${element?.otherCmsField}`, + contentstackField: `${item?.meta?.name} > ${element?.contentstackField}`, + contentstackFieldUid: `${uidCorrector({ uid: item?.meta?.key })}.${ + element?.contentstackFieldUid + }`, + backupFieldUid: `${uidCorrector({ uid: item?.meta?.key })}.${element?.contentstackFieldUid}` + }; + flat?.push(obj); + }); + } + return flat; +}; + +const contentTypeMapper = ({ + components, + standardValues, + content_type, + basePath, + sitecore_folder, + affix +}) => { + const source = helper.readFile( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, DATA_MAPPER_DIR, DATA_MAPPER_CONFIG_FILE) + ); + const sourceTree = helper.readFile( + path.join( + process.cwd(), + MIGRATION_DATA_CONFIG.DATA, + DATA_MAPPER_DIR, + DATA_MAPPER_CONFIG_TREE_FILE + ) + ); + let mainSchema = []; + components?.forEach((item) => { + if (item?.schema?.length) { + const groupSchema = { + data_type: 'group', + display_name: item?.meta?.name, + field_metadata: {}, + schema: [], + uid: uidCorrector({ uid: item?.meta?.key }), + multiple: true, + mandatory: false, + unique: false + }; + for (let i = 0; i < item?.schema?.length; i++) { + const field = item?.schema?.[i]; + const appendStandardValues = standardValues?.fields?.find( + (item) => item?.key === field?.key + ); + if (appendStandardValues) { + field?.fields?.forEach((item) => { + if (item?.content === appendStandardValues?.type) { + item.standardValues = appendStandardValues; + } + }); + } + let compType = {}; + let sourLet = {}; + let sourceType = []; + let advanced = false; + let name = field?.name; + field?.fields?.forEach((item) => { + if (item?.key === 'type') { + compType = item; + } + if (item?.key === 'source') { + sourLet = item; + if (compType?.content === 'Droplink') { + if (sourceTree) { + if (item?.content?.includes(configChecker)) { + sourceType = makeUnique({ data: sourceTree?.[item?.content] }); + compType.content = 'Droplist'; + if (isKeyPresent('key', sourceType)) { + advanced = true; + } + } else { + console.log( + '🚀 ~ file: contenttypes.js:305 ~ field?.fields?.forEach ~ item?.content:', + item?.content + ); + } + } else { + console.log( + '🚀 ~ file: contenttypes.js:371 ~ field?.fields?.forEach ~ compType:', + compType?.standardValues?.key + ); + } + } else { + if (source) { + if (item?.content?.includes('datasource=')) { + const gUid = item?.content?.split('}')?.[0]?.replace('datasource={', ''); + if (gUid) { + const dataSourcePaths = read( + path?.join?.(sitecore_folder, 'master', 'sitecore', 'content', 'Common') + ); + let isDataSourcePresent = dataSourcePaths?.find((sur) => + sur?.includes(`{${gUid}}`) + ); + isDataSourcePresent = isDataSourcePresent?.split(`{${gUid}}`)?.[0]; + if (isDataSourcePresent) { + const optionsPath = read( + path?.join?.( + sitecore_folder, + 'master', + 'sitecore', + 'content', + 'Common', + isDataSourcePresent + ) + ); + const refName = []; + optionsPath?.forEach((newPath) => { + if (newPath?.endsWith('data.json')) { + const data = helper.readFile( + path?.join?.( + sitecore_folder, + 'master', + 'sitecore', + 'content', + 'Common', + isDataSourcePresent, + newPath + ) + ); + if (data?.item?.$?.template) { + refName.push(data?.item?.$?.template); + } + } + }); + if (refName?.length) { + const unique = [...new Set(refName)]; + contentTypeKeyMapper({ + template: { id: content_type?.contentstackUid }, + contentType: { uid: { name, uid: field?.key, unique } }, + contentTypeKey: 'treeListRef' + }); + } + } + } + } else { + sourceType = makeUnique({ data: source?.[item?.content] }); + if (isKeyPresent('key', sourceType)) { + advanced = true; + } + } + } + } + } + if (item?.key === extraField) { + if (item?.content && item?.content !== '') { + name = item?.content; + } + } + }); + if (compType?.content !== 'Droptree') { + groupSchema?.schema?.push( + ContentTypeSchema({ + name, + uid: uidCorrector({ uid: field?.key }), + type: compType?.content, + default_value: compType?.standardValues?.content, + id: field?.id, + choices: sourceType?.slice(0, config?.plan?.dropdown?.optionLimit - 2 ?? 98), + advanced, + sourLet, + sitecoreKey: field?.key, + isFromMapper: true, + affix + }) + ); + } + } + mainSchema?.push(...groupFlat(groupSchema, item)); + } + const isUrlfound = mainSchema?.find( + (rt) => rt?.contentstackFieldUid?.toLowerCase?.() === 'url' + ); + if (isUrlfound === undefined) { + mainSchema?.unshift({ + uid: 'url', + otherCmsField: 'url', + otherCmsType: 'text', + contentstackField: 'Url', + contentstackFieldUid: 'url', + contentstackFieldType: 'url', + backupFieldType: 'url', + backupFieldUid: 'url' + }); + } + const isPresent = mainSchema?.find( + (item) => item?.contentstackFieldUid?.toLowerCase?.() === 'title' + ); + if (isPresent === undefined) { + mainSchema.unshift({ + uid: 'title', + otherCmsField: 'title', + otherCmsType: 'text', + contentstackField: 'Title', + contentstackFieldUid: 'title', + contentstackFieldType: 'text', + backupFieldType: 'text', + backupFieldUid: 'title' + }); + } + }); + return mainSchema; +}; + +const contentTypeMaker = ({ template, basePath, sitecore_folder, affix }) => { + const content_type = { + id: template?.id, + status: 1, + otherCmsTitle: template?.name, + otherCmsUid: template?.key, + isUpdated: false, + updateAt: '', + contentstackTitle: template?.name, + contentstackUid: uidCorrector({ uid: template?.key }) + }; + template?.field?.forEach((item) => { + if (item?.$?.key === '__base template' && item?.$?.type === 'tree list') { + contentTypeKeyMapper({ + template, + contentType: { uid: { ...item?.$, content: item?.content } }, + contentTypeKey: 'base' + }); + } + }); + content_type.fieldMapping = contentTypeMapper({ + components: template?.components, + standardValues: template?.standardValues, + content_type, + basePath, + sitecore_folder, + affix + }); + return content_type; +}; + +function findExactPath(path, searchTerm) { + return path?.endsWith(searchTerm); +} + +function singleContentTypeCreate({ templatePaths, globalPath, sitecore_folder, affix }) { + const newPath = read(templatePaths); + const templatesComponentsPath = []; + let templatesStandaedValuePath = {}; + let templatesMetaDataPath = {}; + for (let i = 0; i < newPath?.length; i++) { + if (findExactPath(newPath?.[i], 'data.json')) { + const data = helper?.readFile(path?.join?.(templatePaths, newPath?.[i])); + if (data?.item?.$?.template === 'template section') { + templatesComponentsPath?.push({ + pth: path?.join?.(templatePaths, newPath?.[i] ?? '')?.split('/{')?.[0], + obj: data + }); + } else if (data?.item?.$?.template === 'template') { + templatesMetaDataPath = data; + } else if (data?.item?.$?.key?.includes?.('standard values')) { + templatesStandaedValuePath = data; + } + } + } + const template = createTemplate({ components: templatesMetaDataPath }); + template.components = templatesComponents({ + path: templatesComponentsPath, + basePath: templatePaths + }); + template.standardValues = templateStandardValues({ components: templatesStandaedValuePath }); + const contentType = contentTypeMaker({ template, basePath: globalPath, sitecore_folder, affix }); + if (contentType?.fieldMapping?.length) { + helper?.writeFile( + path.join(process.cwd(), MIGRATION_DATA_CONFIG.DATA, CONTENT_TYPES_DIR_NAME), + JSON.stringify(contentType, null, 4), + contentType?.contentstackUid, + (err) => { + if (err) throw err; + } + ); + contentTypeKeyMapper({ template, contentType: { uid: contentType?.contentstackUid } }); + } + return true; +} + +function ExtractContentTypes(sitecore_folder, affix, configData) { + config = configData; + const folder = read(sitecore_folder); + const templatePaths = []; + for (let i = 0; i < folder?.length; i++) { + if (folder?.[i]?.includes('templates') && folder?.[i]?.endsWith('data.json')) { + const data = helper?.readFile(path?.join?.(sitecore_folder, folder?.[i])); + if (data?.item?.$?.template === 'template') { + templatePaths?.push(path?.join?.(sitecore_folder, folder?.[i])?.split('/{')?.[0]); + } + } + } + if (templatePaths?.length) { + const unique = [...new Set(templatePaths)]; + unique?.forEach((item) => { + singleContentTypeCreate({ templatePaths: item, globalPath: folder, sitecore_folder, affix }); + }); + } else { + throw { message: 'Templates Not Found.' }; + } +} + +module.exports = ExtractContentTypes; diff --git a/upload-api/migration-sitecore/libs/extractLocales.js b/upload-api/migration-sitecore/libs/extractLocales.js index 7b009de8b..230a67b9f 100644 --- a/upload-api/migration-sitecore/libs/extractLocales.js +++ b/upload-api/migration-sitecore/libs/extractLocales.js @@ -1,33 +1,32 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -const fs = require("fs"); -const path = require("path"); - +const fs = require('fs'); +const path = require('path'); const uniqueLanguages = new Set(); // Define uniqueLanguages globally or pass it as a parameter const extractLocales = (dir) => { - const items = fs?.readdirSync?.(dir, { withFileTypes: true }); + const items = fs?.readdirSync?.(dir, { withFileTypes: true }); - for (const item of items) { - const fullPath = path?.join?.(dir, item?.name); + for (const item of items) { + const fullPath = path?.join?.(dir, item?.name); - if (item?.isDirectory()) { - extractLocales?.(fullPath); // Proper recursion - } else if (item?.isFile() && item?.name === "data.json") { - try { - const rawData = fs?.readFileSync?.(fullPath, "utf8"); - const jsonData = JSON?.parse?.(rawData); - const language = jsonData?.item?.$?.language; + if (item?.isDirectory()) { + extractLocales?.(fullPath); // Proper recursion + } else if (item?.isFile() && item?.name === 'data.json') { + try { + const rawData = fs?.readFileSync?.(fullPath, 'utf8'); + const jsonData = JSON?.parse?.(rawData); + const language = jsonData?.item?.$?.language; - if (language) { - uniqueLanguages?.add?.(language); - } - } catch (error) { - console.error(`Error reading ${fullPath}:`, error?.message); - } + if (language) { + uniqueLanguages?.add?.(language); } + } catch (error) { + console.error(`Error reading ${fullPath}:`, error?.message); + } } - return uniqueLanguages; + } + return uniqueLanguages; }; -module.exports = extractLocales; \ No newline at end of file +module.exports = extractLocales; diff --git a/upload-api/migration-sitecore/package.json b/upload-api/migration-sitecore/package.json index 7dff6c8f0..a50f05c7e 100644 --- a/upload-api/migration-sitecore/package.json +++ b/upload-api/migration-sitecore/package.json @@ -1,43 +1,43 @@ -{ - "name": "migration-sitecore", - "version": "1.0.0", - "description": "", - "main": "index.js", - "typings": "index.d.ts", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/contentstack-expert-services/migration-wordpress-xml.git" - }, - "keywords": [], - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/contentstack-expert-services/migration-wordpress-xml/issues" - }, - "homepage": "https://github.com/contentstack-expert-services/migration-wordpress-xml#readme", - "dependencies": { - "@contentstack/json-rte-serializer": "^2.0.2", - "@sitecore-jss/sitecore-jss-cli": "^21.2.4", - "ansi-colors": "^4.1.3", - "axios": "^1.3.4", - "chalk": "^4.1.0", - "cheerio": "^1.0.0-rc.12", - "cli-progress": "^3.11.1", - "fs-readdir-recursive": "^1.1.0", - "inquirer": "^8.2.4", - "jsdom": "^19.0.0", - "lodash": "^4.17.21", - "mkdirp": "^1.0.4", - "moment": "^2.30.1", - "pdf-stream": "^1.3.2", - "request": "^2.88.2", - "rimraf": "^4.1.2", - "uid": "^2.0.2", - "when": "^3.7.8", - "winston": "^3.7.2", - "xml2js": "^0.4.23" - } -} \ No newline at end of file +{ + "name": "migration-sitecore", + "version": "1.0.0", + "description": "", + "main": "index.js", + "typings": "index.d.ts", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/contentstack-expert-services/migration-wordpress-xml.git" + }, + "keywords": [], + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/contentstack-expert-services/migration-wordpress-xml/issues" + }, + "homepage": "https://github.com/contentstack-expert-services/migration-wordpress-xml#readme", + "dependencies": { + "@contentstack/json-rte-serializer": "^2.0.2", + "@sitecore-jss/sitecore-jss-cli": "^21.2.4", + "ansi-colors": "^4.1.3", + "axios": "^1.3.4", + "chalk": "^4.1.0", + "cheerio": "^1.0.0-rc.12", + "cli-progress": "^3.11.1", + "fs-readdir-recursive": "^1.1.0", + "inquirer": "^8.2.4", + "jsdom": "^19.0.0", + "lodash": "^4.17.21", + "mkdirp": "^1.0.4", + "moment": "^2.30.1", + "pdf-stream": "^1.3.2", + "request": "^2.88.2", + "rimraf": "^4.1.2", + "uid": "^2.0.2", + "when": "^3.7.8", + "winston": "^3.7.2", + "xml2js": "^0.4.23" + } +} diff --git a/upload-api/migration-sitecore/utils/dateChanger.js b/upload-api/migration-sitecore/utils/dateChanger.js index 5ad1d8d8e..f9aeeaa16 100644 --- a/upload-api/migration-sitecore/utils/dateChanger.js +++ b/upload-api/migration-sitecore/utils/dateChanger.js @@ -8,5 +8,4 @@ function dateConverter({ inputDate }) { return null; } - -module.exports = dateConverter; \ No newline at end of file +module.exports = dateConverter; diff --git a/upload-api/migration-sitecore/utils/helper.js b/upload-api/migration-sitecore/utils/helper.js index 347ced6b2..cb28b25b4 100644 --- a/upload-api/migration-sitecore/utils/helper.js +++ b/upload-api/migration-sitecore/utils/helper.js @@ -1,62 +1,59 @@ -/* eslint-disable @typescript-eslint/no-var-requires */ -/** - * External module Dependencies. - */ -var fs = require("fs"); -var path = require("path"); -var mkdirp = require("mkdirp"); - -/** - * Internal module Dependencies. - */ - -// for checking XML file -exports.readXMLFile = function (filePath) { - let data; - if (fs.existsSync(filePath)) data = fs.readFileSync(filePath, "utf8"); - return data; -}; - -exports.readFile = function (filePath, parse) { - try { - parse = typeof parse == "undefined" ? true : parse; - filePath = path.resolve(filePath); - let data; - if (fs.existsSync(filePath)) { - data = parse ? JSON.parse(fs.readFileSync(filePath, "utf8")) : data; - return data; - } else { - return undefined; - } - } catch (err) { - console.log("🚀 ~ file: helper.js:29 ~ err:", err) - } -}; - -exports.writeFile = function (filePath, data, pathName) { - if (fs.existsSync(filePath)) { - filePath = path.resolve(`${filePath}/${pathName}`); - data = typeof data === "object" ? JSON.stringify(data) : data || "{}"; - fs.writeFileSync(`${filePath}.json`, data, "utf8"); - } else { - fs.mkdirSync( - filePath, - { recursive: true } - ); - filePath = path.resolve(`${filePath}/${pathName}`); - data = typeof data == "object" ? JSON.stringify(data) : data || "{}"; - fs.writeFileSync(`${filePath}.json`, data, "utf8"); - } -}; - -exports.appendFile = function (filePath, data) { - filePath = path.resolve(filePath); - fs.appendFileSync(filePath, data); -}; - -exports.makeDirectory = function () { - for (var key in arguments) { - var dirname = path.resolve(arguments[key]); - if (!fs.existsSync(dirname)) mkdirp.sync(dirname); - } -}; +/* eslint-disable @typescript-eslint/no-var-requires */ +/** + * External module Dependencies. + */ +var fs = require('fs'); +var path = require('path'); +var mkdirp = require('mkdirp'); + +/** + * Internal module Dependencies. + */ + +// for checking XML file +exports.readXMLFile = function (filePath) { + let data; + if (fs.existsSync(filePath)) data = fs.readFileSync(filePath, 'utf8'); + return data; +}; + +exports.readFile = function (filePath, parse) { + try { + parse = typeof parse == 'undefined' ? true : parse; + filePath = path.resolve(filePath); + let data; + if (fs.existsSync(filePath)) { + data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf8')) : data; + return data; + } else { + return undefined; + } + } catch (err) { + console.log('🚀 ~ file: helper.js:29 ~ err:', err); + } +}; + +exports.writeFile = function (filePath, data, pathName) { + if (fs.existsSync(filePath)) { + filePath = path.resolve(`${filePath}/${pathName}`); + data = typeof data === 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFileSync(`${filePath}.json`, data, 'utf8'); + } else { + fs.mkdirSync(filePath, { recursive: true }); + filePath = path.resolve(`${filePath}/${pathName}`); + data = typeof data == 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFileSync(`${filePath}.json`, data, 'utf8'); + } +}; + +exports.appendFile = function (filePath, data) { + filePath = path.resolve(filePath); + fs.appendFileSync(filePath, data); +}; + +exports.makeDirectory = function () { + for (var key in arguments) { + var dirname = path.resolve(arguments[key]); + if (!fs.existsSync(dirname)) mkdirp.sync(dirname); + } +}; diff --git a/upload-api/migration-sitecore/utils/index.js b/upload-api/migration-sitecore/utils/index.js index 519e6ac8d..f48d4a9be 100644 --- a/upload-api/migration-sitecore/utils/index.js +++ b/upload-api/migration-sitecore/utils/index.js @@ -1,73 +1,73 @@ const restrictedUid = [ - "api_key", - "built_io_application_user", - "built_io_application_user_role", - "built_io_audit_log", - "built_io_environment", - "built_io_extensions", - "built_io_installation_data", - "built_io_label", - "built_io_language", - "built_io_publish_queue", - "built_io_release", - "built_io_upload", - "cs_branches", - "org_uid", - "sys_asset", - "sys_metadata", - "_exists", - "_regex", - "*_ids", // Fields cannot have UID ending with "_ids." - "__indexes", - "__loc", - "__meta", - "__v", - "_id", - "_owner", - "_publish_locales", - "_shouldFilter", - "_shouldLean", - "_version", - "ACL", - "api_key", - "app_user_object_uid", - "applikation_id", - "built_io_upload", - "contentstackFilters", - "created_at", - "created_by", - "DEFAULT_ACL", - "deleted_at", - "dimension", - "domain", - "embedded_items", - "hook", - "id", - "inbuilt_class", - "isApplicationUser", - "isNew", - "isSystemUser", - "klass_id", - "locale", - "options", - "org_uid", - "publish_details", - "save", - "shard_account", - "shard_app", - "shard_random", - "SYS_ACL", - "sys_assets", - "sys_metadata", - "tags", - "tags_array", - "taxonomies", - "tenant_id", - "toJSON", - "uid", - "update", - "updated_at", - "updated_by" + 'api_key', + 'built_io_application_user', + 'built_io_application_user_role', + 'built_io_audit_log', + 'built_io_environment', + 'built_io_extensions', + 'built_io_installation_data', + 'built_io_label', + 'built_io_language', + 'built_io_publish_queue', + 'built_io_release', + 'built_io_upload', + 'cs_branches', + 'org_uid', + 'sys_asset', + 'sys_metadata', + '_exists', + '_regex', + '*_ids', // Fields cannot have UID ending with "_ids." + '__indexes', + '__loc', + '__meta', + '__v', + '_id', + '_owner', + '_publish_locales', + '_shouldFilter', + '_shouldLean', + '_version', + 'ACL', + 'api_key', + 'app_user_object_uid', + 'applikation_id', + 'built_io_upload', + 'contentstackFilters', + 'created_at', + 'created_by', + 'DEFAULT_ACL', + 'deleted_at', + 'dimension', + 'domain', + 'embedded_items', + 'hook', + 'id', + 'inbuilt_class', + 'isApplicationUser', + 'isNew', + 'isSystemUser', + 'klass_id', + 'locale', + 'options', + 'org_uid', + 'publish_details', + 'save', + 'shard_account', + 'shard_app', + 'shard_random', + 'SYS_ACL', + 'sys_assets', + 'sys_metadata', + 'tags', + 'tags_array', + 'taxonomies', + 'tenant_id', + 'toJSON', + 'uid', + 'update', + 'updated_at', + 'updated_by' ]; -module.exports = restrictedUid; \ No newline at end of file +module.exports = restrictedUid; diff --git a/upload-api/migration-wordpress/config/index.json b/upload-api/migration-wordpress/config/index.json index 661544d9f..5d097db39 100644 --- a/upload-api/migration-wordpress/config/index.json +++ b/upload-api/migration-wordpress/config/index.json @@ -10,7 +10,7 @@ "dirName": "content_types", "fileName": "contenttype.json", "masterfile": "contenttypes.json", - "schemaFile" : "schema.json" + "schemaFile": "schema.json" }, "authors": { "dirName": "authors", @@ -46,7 +46,6 @@ "dirName": "posts", "fileName": "en-us.json", "masterfile": "posts.json" - } } } diff --git a/upload-api/migration-wordpress/index.js b/upload-api/migration-wordpress/index.js index 830906da6..6b04f40f0 100644 --- a/upload-api/migration-wordpress/index.js +++ b/upload-api/migration-wordpress/index.js @@ -2,10 +2,10 @@ const extractContentTypes = require('./libs/content_types.js'); const contentTypeMaker = require('./libs/contenttypemapper.js'); -const extractLocale = require('./libs/extractLocale.js') +const extractLocale = require('./libs/extractLocale.js'); module.exports = { - extractContentTypes, - contentTypeMaker, + extractContentTypes, + contentTypeMaker, extractLocale }; diff --git a/upload-api/migration-wordpress/libs/extractLocale.js b/upload-api/migration-wordpress/libs/extractLocale.js index c4e73d739..c688380fd 100644 --- a/upload-api/migration-wordpress/libs/extractLocale.js +++ b/upload-api/migration-wordpress/libs/extractLocale.js @@ -3,7 +3,7 @@ const fs = require('fs'); /** * Extracts unique languages/locales from a WordPress exported JSON file. - * + * * @param {string} path - The file path to the WordPress JSON export. * @returns {string[]} - An array of unique language codes found in the JSON data. * @throws {Error} - Throws an error if the file cannot be read or parsed. @@ -23,7 +23,9 @@ const extractLocale = (path) => { const items = jsonData?.rss?.channel?.item || []; items.forEach((item) => { if (item['wp:postmeta']) { - const postMeta = Array.isArray(item['wp:postmeta']) ? item['wp:postmeta'] : [item['wp:postmeta']]; + const postMeta = Array.isArray(item['wp:postmeta']) + ? item['wp:postmeta'] + : [item['wp:postmeta']]; postMeta.forEach((meta) => { if (meta['wp:meta_key']?.toLowerCase() === 'language' && meta['wp:meta_value']) { uniqueLanguages.add(meta['wp:meta_value']); @@ -32,11 +34,10 @@ const extractLocale = (path) => { } }); - return [...uniqueLanguages]; + return [...uniqueLanguages]; } catch (err) { throw new Error(`Error reading JSON file: ${err.message}`); } }; - module.exports = extractLocale; diff --git a/upload-api/migration-wordpress/utils/helper.js b/upload-api/migration-wordpress/utils/helper.js index 4c7a81ac9..bd0f6dd40 100644 --- a/upload-api/migration-wordpress/utils/helper.js +++ b/upload-api/migration-wordpress/utils/helper.js @@ -1,10 +1,10 @@ /** * External module Dependencies. */ -var fs = require("fs"); -var path = require("path"); -var mkdirp = require("mkdirp"); -const xml2js = require("xml2js"); +var fs = require('fs'); +var path = require('path'); +var mkdirp = require('mkdirp'); +const xml2js = require('xml2js'); /** * Internal module Dependencies. */ @@ -12,7 +12,7 @@ const xml2js = require("xml2js"); // for checking XML file exports.readXMLFile = function (filePath) { var data; - if (fs.existsSync(filePath)) data = fs.readFileSync(filePath, "utf-8"); + if (fs.existsSync(filePath)) data = fs.readFileSync(filePath, 'utf-8'); return data; }; @@ -21,35 +21,34 @@ exports.readXMLFile = function (filePath) { exports.parseXmlToJson = async (xml) => { try { const parser = new xml2js.Parser({ - attrkey: "attributes", - charkey: "text", - explicitArray: false, + attrkey: 'attributes', + charkey: 'text', + explicitArray: false }); return await parser.parseStringPromise(xml); } catch (err) { console.log(chalk.red(`Error parsing XML: ${err.message}`)); } -} +}; exports.writeFileAsync = async function (filePath, data, tabSpaces) { filePath = path.resolve(filePath); - data = typeof data == "object" ? JSON.stringify(data, null, tabSpaces) : data || "{}"; - await fs.promises.writeFile(filePath, data, "utf-8"); + data = typeof data == 'object' ? JSON.stringify(data, null, tabSpaces) : data || '{}'; + await fs.promises.writeFile(filePath, data, 'utf-8'); }; exports.readFile = function (filePath, parse) { - parse = typeof parse == "undefined" ? true : parse; + parse = typeof parse == 'undefined' ? true : parse; filePath = path.resolve(filePath); var data; - if (fs.existsSync(filePath)) - data = parse ? JSON.parse(fs.readFileSync(filePath, "utf-8")) : data; + if (fs.existsSync(filePath)) data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data; return data; }; exports.writeFile = function (filePath, data) { filePath = path.resolve(filePath); - data = typeof data == "object" ? JSON.stringify(data) : data || "{}"; - fs.writeFileSync(filePath, data, "utf-8"); + data = typeof data == 'object' ? JSON.stringify(data) : data || '{}'; + fs.writeFileSync(filePath, data, 'utf-8'); }; exports.appendFile = function (filePath, data) { diff --git a/upload-api/migration-wordpress/utils/index.js b/upload-api/migration-wordpress/utils/index.js index cae85bd5f..4f7cd1f0f 100644 --- a/upload-api/migration-wordpress/utils/index.js +++ b/upload-api/migration-wordpress/utils/index.js @@ -1,48 +1,48 @@ const restrictedUid = [ - "uid", - "api_key", - "created_at", - "deleted_at", - "updated_at", - "tags_array", - "klass_id", - "applikation_id", - "*_ids", - "id", - "_id", - "ACL", - "SYS_ACL", - "DEFAULT_ACL", - "app_user_object_uid", - "built_io_upload", - "__loc", - "tags", - "_owner", - "_version", - "toJSON", - "save", - "update", - "domain", - "shard_account", - "shard_app", - "shard_random", - "hook", - "__indexes", - "__meta", - "created_by", - "updated_by", - "inbuilt_class", - "tenant_id", - "isSystemUser", - "isApplicationUser", - "isNew", - "_shouldLean", - "_shouldFilter", - "options", - "_version", - "__v", - "locale", - "publish_details" + 'uid', + 'api_key', + 'created_at', + 'deleted_at', + 'updated_at', + 'tags_array', + 'klass_id', + 'applikation_id', + '*_ids', + 'id', + '_id', + 'ACL', + 'SYS_ACL', + 'DEFAULT_ACL', + 'app_user_object_uid', + 'built_io_upload', + '__loc', + 'tags', + '_owner', + '_version', + 'toJSON', + 'save', + 'update', + 'domain', + 'shard_account', + 'shard_app', + 'shard_random', + 'hook', + '__indexes', + '__meta', + 'created_by', + 'updated_by', + 'inbuilt_class', + 'tenant_id', + 'isSystemUser', + 'isApplicationUser', + 'isNew', + '_shouldLean', + '_shouldFilter', + 'options', + '_version', + '__v', + 'locale', + 'publish_details' ]; -module.exports = restrictedUid; \ No newline at end of file +module.exports = restrictedUid; diff --git a/upload-api/package.json b/upload-api/package.json index a554beadc..2c949b21f 100644 --- a/upload-api/package.json +++ b/upload-api/package.json @@ -56,4 +56,4 @@ "prettier": "^3.3.3", "xml2js": "^0.6.2" } -} \ No newline at end of file +} diff --git a/upload-api/src/constants/index.ts b/upload-api/src/constants/index.ts index 5f0fc2f68..63904b359 100644 --- a/upload-api/src/constants/index.ts +++ b/upload-api/src/constants/index.ts @@ -36,10 +36,10 @@ export const HTTP_RESPONSE_HEADERS = { export const MIGRATION_DATA_CONFIG = { DATA: "cmsMigrationData", - BACKUP_DATA: "migration-data", - BACKUP_LOG_DIR: "logs", - BACKUP_FOLDER_NAME: "import", - BACKUP_FILE_NAME: "success.log", + BACKUP_DATA: 'migration-data', + BACKUP_LOG_DIR: 'logs', + BACKUP_FOLDER_NAME: 'import', + BACKUP_FILE_NAME: 'success.log', LOCALE_DIR_NAME: "locale", LOCALE_FILE_NAME: "locales.json", @@ -75,5 +75,5 @@ export const MIGRATION_DATA_CONFIG = { GLOBAL_FIELDS_DIR_NAME: "global_fields", GLOBAL_FIELDS_FILE_NAME: "globalfields.json", - EXPORT_INFO_FILE: "export-info.json" -} + EXPORT_INFO_FILE: 'export-info.json' +}; diff --git a/upload-api/src/models/types.ts b/upload-api/src/models/types.ts index 7039e1b53..c86ca5ca2 100644 --- a/upload-api/src/models/types.ts +++ b/upload-api/src/models/types.ts @@ -1,18 +1,18 @@ export interface Config { - plan: { - dropdown: { - optionLimit: number; - }; + plan: { + dropdown: { + optionLimit: number; }; - cmsType: string; - isLocalPath: boolean; - awsData: { - awsRegion: string; - awsAccessKeyId: string; - awsSecretAccessKey: string; - awsSessionToken: string; - bucketName: string; - buketKey: string; - }; - localPath: string; - } \ No newline at end of file + }; + cmsType: string; + isLocalPath: boolean; + awsData: { + awsRegion: string; + awsAccessKeyId: string; + awsSecretAccessKey: string; + awsSessionToken: string; + bucketName: string; + buketKey: string; + }; + localPath: string; +} diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index c11922192..f578eec28 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,7 +7,6 @@ import { Config } from '../../models/types'; const { extractContentTypes, createInitialMapper, extractLocale } = require('migration-contentful'); - const createContentfulMapper = async ( projectId: string | string[], app_token: string | string[], @@ -16,7 +15,7 @@ const createContentfulMapper = async ( ) => { try { const { localPath } = config; - const fetchedLocales:[]=await extractLocale(localPath) + const fetchedLocales: [] = await extractLocale(localPath); await extractContentTypes(localPath, affix); const initialMapper = await createInitialMapper(); @@ -30,14 +29,14 @@ const createContentfulMapper = async ( }, data: JSON.stringify(initialMapper) }; - const response = await axios.request(req) + const response = await axios.request(req); if (response?.data?.content_mapper?.length) { logger.info('Validation success:', { status: HTTP_CODES?.OK, message: HTTP_TEXTS?.MAPPER_SAVED }); } - + const mapperConfig = { method: 'post', maxBodyLength: Infinity, @@ -47,17 +46,17 @@ const createContentfulMapper = async ( 'Content-Type': 'application/json' }, data: { - locale:Array.from(fetchedLocales) - }, + locale: Array.from(fetchedLocales) + } }; - const mapRes = await axios.request(mapperConfig) - if(mapRes?.status==200){ + const mapRes = await axios.request(mapperConfig); + if (mapRes?.status == 200) { logger.info('Legacy CMS', { status: HTTP_CODES?.OK, - message: HTTP_TEXTS?.LOCALE_SAVED, + message: HTTP_TEXTS?.LOCALE_SAVED }); - } + } } catch (err: any) { console.error('🚀 ~ createContentfulMapper ~ err:', err?.response?.data ?? err); logger.warn('Validation error:', { diff --git a/upload-api/src/services/createMapper.ts b/upload-api/src/services/createMapper.ts index d1ed59c65..759d329a3 100644 --- a/upload-api/src/services/createMapper.ts +++ b/upload-api/src/services/createMapper.ts @@ -3,7 +3,13 @@ import createWordpressMapper from '../controllers/wordpress'; import { Config } from '../models/types'; import createContentfulMapper from './contentful'; -const createMapper = async (filePath: string = "", projectId: string | string[], app_token: string | string[], affix: string | string[], config: Config) => { +const createMapper = async ( + filePath: string = '', + projectId: string | string[], + app_token: string | string[], + affix: string | string[], + config: Config +) => { const CMSIdentifier = config?.cmsType?.toLowerCase(); switch (CMSIdentifier) { case 'sitecore': { @@ -27,4 +33,4 @@ const createMapper = async (filePath: string = "", projectId: string | string[], } }; -export default createMapper; \ No newline at end of file +export default createMapper; diff --git a/upload-api/src/services/fileProcessing.ts b/upload-api/src/services/fileProcessing.ts index 45bd66454..0e538a4e0 100644 --- a/upload-api/src/services/fileProcessing.ts +++ b/upload-api/src/services/fileProcessing.ts @@ -6,16 +6,21 @@ import config from '../config/index'; import logger from '../utils/logger.js'; import * as Cheerio from 'cheerio'; -const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: string, name :string) => { +const handleFileProcessing = async ( + fileExt: string, + zipBuffer: any, + cmsType: string, + name: string +) => { if (fileExt === 'zip') { const zip = new JSZip(); - await zip.loadAsync(zipBuffer); - if (await validator({ data: zip, type: cmsType, extension: fileExt })) { + await zip.loadAsync(zipBuffer); + if (await validator({ data: zip, type: cmsType, extension: fileExt })) { const isSaved = await saveZip(zip, name); if (isSaved) { logger.info('Validation success:', { status: HTTP_CODES?.OK, - message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL, + message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL }); return { status: HTTP_CODES?.OK, @@ -26,7 +31,7 @@ const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: st } else { logger.warn('Validation error:', { status: HTTP_CODES?.UNAUTHORIZED, - message: HTTP_TEXTS?.VALIDATION_ERROR, + message: HTTP_TEXTS?.VALIDATION_ERROR }); return { status: HTTP_CODES?.UNAUTHORIZED, @@ -35,11 +40,11 @@ const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: st }; } } else if (fileExt === 'xml') { - if (await validator({ data: zipBuffer, type: cmsType, extension: fileExt }) ) { + if (await validator({ data: zipBuffer, type: cmsType, extension: fileExt })) { const $ = Cheerio.load(zipBuffer, { xmlMode: true }); - const fixedXml = $.xml(); + const fixedXml = $.xml(); const parsedJson = await parseXmlToJson(fixedXml); - const isSaved = await saveJson(parsedJson,`${name}.json`); + const isSaved = await saveJson(parsedJson, `${name}.json`); if (isSaved) { logger.info('Validation success:', { status: HTTP_CODES?.OK, @@ -69,7 +74,7 @@ const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: st if (await validator({ data: jsonString, type: cmsType, extension: fileExt })) { logger.info('Validation success:', { status: HTTP_CODES?.OK, - message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL, + message: HTTP_TEXTS?.VALIDATION_SUCCESSFULL }); return { status: HTTP_CODES?.OK, @@ -79,7 +84,7 @@ const handleFileProcessing = async (fileExt: string, zipBuffer: any, cmsType: st } else { logger.warn('Validation error:', { status: HTTP_CODES?.UNAUTHORIZED, - message: HTTP_TEXTS?.VALIDATION_ERROR, + message: HTTP_TEXTS?.VALIDATION_ERROR }); return { status: HTTP_CODES?.UNAUTHORIZED, diff --git a/upload-api/tsconfig.json b/upload-api/tsconfig.json index dbb6d8bba..acddda238 100644 --- a/upload-api/tsconfig.json +++ b/upload-api/tsconfig.json @@ -32,7 +32,10 @@ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ - "typeRoots": ["./node_modules/@types", "./expree.d.ts"], /* Specify multiple folders that act like './node_modules/@types'. */ + "typeRoots": [ + "./node_modules/@types", + "./expree.d.ts" + ] /* Specify multiple folders that act like './node_modules/@types'. */, // "types": [], /* Specify type package names to be included without being referenced in a source file. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */