Skip to content

Commit b1d1563

Browse files
RohitKiniRohitKini
authored
Merge pull request #530 from contentstack/dev
Dev
2 parents 3c16405 + fcf48ac commit b1d1563

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+1847
-774
lines changed

‎.github/workflows/repo-sync.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,27 @@ jobs:
8989
cd migration-tool-sitecore
9090
git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }}
9191
git pull origin ${{ env.BRANCH }}
92+
93+
# Cleanup step to remove unused or deleted file references
94+
echo "Cleaning up unused or deleted file references..."
95+
git fetch --all
96+
git reset --hard origin/${{ env.BRANCH }}
97+
git clean -fdX # Remove untracked files and directories
98+
git gc --prune=now # Garbage collect and prune unreachable objects
99+
100+
# Remove unused imports and missing file references
101+
# echo "Running ESLint to remove unused imports..."
102+
# npm install eslint -g
103+
# eslint . --fix
104+
105+
echo "Running autoflake to remove unused imports..."
106+
pip install autoflake
107+
autoflake --remove-all-unused-imports --recursive --in-place .
108+
109+
echo "Running ts-prune to remove unused file references..."
110+
npm install -g ts-prune
111+
ts-prune | awk '{print $1}' | xargs rm -f
112+
92113
rsync -av --delete ${{ env.RSYNC_SITECORE_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/
93114
rsync -av --delete ../cli/ ./cli/
94115
rsync -av --delete ../ui/ ./ui/
@@ -134,6 +155,27 @@ jobs:
134155
cd migration-tool-contentful
135156
git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }}
136157
git pull origin ${{ env.BRANCH }}
158+
159+
# Cleanup step to remove unused or deleted file references
160+
echo "Cleaning up unused or deleted file references..."
161+
git fetch --all
162+
git reset --hard origin/${{ env.BRANCH }}
163+
git clean -fdX # Remove untracked files and directories
164+
git gc --prune=now # Garbage collect and prune unreachable objects
165+
166+
# Remove unused imports and missing file references
167+
# echo "Running ESLint to remove unused imports..."
168+
# npm install eslint -g
169+
# eslint . --fix
170+
171+
echo "Running autoflake to remove unused imports..."
172+
pip install autoflake
173+
autoflake --remove-all-unused-imports --recursive --in-place .
174+
175+
echo "Running ts-prune to remove unused file references..."
176+
npm install -g ts-prune
177+
ts-prune | awk '{print $1}' | xargs rm -f
178+
137179
rsync -av --delete ${{ env.RSYNC_CONTENTFUL_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/
138180
rsync -av --delete ../cli/ ./cli/
139181
rsync -av --delete ../ui/ ./ui/
@@ -179,6 +221,27 @@ jobs:
179221
cd migration-tool-wordpress
180222
git checkout -b sync-from-migration-v2-${{ github.event.pull_request.number }}
181223
git pull origin ${{ env.BRANCH }}
224+
225+
# Cleanup step to remove unused or deleted file references
226+
echo "Cleaning up unused or deleted file references..."
227+
git fetch --all
228+
git reset --hard origin/${{ env.BRANCH }}
229+
git clean -fdX # Remove untracked files and directories
230+
git gc --prune=now # Garbage collect and prune unreachable objects
231+
232+
# Remove unused imports and missing file references
233+
# echo "Running ESLint to remove unused imports..."
234+
# npm install eslint -g
235+
# eslint . --fix
236+
237+
echo "Running autoflake to remove unused imports..."
238+
pip install autoflake
239+
autoflake --remove-all-unused-imports --recursive --in-place .
240+
241+
echo "Running ts-prune to remove unused file references..."
242+
npm install -g ts-prune
243+
ts-prune | awk '{print $1}' | xargs rm -f
244+
182245
rsync -av --delete ${{ env.RSYNC_WORDPRESS_API_SRC_SERVICES_EXCLUDES }} ../api/ ./api/
183246
rsync -av --delete ../cli/ ./cli/
184247
rsync -av --delete ../ui/ ./ui/

‎api/.eslintrc.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,6 @@
2828
],
2929
"func-names": [0],
3030
"no-console": ["error", { "allow": ["warn", "error", "info"] }],
31-
"@typescript-eslint/no-explicit-any": "off"
31+
"@typescript-eslint/no-explicit-any": "warn"
3232
}
3333
}

‎api/package-lock.json‎

Lines changed: 5 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
"@contentstack/cli-utilities": "^1.7.1",
3030
"@contentstack/json-rte-serializer": "^2.0.7",
3131
"@contentstack/marketplace-sdk": "^1.2.4",
32-
"axios": "^1.7.8",
32+
"axios": "^1.8.2",
3333
"chokidar": "^3.6.0",
3434
"cors": "^2.8.5",
3535
"dotenv": "^16.3.1",

‎api/src/services/contentful.service.ts‎

Lines changed: 37 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,7 @@ import jsonpath from "jsonpath";
88
import pLimit from 'p-limit';
99
import { JSDOM } from "jsdom";
1010
import { jsonToHtml, jsonToMarkdown, htmlToJson } from '@contentstack/json-rte-serializer';
11-
12-
13-
import { CHUNK_SIZE, MIGRATION_DATA_CONFIG, LOCALE_MAPPER } from "../constants/index.js";
11+
import { CHUNK_SIZE, LOCALE_MAPPER, MIGRATION_DATA_CONFIG } from "../constants/index.js";
1412
import { Locale } from "../models/types.js";
1513
import jsonRTE from "./contentful/jsonRTE.js";
1614
import { getAllLocales, getLogMessage } from "../utils/index.js";
@@ -86,6 +84,18 @@ function makeChunks(assetData: any) {
8684
return chunks;
8785
}
8886

87+
const mapLocales = ({ masterLocale, locale, locales }: any) => {
88+
if (locales?.masterLocale?.[masterLocale ?? ''] === locale) {
89+
return Object?.keys(locales?.masterLocale)?.[0]
90+
}
91+
for (const [key, value] of Object?.entries?.(locales) ?? {}) {
92+
if (typeof value !== 'object' && value === locale) {
93+
return key;
94+
}
95+
}
96+
return locale.toLowerCase();
97+
}
98+
8999
const transformCloudinaryObject = (input: any) => {
90100
const result: any = [];
91101
if (!Array.isArray(input)) {
@@ -122,7 +132,7 @@ const transformCloudinaryObject = (input: any) => {
122132
id: uuidv4(),
123133
folder: "",
124134
cs_metadata: {
125-
config_label: "config"
135+
config_label: "default_multi_config_key"
126136
}
127137
});
128138
}
@@ -702,17 +712,6 @@ const createEnvironment = async (packagePath: any, destination_stack_id: string,
702712
}
703713
};
704714

705-
const mapLocales = ({ masterLocale, locale, locales }: any) => {
706-
if (locales?.masterLocale?.[masterLocale ?? ''] === locale) {
707-
return Object?.keys(locales?.masterLocale)?.[0]
708-
}
709-
for (const [key, value] of Object?.entries?.(locales) ?? {}) {
710-
if (typeof value !== 'object' && value === locale) {
711-
return key;
712-
}
713-
}
714-
return locale.toLowerCase();
715-
}
716715

717716
/**
718717
* Creates and processes entries from a given package file and saves them to the destination stack directory.
@@ -740,7 +739,7 @@ const mapLocales = ({ masterLocale, locale, locales }: any) => {
740739
*
741740
* @throws Will log errors encountered during file reading, processing, or writing of entries.
742741
*/
743-
const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string): Promise<void> => {
742+
const createEntry = async (packagePath: any, destination_stack_id: string, projectId: string, contentTypes: any, mapperKeys: any, master_locale: string, project: any): Promise<void> => {
744743
const srcFunc = 'createEntry';
745744
try {
746745
const entriesSave = path.join(DATA, destination_stack_id, ENTRIES_DIR_NAME);
@@ -749,7 +748,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
749748
const data = await fs.promises.readFile(packagePath, "utf8");
750749
const entries = JSON.parse(data)?.entries;
751750
const content = JSON.parse(data)?.contentTypes;
752-
751+
const LocaleMapper = { masterLocale: project?.master_locale ?? LOCALE_MAPPER?.masterLocale, ...project?.locales ?? {} };
753752
if (entries && entries.length > 0) {
754753
const assetId = await readFile(assetsSave, ASSETS_SCHEMA_FILE) ?? [];
755754
const entryId = await readFile(path.join(DATA, destination_stack_id, REFERENCES_DIR_NAME), REFERENCES_FILE_NAME);
@@ -799,13 +798,13 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
799798
});
800799
const pathName = getDisplayName(name, displayField);
801800
locales.forEach((locale) => {
802-
const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LOCALE_MAPPER });
801+
const localeCode = mapLocales({ masterLocale: master_locale, locale, locales: LocaleMapper });
803802
const publishDetails = Object?.values?.(environmentsId)?.length ? Object?.values?.(environmentsId)
804803
.filter((env: any) => env?.name === environment_id)
805804
?.map((env: any) => ({
806805
environment: env?.uid,
807806
version: 1,
808-
locale: locale?.toLowerCase?.(),
807+
locale: localeCode,
809808
})) : [];
810809
const title = fields?.[pathName]?.[locale] || "";
811810
const urlTitle = title
@@ -816,7 +815,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
816815
title: title?.trim?.() === "" ? (urlTitle || id) : title,
817816
uid: id,
818817
url: `/${name?.toLowerCase?.()}/${urlTitle}`,
819-
locale: locale?.toLowerCase?.(),
818+
locale: localeCode,
820819
publish_details: publishDetails,
821820
};
822821
// Format object keys to snake_case
@@ -843,11 +842,12 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
843842
for await (const [localeKey, localeValues] of Object.entries(
844843
values as { [key: string]: any }
845844
)) {
845+
const localeCode = mapLocales({ masterLocale: master_locale, locale: localeKey, locales: LocaleMapper });
846846
const chunks = makeChunks(localeValues);
847847
for (const [entryKey, entryValue] of Object.entries(localeValues)) {
848848
const message = getLogMessage(
849849
srcFunc,
850-
`Entry title "${(entryValue as { title: string })?.title}"(${ctName}) in the ${localeKey} locale has been successfully transformed.`,
850+
`Entry title "${(entryValue as { title: string })?.title}"(${ctName}) in the ${localeCode} locale has been successfully transformed.`,
851851
{}
852852
);
853853
await customLogger(projectId, destination_stack_id, "info", message);
@@ -856,7 +856,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
856856
let chunkIndex = 1;
857857
const filePath = path.join(
858858
entriesSave,
859-
ctName, localeKey.toLowerCase()
859+
ctName, localeCode
860860
);
861861
for await (const [chunkId, chunkData] of Object.entries(chunks)) {
862862
refs[chunkIndex++] = `${chunkId}-entries.json`;
@@ -874,7 +874,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
874874
await customLogger(projectId, destination_stack_id, 'info', message);
875875
}
876876
} catch (err) {
877-
console.info("🚀 ~ createEntry ~ err:", err)
877+
console.error("🚀 ~ createEntry ~ err:", err)
878878
const message = getLogMessage(
879879
srcFunc,
880880
`Error encountered while creating entries.`,
@@ -885,6 +885,10 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
885885
}
886886
};
887887

888+
function getKeyByValue(obj: Record<string, string>, targetValue: string): string | undefined {
889+
return Object.entries(obj).find(([_, value]) => value === targetValue)?.[0];
890+
}
891+
888892
/**
889893
* Processes and creates locale configurations from a given package file and saves them to the destination stack directory.
890894
*
@@ -909,7 +913,7 @@ const createEntry = async (packagePath: any, destination_stack_id: string, proje
909913
*
910914
* @throws Will log errors encountered during file reading, processing, or writing of locale configurations.
911915
*/
912-
const createLocale = async (packagePath: string, destination_stack_id: string, projectId: string) => {
916+
const createLocale = async (packagePath: string, destination_stack_id: string, projectId: string, project: any) => {
913917
const srcFunc = 'createLocale';
914918
const localeSave = path.join(DATA, destination_stack_id, LOCALE_DIR_NAME);
915919
const globalFieldSave = path.join(DATA, destination_stack_id, GLOBAL_FIELDS_DIR_NAME);
@@ -933,30 +937,30 @@ const createLocale = async (packagePath: string, destination_stack_id: string, p
933937
)
934938
await customLogger(projectId, destination_stack_id, 'error', message);
935939
}
936-
937-
await Promise.all(locales.map(async (localeData: any) => {
938-
const title = localeData.sys.id;
940+
const fallbackMapLocales: any = { ...project?.master_locale ?? {}, ...project?.locales ?? {} }
941+
await Promise?.all(locales?.map?.(async (localeData: any) => {
942+
const currentMapLocale = getKeyByValue?.(fallbackMapLocales, localeData?.code) ?? `${localeData.code.toLowerCase()}`;
943+
const title = localeData?.sys?.id;
939944
const newLocale: Locale = {
940-
code: `${localeData.code.toLowerCase()}`,
941-
name: localeCodes?.[localeData.code.toLowerCase()] || "English - United States",
942-
fallback_locale: "",
945+
code: currentMapLocale,
946+
name: localeCodes?.[currentMapLocale] || "English - United States",
947+
fallback_locale: getKeyByValue(fallbackMapLocales, localeData?.fallbackCode) ?? '',
943948
uid: `${title}`,
944949
};
945950

946951
if (localeData.default === true) {
947952
msLocale[title] = newLocale;
948953
const message = getLogMessage(
949954
srcFunc,
950-
`Master Locale ${newLocale.code} has been successfully transformed.`,
955+
`Master Locale ${newLocale?.code} has been successfully transformed.`,
951956
{}
952957
)
953958
await customLogger(projectId, destination_stack_id, 'info', message);
954959
} else {
955-
newLocale.name = `${localeData.name}`;
956960
allLocales[title] = newLocale;
957961
const message = getLogMessage(
958962
srcFunc,
959-
`Locale ${newLocale.code} has been successfully transformed.`,
963+
`Locale ${newLocale?.code} has been successfully transformed.`,
960964
{}
961965
)
962966
await customLogger(projectId, destination_stack_id, 'info', message);

0 commit comments

Comments
 (0)