Skip to content

Commit 666876d

Browse files
sayalijoshi27sayalijoshi27
authored
Merge pull request #419 from contentstack/bugfix/contentful-migration
Fix: Fixed test migration Issues
2 parents 1736af6 + 7a0a83e commit 666876d

File tree

7 files changed

+73
-57
lines changed

7 files changed

+73
-57
lines changed

‎api/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"jsonwebtoken": "^9.0.2",
4646
"lowdb": "^7.0.1",
4747
"mkdirp": "^3.0.1",
48+
"p-limit": "^6.2.0",
4849
"path-to-regexp": "^8.2.0",
4950
"router": "^2.0.0",
5051
"shelljs": "^0.8.5",

‎api/src/services/contentful.service.ts‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { v4 as uuidv4 } from "uuid";
55
import _ from "lodash";
66
import axios from "axios";
77
import jsonpath from "jsonpath";
8+
// import pLimit from 'p-limit';
89

910

1011
import {CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js";
@@ -24,6 +25,7 @@ const {
2425
RTE_REFERENCES_DIR_NAME,
2526
ENTRIES_DIR_NAME,
2627
ASSETS_DIR_NAME,
28+
GLOBAL_FIELDS_DIR_NAME,
2729
// FILE
2830
LOCALE_MASTER_LOCALE,
2931
LOCALE_FILE_NAME,
@@ -38,6 +40,7 @@ const {
3840
ENTRIES_MASTER_FILE,
3941
WEBHOOKS_FILE_NAME,
4042
RTE_REFERENCES_FILE_NAME,
43+
GLOBAL_FIELDS_FILE_NAME,
4144

4245
} = MIGRATION_DATA_CONFIG;
4346

@@ -152,8 +155,8 @@ const processField = (
152155
if (lang_value.sys) {
153156
const { linkType, id } = lang_value.sys;
154157

155-
if (linkType === "Entry" && id in entryId) return [entryId[id]];
156-
if (linkType === "Asset" && id in assetId) return assetId[id];
158+
if (linkType === "Entry" && id in entryId) return [entryId?.[id]];
159+
if (linkType === "Asset" && id in assetId) return assetId?.[id];
157160
}
158161

159162
// Handle arrays and nested objects
@@ -181,7 +184,7 @@ const processArrayFields = (array: any, entryId: any, assetId: any) => {
181184
if (id in entryId) {
182185
array.splice(i, 1, entryId[id]);
183186
} else if (id in assetId) {
184-
array.splice(i, 1, assetId[id]);
187+
array.splice(i, 1, assetId?.[id]);
185188
}
186189
});
187190
// Clean up empty objects
@@ -342,10 +345,10 @@ const saveAsset = async (
342345
await fs.promises.mkdir(assetPath, { recursive: true });
343346
// Write file as binary
344347
await fs.promises.writeFile(path.join(assetPath, fileName), Buffer.from(response.data), "binary");
345-
await customLogger(projectId, destination_stack_id, 'info', message);
346348
await writeFile(assetPath, `_contentstack_${assets.sys.id}.json`, assetData[assets.sys.id]);
347349
metadata.push({ uid: assets.sys.id, url: fileUrl, filename: fileName });
348350
delete failedJSON[assets.sys.id];
351+
await customLogger(projectId, destination_stack_id, 'info', message);
349352
} catch (err: any) {
350353
if (retryCount === 1) {
351354
failedJSON[assets.sys.id] = {
@@ -401,31 +404,41 @@ const createAssets = async (packagePath: any, destination_stack_id:string, proje
401404
const failedJSON: any = {};
402405
const assetData: any = {};
403406
const metadata: AssetMetaData[] = [];
404-
407+
const fileMeta = { "1": ASSETS_SCHEMA_FILE };
405408
const assets = JSON.parse(data)?.assets;
406409

407410
if (assets && assets.length > 0) {
408411
const tasks = assets.map(
409-
async (asset: any) =>
410-
await saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0)
412+
async (asset: any, index: any) =>
413+
saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0)
411414
);
415+
416+
// This code is intentionally commented out
417+
// for testing purposes.
418+
419+
// const limit = pLimit(10); // Limit concurrent operations to 10
420+
// const tasks = assets.map((asset: any) =>
421+
// limit(() => saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0))
422+
// );
423+
// await Promise.all(tasks);
424+
412425
await Promise.all(tasks);
413426
const assetMasterFolderPath = path.join(assetsSave, ASSETS_FAILED_FILE);
414427

415428
await writeOneFile(path.join(assetsSave, ASSETS_SCHEMA_FILE), assetData);
416-
const chunks: { [key: string]: any } = makeChunks(assetData);
417-
const refs: any = {};
418-
419-
for (const [index, chunkId] of Object.keys(chunks).entries()) {
420-
refs[index + 1] = `${chunkId}-${ASSETS_FILE_NAME}`;
421-
await writeOneFile(
422-
path.join(assetsSave, `${chunkId}-${ASSETS_FILE_NAME}`),
423-
chunks[chunkId]
424-
);
425-
}
426-
427-
await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), refs);
428-
await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata);
429+
// const chunks: { [key: string]: any } = makeChunks(assetData);
430+
// const refs: any = {};
431+
432+
// for (const [index, chunkId] of Object.keys(chunks).entries()) {
433+
// refs[index + 1] = `${chunkId}-${ASSETS_FILE_NAME}`;
434+
// await writeOneFile(
435+
// path.join(assetsSave, `${chunkId}-${ASSETS_FILE_NAME}`),
436+
// chunks[chunkId]
437+
// );
438+
// }
439+
440+
await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), fileMeta);
441+
// await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata);
429442
failedJSON && await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON);
430443
} else {
431444
const message = getLogMessage(
@@ -720,6 +733,7 @@ const createEntry = async (packagePath: any, destination_stack_id:string, projec
720733
const createLocale = async (packagePath: string, destination_stack_id:string, projectId:string) => {
721734
const srcFunc = 'createLocale';
722735
const localeSave = path.join(DATA, destination_stack_id, LOCALE_DIR_NAME);
736+
const globalFieldSave = path.join(DATA, destination_stack_id, GLOBAL_FIELDS_DIR_NAME);
723737

724738
try {
725739
const msLocale: Record<string, Locale> = {};
@@ -773,6 +787,7 @@ const createLocale = async (packagePath: string, destination_stack_id:string, pr
773787
await writeFile(localeSave, LOCALE_FILE_NAME, allLocales)
774788
await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale)
775789
await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList)
790+
await writeFile(globalFieldSave, GLOBAL_FIELDS_FILE_NAME, [])
776791
const message = getLogMessage(
777792
srcFunc,
778793
`locales have been successfully transformed.`,

‎api/src/services/contentful/jsonRTE.ts‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,15 @@ function parseDocument(obj: any, lang?: LangType, destination_stack_id?:StackId)
8585
};
8686
}
8787

88-
function parseTable(obj: any): any {
88+
function parseTable(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
8989
const rowCount = obj.content.length;
9090
const colCount = Math.max(...obj.content.map((e: any) => e.content.length));
9191
const attrs = {
9292
rows: rowCount,
9393
cols: colCount,
9494
colWidths: Array(colCount).fill(250),
9595
};
96-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).concat(parsers.get('tbody')?.(obj)).filter(Boolean);
96+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).concat(parsers.get('tbody')?.(obj)).filter(Boolean);
9797

9898
return {
9999
type: 'table',
@@ -103,19 +103,19 @@ function parseTable(obj: any): any {
103103
};
104104
}
105105

106-
function parseTableRow(obj: any): any {
106+
function parseTableRow(obj: any,lang?: LangType, destination_stack_id?:StackId): any {
107107
const types = new Set<string>();
108108
const children = obj.content.map((e: any) => {
109109
types.add(e.nodeType);
110-
return parsers.get(e.nodeType)?.(e);
110+
return parsers.get(e.nodeType)?.(e, lang, destination_stack_id);
111111
}).filter(Boolean);
112112

113113
const type = types.has('table-header-cell') ? 'thead' : '';
114114
return children.length ? { type, attrs: {}, uid: generateUID('tabletype'), children } : null;
115115
}
116116

117-
function parseHeadTR(obj: any[]): any {
118-
const children = obj.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
117+
function parseHeadTR(obj: any[],lang?: LangType, destination_stack_id?:StackId): any {
118+
const children = obj.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
119119
return {
120120
type: 'tr',
121121
attrs: {},
@@ -124,8 +124,8 @@ function parseHeadTR(obj: any[]): any {
124124
};
125125
}
126126

127-
function parseTableHead(obj: any): any {
128-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
127+
function parseTableHead(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
128+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
129129
return {
130130
type: 'th',
131131
attrs: {},
@@ -134,8 +134,8 @@ function parseTableHead(obj: any): any {
134134
};
135135
}
136136

137-
function parseTBody(obj: any): any {
138-
const children = obj.content.map((e: any) => parsers.get('body-tr')?.(e)).filter(Boolean);
137+
function parseTBody(obj: any,lang?: LangType, destination_stack_id?:StackId ): any {
138+
const children = obj.content.map((e: any) => parsers.get('body-tr')?.(e, lang, destination_stack_id)).filter(Boolean);
139139
return {
140140
type: 'tbody',
141141
attrs: {},
@@ -144,18 +144,18 @@ function parseTBody(obj: any): any {
144144
};
145145
}
146146

147-
function parseBodyTR(obj: any): any {
148-
const children = obj.content.filter((e: any) => e.nodeType === 'table-cell').map((e: any) => parsers.get('table-cell')?.(e)).filter(Boolean);
147+
function parseBodyTR(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
148+
const children = obj.content.filter((e: any) => e.nodeType === 'table-cell').map((e: any) => parsers.get('table-cell')?.(e, lang, destination_stack_id)).filter(Boolean);
149149
return children.length ? { type: 'tr', attrs: {}, uid: generateUID('tr'), children } : null;
150150
}
151151

152-
function parseTableBody(obj: any): any {
153-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
152+
function parseTableBody(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
153+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
154154
return children.length ? { type: 'td', attrs: {}, uid: generateUID('td'), children } : null;
155155
}
156156

157-
function parseParagraph(obj: any, lang?: LangType): any {
158-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang)).filter(Boolean);
157+
function parseParagraph(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
158+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
159159
return {
160160
type: 'p',
161161
attrs: {},
@@ -181,8 +181,8 @@ function parseHR(): any {
181181
};
182182
}
183183

184-
function parseUL(obj: any): any {
185-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
184+
function parseUL(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
185+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e,lang, destination_stack_id)).filter(Boolean);
186186
return {
187187
type: 'ul',
188188
attrs: {},
@@ -192,8 +192,8 @@ function parseUL(obj: any): any {
192192
};
193193
}
194194

195-
function parseOL(obj: any): any {
196-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
195+
function parseOL(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
196+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
197197
return {
198198
type: 'ol',
199199
attrs: {},
@@ -203,8 +203,8 @@ function parseOL(obj: any): any {
203203
};
204204
}
205205

206-
function parseLI(obj: any): any {
207-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).flat().filter(Boolean);
206+
function parseLI(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
207+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).flat().filter(Boolean);
208208
return {
209209
type: 'li',
210210
attrs: {},

‎api/src/utils/content-type-creator.utils.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => {
171171

172172
case 'dropdown': {
173173
const data = {
174-
"data_type": ['Integer', 'Number'].includes(field.otherCmsType) ? 'number' : "text",
174+
"data_type": ['dropdownNumber', 'radioNumber', 'ratingNumber'].includes(field.otherCmsType) ? 'number' : "text",
175175
"display_name": field?.title,
176176
"display_type": "dropdown",
177177
"enum": {
@@ -198,7 +198,7 @@ const convertToSchemaFormate = ({ field, advanced = true }: any) => {
198198
}
199199
case 'radio': {
200200
const data = {
201-
"data_type": ['Integer', 'Number'].includes(field.otherCmsType) ? 'number' : "text",
201+
"data_type": ['dropdownNumber', 'radioNumber', 'ratingNumber'].includes(field.otherCmsType) ? 'number' : "text",
202202
"display_name": field?.title,
203203
"display_type": "radio",
204204
"enum": {

‎upload-api/migration-contentful/libs/contentTypeMapper.js‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ const extractAdvancedFields = (
100100
const createFieldObject = (item, contentstackFieldType, backupFieldType, referenceFields = []) => ({
101101
uid: item.id,
102102
otherCmsField: item.name,
103-
otherCmsType: item.type,
103+
otherCmsType: item.widgetId,
104104
contentstackField: item.name,
105105
contentstackFieldUid: uidCorrector(item.id),
106106
contentstackFieldType: contentstackFieldType,
@@ -197,10 +197,15 @@ const contentTypeMapper = (data) => {
197197
acc.push(createFieldObject(item, 'number', 'number'));
198198
break;
199199
case 'dropdown':
200+
item.widgetId = 'dropdownNumber';
201+
acc.push(createDropdownOrRadioFieldObject(item, 'dropdown'));
202+
break;
200203
case 'radio':
201-
acc.push(createDropdownOrRadioFieldObject(item, item.widgetId));
204+
item.widgetId = 'radioNumber';
205+
acc.push(createDropdownOrRadioFieldObject(item, 'radio'));
202206
break;
203207
case 'rating':
208+
item.widgetId = 'ratingNumber';
204209
acc.push(createDropdownOrRadioFieldObject(item, 'dropdown'));
205210
break;
206211
}
@@ -264,7 +269,7 @@ const contentTypeMapper = (data) => {
264269
: item?.contentNames?.slice(0, 9);
265270
}
266271
}
267-
acc.push(createFieldObject(item, 'file', 'file', referenceFields));
272+
acc.push(createFieldObject(item, 'reference', 'reference', referenceFields));
268273
break;
269274
}
270275
case 'checkbox':

‎upload-api/migration-contentful/libs/createInitialMapper.js‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,7 @@ const createInitialMapper = async () => {
8585
type: 'content_type',
8686
fieldMapping: []
8787
};
88-
const titleArray = data.map((item) => item.id);
89-
const uidTitle = titleArray.includes('title') ? []
90-
: [
88+
const uidTitle = [
9189
{
9290
uid: 'title',
9391
otherCmsField: 'title',
@@ -97,10 +95,7 @@ const createInitialMapper = async () => {
9795
contentstackFieldType: 'text',
9896
backupFieldType: 'text',
9997
advanced:{ mandatory:true}
100-
}
101-
];
102-
const uidUrl = titleArray.includes('url') ? []
103-
: [
98+
},
10499
{
105100
uid: 'url',
106101
otherCmsField: 'url',
@@ -112,7 +107,8 @@ const createInitialMapper = async () => {
112107
advanced:{ mandatory:true}
113108
}
114109
];
115-
const contentstackFields = [...uidTitle, ...uidUrl, ...contentTypeMapper(data)].filter(
110+
const dataArray = data.filter((item) => item.id!=='title' && item.id !== 'url');
111+
const contentstackFields = [...uidTitle, ...contentTypeMapper(dataArray)].filter(
116112
Boolean
117113
);
118114

‎upload-api/migration-contentful/libs/extractContentTypes.js‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,7 @@ const saveContentType = (contentTypes, editorInterface, prefix) => {
7070
path.join(
7171
contentfulFolderPath,
7272
`${
73-
content.name.charAt(0).toUpperCase() +
74-
content.name.slice(1).replace(/[^\w\s]/g, "")
73+
(content?.name?.charAt(0)?.toUpperCase() + content?.name?.slice(1))?.replace(/[^\w\s]/g, "")
7574
}.json`
7675
),
7776
JSON.stringify(jsonObj, null, 4)

0 commit comments

Comments
 (0)