Merge pull request #76 from contentstack/development

DX | 07-07-2025 | Release

Author: harshithad0703

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/datasync-manager",
   "author": "Contentstack LLC <[email protected]>",
-  "version": "2.0.10",
+  "version": "2.1.0",
   "description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API",
   "main": "dist/index.js",
   "dependencies": {

package.json

@@ -127,4 +127,9 @@ export const config = {
     saveFailedItems: true,
     saveFilteredItems: true,
   },
+  checkpoint: {
+    enabled: false, // Set to true if you want to enable checkpoint
+    filePath: ".checkpoint",
+    preserve: false // Set to true if you want to preserve the checkpoint file during clean operation
+  },
 }

src/config.ts

@@ -3,7 +3,8 @@
  * Copyright (c) 2019 Contentstack LLC
  * MIT Licensed
  */
-
+import * as fs from 'fs'
+import * as path from 'path'
 import Debug from 'debug'
 import { EventEmitter } from 'events'
 import { cloneDeep, remove } from 'lodash'
@@ -16,6 +17,7 @@ import { map } from '../util/promise.map'
 import { netConnectivityIssues } from './inet'
 import { Q as Queue } from './q'
 import { getToken, saveCheckpoint } from './token-management'
+import { sanitizePath } from '../plugins/helper'
 
 interface IQueryString {
   init?: true,
@@ -46,6 +48,11 @@ interface IToken {
   name: string
   token: string
 }
+interface ICheckpoint {
+  enabled: boolean,
+  filePath: string,
+  preserve:boolean
+}
 
 const debug = Debug('sync-core')
 const emitter = new EventEmitter()
@@ -74,6 +81,7 @@ export const init = (contentStore, assetStore) => {
   return new Promise((resolve, reject) => {
     try {
       Contentstack = config.contentstack
+      const checkPointConfig: ICheckpoint = config.checkpoint
       const paths = config.paths
       const environment = Contentstack.environment || process.env.NODE_ENV || 'development'
       debug(`Environment: ${environment}`)
@@ -83,6 +91,7 @@ export const init = (contentStore, assetStore) => {
           limit: config.syncManager.limit,
         },
       }
+      loadCheckpoint(checkPointConfig, paths);
       if (typeof Contentstack.sync_token === 'string' && Contentstack.sync_token.length !== 0) {
         request.qs.sync_token = Contentstack.sync_token
       } else if (typeof Contentstack.pagination_token === 'string' && Contentstack.pagination_token.length !== 0) {
@@ -110,6 +119,44 @@ export const init = (contentStore, assetStore) => {
   })
 }
 
+const loadCheckpoint = (checkPointConfig: ICheckpoint, paths: any): void => {
+  if (!checkPointConfig?.enabled) return;
+
+  // Try reading checkpoint from primary path
+  let checkpoint = readHiddenFile(paths.checkpoint);
+
+  // Fallback to filePath in config if not found
+  if (!checkpoint) {
+    const fallbackPath = path.join(
+      sanitizePath(__dirname),
+      sanitizePath(checkPointConfig.filePath || ".checkpoint")
+    );
+    checkpoint = readHiddenFile(fallbackPath);
+  }
+
+  // Set sync token if checkpoint is found
+  if (checkpoint) {
+    debug("Found sync token in checkpoint file:", checkpoint);
+    Contentstack.sync_token = checkpoint.token;
+    debug("Using sync token:", Contentstack.sync_token);
+  }
+};
+
+
+function readHiddenFile(filePath: string) {
+  try {
+    if (!fs.existsSync(filePath)) {
+      logger.error("File does not exist:", filePath);
+      return;
+    }
+    const data = fs.readFileSync(filePath, "utf8"); 
+    return JSON.parse(data); 
+  } catch (err) {
+    logger.error("Error reading file:", err);
+    return undefined;
+  }
+}
+
 export const push = (data) => {
   Q.emit('push', data)
 }

src/core/index.ts

@@ -19,47 +19,52 @@ const pluginMethods = ['beforeSync', 'afterSync']
  */
 export const load = (config) => {
   debug('Plugins load called')
-  const pluginInstances = {
-    external: {},
-    internal: {},
-  }
-  const plugins = config.plugins || []
-  pluginMethods.forEach((pluginMethod) => {
-    pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || []
-    pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || []
-  })
-
-  plugins.forEach((plugin) => {
-    validatePlugin(plugin)
-
-    const pluginName = plugin.name
-    const slicedName = pluginName.slice(0, 13)
-    let isInternal = false
-    if (slicedName === '_cs_internal_') {
-      isInternal = true
+  try {
+    const pluginInstances = {
+      external: {},
+      internal: {},
     }
-
-    const pluginPath = normalizePluginPath(config, plugin, isInternal)
-    const Plugin = require(pluginPath)
-    Plugin.options = plugin.options || {}
-    // execute/initiate plugin
-    Plugin()
+    const plugins = config.plugins || []
     pluginMethods.forEach((pluginMethod) => {
-      if (hasIn(Plugin, pluginMethod)) {
-        if (plugin.disabled) {
-          // do nothing
-        } else if (isInternal) {
-          pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod])
+      pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || []
+      pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || []
+    })
+  
+    plugins.forEach((plugin) => {
+      validatePlugin(plugin)
+  
+      const pluginName = plugin.name
+      const slicedName = pluginName.slice(0, 13)
+      let isInternal = false
+      if (slicedName === '_cs_internal_') {
+        isInternal = true
+      }
+  
+      const pluginPath = normalizePluginPath(config, plugin, isInternal)
+      const Plugin = require(pluginPath)
+      Plugin.options = plugin.options || {}
+      // execute/initiate plugin
+      Plugin()
+      pluginMethods.forEach((pluginMethod) => {
+        if (hasIn(Plugin, pluginMethod)) {
+          if (plugin.disabled) {
+            // do nothing
+          } else if (isInternal) {
+            pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod])
+          } else {
+            pluginInstances.external[pluginMethod].push(Plugin[pluginMethod])
+          }
+          debug(`${pluginMethod} loaded from ${pluginName} successfully!`)
         } else {
-          pluginInstances.external[pluginMethod].push(Plugin[pluginMethod])
+          debug(`${pluginMethod} not found in ${pluginName}`)
         }
-        debug(`${pluginMethod} loaded from ${pluginName} successfully!`)
-      } else {
-        debug(`${pluginMethod} not found in ${pluginName}`)
-      }
+      })
     })
-  })
-  debug('Plugins loaded successfully!')
-
-  return pluginInstances
+    debug('Plugins loaded successfully!')
+  
+    return pluginInstances
+  } catch (error) {
+    debug('Error while loading plugins:', error)
+    throw new Error(`Failed to load plugins: ${error?.message}`)   
+  }
 }

src/core/plugins.ts

@@ -149,8 +149,9 @@ export const formatItems = (items, config) => {
         items[i]._type = config.contentstack.actions.publish
         // extra keys
         items[i]._synced_at = time
-        items[i].locale = items[i].data.publish_details.locale
-        items[i] = merge(items[i], items[i].data)
+        const assetLocale = items[i].data.publish_details.locale
+        items[i] = merge(cloneDeep(items[i]), items[i].data)
+        items[i].locale = assetLocale
         break
       case 'asset_unpublished':
         delete items[i].type
@@ -170,8 +171,9 @@ export const formatItems = (items, config) => {
         items[i]._content_type_uid = items[i].content_type_uid
         // extra keys
         items[i]._synced_at = time
-        items[i].locale = items[i].data.publish_details.locale
-        items[i] = merge(items[i], items[i].data)
+        const entryLocale = items[i].data.publish_details.locale
+        items[i] = merge(cloneDeep(items[i]), items[i].data)
+        items[i].locale = entryLocale
         break
       case 'entry_unpublished':
         delete items[i].type