diff --git a/package-lock.json b/package-lock.json index fbc45fe..0fdbcd9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@contentstack/datasync-manager", - "version": "2.0.11", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@contentstack/datasync-manager", - "version": "2.0.11", + "version": "2.1.0", "license": "MIT", "dependencies": { "@braintree/sanitize-url": "^7.1.1", diff --git a/package.json b/package.json index bc3b029..4861600 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/datasync-manager", "author": "Contentstack LLC ", - "version": "2.0.11", + "version": "2.1.0", "description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API", "main": "dist/index.js", "dependencies": { diff --git a/src/config.ts b/src/config.ts index f654586..eff13de 100644 --- a/src/config.ts +++ b/src/config.ts @@ -127,4 +127,9 @@ export const config = { saveFailedItems: true, saveFilteredItems: true, }, + checkpoint: { + enabled: false, // Set to true if you want to enable checkpoint + filePath: ".checkpoint", + preserve: false // Set to true if you want to preserve the checkpoint file during clean operation + }, } diff --git a/src/core/index.ts b/src/core/index.ts index de1b822..4ce62aa 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -3,7 +3,8 @@ * Copyright (c) 2019 Contentstack LLC * MIT Licensed */ - +import * as fs from 'fs' +import * as path from 'path' import Debug from 'debug' import { EventEmitter } from 'events' import { cloneDeep, remove } from 'lodash' @@ -16,6 +17,7 @@ import { map } from '../util/promise.map' import { netConnectivityIssues } from './inet' import { Q as Queue } from './q' import { getToken, saveCheckpoint } from './token-management' +import { sanitizePath } from '../plugins/helper' interface IQueryString { init?: true, @@ -46,6 +48,11 @@ interface IToken { name: string token: string } +interface ICheckpoint { + enabled: boolean, + filePath: string, + preserve:boolean +} const debug = Debug('sync-core') const emitter = new EventEmitter() @@ -74,6 +81,7 @@ export const init = (contentStore, assetStore) => { return new Promise((resolve, reject) => { try { Contentstack = config.contentstack + const checkPointConfig: ICheckpoint = config.checkpoint const paths = config.paths const environment = Contentstack.environment || process.env.NODE_ENV || 'development' debug(`Environment: ${environment}`) @@ -83,6 +91,7 @@ export const init = (contentStore, assetStore) => { limit: config.syncManager.limit, }, } + loadCheckpoint(checkPointConfig, paths); if (typeof Contentstack.sync_token === 'string' && Contentstack.sync_token.length !== 0) { request.qs.sync_token = Contentstack.sync_token } else if (typeof Contentstack.pagination_token === 'string' && Contentstack.pagination_token.length !== 0) { @@ -110,6 +119,44 @@ export const init = (contentStore, assetStore) => { }) } +const loadCheckpoint = (checkPointConfig: ICheckpoint, paths: any): void => { + if (!checkPointConfig?.enabled) return; + + // Try reading checkpoint from primary path + let checkpoint = readHiddenFile(paths.checkpoint); + + // Fallback to filePath in config if not found + if (!checkpoint) { + const fallbackPath = path.join( + sanitizePath(__dirname), + sanitizePath(checkPointConfig.filePath || ".checkpoint") + ); + checkpoint = readHiddenFile(fallbackPath); + } + + // Set sync token if checkpoint is found + if (checkpoint) { + debug("Found sync token in checkpoint file:", checkpoint); + Contentstack.sync_token = checkpoint.token; + debug("Using sync token:", Contentstack.sync_token); + } +}; + + +function readHiddenFile(filePath: string) { + try { + if (!fs.existsSync(filePath)) { + logger.error("File does not exist:", filePath); + return; + } + const data = fs.readFileSync(filePath, "utf8"); + return JSON.parse(data); + } catch (err) { + logger.error("Error reading file:", err); + return undefined; + } +} + export const push = (data) => { Q.emit('push', data) } diff --git a/src/core/plugins.ts b/src/core/plugins.ts index 3267b5e..d330fb9 100644 --- a/src/core/plugins.ts +++ b/src/core/plugins.ts @@ -19,47 +19,52 @@ const pluginMethods = ['beforeSync', 'afterSync'] */ export const load = (config) => { debug('Plugins load called') - const pluginInstances = { - external: {}, - internal: {}, - } - const plugins = config.plugins || [] - pluginMethods.forEach((pluginMethod) => { - pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || [] - pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [] - }) - - plugins.forEach((plugin) => { - validatePlugin(plugin) - - const pluginName = plugin.name - const slicedName = pluginName.slice(0, 13) - let isInternal = false - if (slicedName === '_cs_internal_') { - isInternal = true + try { + const pluginInstances = { + external: {}, + internal: {}, } - - const pluginPath = normalizePluginPath(config, plugin, isInternal) - const Plugin = require(pluginPath) - Plugin.options = plugin.options || {} - // execute/initiate plugin - Plugin() + const plugins = config.plugins || [] pluginMethods.forEach((pluginMethod) => { - if (hasIn(Plugin, pluginMethod)) { - if (plugin.disabled) { - // do nothing - } else if (isInternal) { - pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod]) + pluginInstances.external[pluginMethod] = pluginInstances[pluginMethod] || [] + pluginInstances.internal[pluginMethod] = pluginInstances[pluginMethod] || [] + }) + + plugins.forEach((plugin) => { + validatePlugin(plugin) + + const pluginName = plugin.name + const slicedName = pluginName.slice(0, 13) + let isInternal = false + if (slicedName === '_cs_internal_') { + isInternal = true + } + + const pluginPath = normalizePluginPath(config, plugin, isInternal) + const Plugin = require(pluginPath) + Plugin.options = plugin.options || {} + // execute/initiate plugin + Plugin() + pluginMethods.forEach((pluginMethod) => { + if (hasIn(Plugin, pluginMethod)) { + if (plugin.disabled) { + // do nothing + } else if (isInternal) { + pluginInstances.internal[pluginMethod].push(Plugin[pluginMethod]) + } else { + pluginInstances.external[pluginMethod].push(Plugin[pluginMethod]) + } + debug(`${pluginMethod} loaded from ${pluginName} successfully!`) } else { - pluginInstances.external[pluginMethod].push(Plugin[pluginMethod]) + debug(`${pluginMethod} not found in ${pluginName}`) } - debug(`${pluginMethod} loaded from ${pluginName} successfully!`) - } else { - debug(`${pluginMethod} not found in ${pluginName}`) - } + }) }) - }) - debug('Plugins loaded successfully!') - - return pluginInstances + debug('Plugins loaded successfully!') + + return pluginInstances + } catch (error) { + debug('Error while loading plugins:', error) + throw new Error(`Failed to load plugins: ${error?.message}`) + } }