Merge pull request #37 from SAP/develop

Synchronize repository - 2025-01-21 - 1.2.8

Author: mateuszporzucek

commercedbsync/extensioninfo.xml

@@ -7,9 +7,11 @@
   --><extensioninfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="extensioninfo.xsd">
 
     <extension abstractclassprefix="Generated" classprefix="Commercedbsync" managername="CommercedbsyncManager" managersuperclass="de.hybris.platform.jalo.extension.Extension" name="commercedbsync" usemaven="true">
-                
+
+        <requires-extension name="azurecloud"/>
+
         <coremodule generated="true" manager="de.hybris.platform.jalo.extension.GenericManager" packageroot="com.sap.cx.boosters.commercedbsync"/>
-                
+
 
     </extension>
 

commercedbsync/external-dependencies.xml

@@ -28,11 +28,6 @@
             <artifactId>commons-dbcp2</artifactId>
             <version>2.7.0</version>
         </dependency>
-        <dependency>
-            <groupId>com.microsoft.azure</groupId>
-            <artifactId>azure-storage</artifactId>
-            <version>8.1.0</version>
-        </dependency>
         <dependency>
             <groupId>com.zaxxer</groupId>
             <artifactId>HikariCP</artifactId>

commercedbsync/project.properties

@@ -254,12 +254,20 @@ migration.schema.target.columns.remove.enabled=true
 ##
 migration.schema.autotrigger.enabled=false
 ##
-# Activate data export to external DB via cron jobs
+# Specifies where to store the internal tables
+# If empty and migration.data.synchronization.enabled is true, then it is set to "target", otherwise "source"
+#
+# @values target or source
+# @optional true
+##
+migration.internal.tables.storage=
+##
+# Activate data synchronization to external DB via cron jobs
 #
 # @values true or false
 # @optional true
 ##
-migration.data.export.enabled=false
+migration.data.synchronization.enabled=false
 ##
 # Specifies the number of rows to read per batch. This only affects tables which can be batched.
 #
@@ -370,6 +378,13 @@ migration.data.columns.excluded.attributedescriptors=
 ##
 migration.data.columns.nullify.attributedescriptors=
 ##
+# Configure columns to use when reading by offset
+#
+# @values comma separated list of column names
+# @optional true
+##
+migration.data.columns.batch.TABLE=
+##
 # If set to true, all indices in the target table will be removed before copying over the data.
 #
 # @values true or false
@@ -429,13 +444,35 @@ migration.data.tables.excluded=SYSTEMINIT,StoredHttpSessions,itemdeletionmarkers
 ##
 migration.data.tables.included=
 ##
+# Tables to be migrated as first (use table names with prefix and suffix)
+#
+# @values comma separated list of table full names.
+# @optional true
+##
+migration.data.tables.order.first=
+##
+# Tables to be migrated as last (use table names with prefix and suffix)
+#
+# @values comma separated list of table full names.
+# @optional true
+##
+migration.data.tables.order.last=
+##
+# List of partitioned tables (use table names with prefix and suffix). Separate batches will be created for each table partition.
+# This only applies when using HANA as source database.
+#
+# @values comma separated list of table full names.
+# @optional true
+##
+migration.data.tables.partitioned=
+##
 # Run migration in the cluster (based on commerce cluster config). The 'HAC' node will be the primary one.
 # A scheduling algorithm decides which table will run on which node. Nodes are notified using cluster events.
 #
 # @values true or false
 # @optional true
 ##
-migration.cluster.enabled=false
+migration.cluster.enabled=true
 ##
 # If set to true, the migration will resume from where it stopped (either due to errors or cancellation).
 #
@@ -502,7 +539,7 @@ migration.data.report.connectionstring=${media.globalSettings.cloudAzureBlobStor
 # @values any property key
 # @optional true
 ##
-migration.properties.masked=migration.data.report.connectionstring,migration.ds.source.db.password,migration.ds.target.db.password
+migration.properties.masked=migration.data.report.connectionstring,migration.ds.source.db.username,migration.ds.source.db.password,migration.ds.target.db.username,migration.ds.target.db.password
 ##
 # Specifies the default locale used.
 #
@@ -657,5 +694,3 @@ log4j2.logger.migrationToolkit.name=com.sap.cx.boosters.commercedbsync
 log4j2.logger.migrationToolkit.level=INFO
 log4j2.logger.migrationToolkit.appenderRef.migration.ref=MigrationAppender
 log4j2.logger.migrationToolkit.additivity=false
-
-

commercedbsync/resources/commercedbsync-beans.xml

@@ -153,6 +153,7 @@
         <property name="lastColumnValue" type="Object" />
         <property name="nextColumnValue" type="Object" />
         <property name="batchSize" type="long" />
+        <property name="partition" type="String" />
     </bean>
 
     <bean class="com.sap.cx.boosters.commercedbsync.OffsetQueryDefinition">
@@ -170,6 +171,7 @@
         <property name="batchSize" type="long" />
         <property name="lowerColumnBoundary" type="Object" />
         <property name="upperColumnBoundary" type="Object" />
+        <property name="partition" type="String" />
     </bean>
 
      <bean class="com.sap.cx.boosters.commercedbsync.SeekQueryDefinition">

commercedbsync/resources/commercedbsync-spring.xml

@@ -60,11 +60,22 @@
         <constructor-arg name="dataRepositoryFactory" ref="dataRepositoryFactory"/>
         <constructor-arg name="dataSourceConfigurationFactory" ref="dataSourceConfigurationFactory"/>
         <constructor-arg name="configuration" value="#{configurationService.getConfiguration()}"/>
+        <constructor-arg name="reversed" value="false"/>
+    </bean>
+
+    <bean name="defaultReversedMigrationContextFactory" class="com.sap.cx.boosters.commercedbsync.context.MigrationContextFactory">
+        <constructor-arg name="dataRepositoryFactory" ref="dataRepositoryFactory"/>
+        <constructor-arg name="dataSourceConfigurationFactory" ref="dataSourceConfigurationFactory"/>
+        <constructor-arg name="configuration" value="#{configurationService.getConfiguration()}"/>
+        <constructor-arg name="reversed" value="true"/>
     </bean>
 
     <alias name="defaultMigrationContext" alias="migrationContext"/>
     <bean name="defaultMigrationContext" factory-bean="defaultMigrationContextFactory" factory-method="create"/>
 
+    <alias name="defaultReverseIncrementalMigrationContext" alias="reverseIncrementalMigrationContext"/>
+    <bean name="defaultReverseIncrementalMigrationContext" factory-bean="defaultReversedMigrationContextFactory" factory-method="create"/>
+
     <alias name="defaultMigrationContextValidator" alias="migrationContextValidator"/>
     <bean name="defaultMigrationContextValidator" class="com.sap.cx.boosters.commercedbsync.context.validation.impl.DefaultMigrationContextValidator">
         <property name="configurationService" ref="configurationService" />
@@ -202,6 +213,7 @@
         <property name="configurationService" ref="configurationService" />
         <property name="databaseCopyTaskRepository" ref="databaseCopyTaskRepository"/>
         <property name="migrationContext" ref="migrationContext"/>
+        <property name="reverseMigrationContext" ref="reverseIncrementalMigrationContext"/>
         <property name="performanceProfiler" ref="performanceProfiler"/>
     </bean>
 
@@ -261,6 +273,7 @@
         <property name="migrationContext" ref="migrationContext"/>
         <property name="databaseCopyTaskRepository" ref="databaseCopyTaskRepository"/>
         <property name="performanceProfiler" ref="performanceProfiler"/>
+        <property name="reverseMigrationContext" ref="reverseIncrementalMigrationContext"/>
         <property name="postProcessors">
             <util:list>
                 <ref bean="migrationPostProcessor"/>
@@ -371,7 +384,15 @@
 
   <bean id="defaultCMTAfterSaveListener" class="com.sap.cx.boosters.commercedbsync.listeners.DefaultCMTAfterSaveListener">
     <property name="modelService" ref="modelService" />
-    <property name="typeService" ref="typeService" />
   </bean>
 
+    <bean class="com.sap.cx.boosters.commercedbsync.jobs.IncrementalMigrationJob" id="reverseIncrementalMigrationJob"
+          parent="abstractMigrationJobPerformable">
+        <property name="migrationContext" ref="reverseIncrementalMigrationContext"/>
+    </bean>
+
+    <bean class="com.sap.cx.boosters.commercedbsync.jobs.FullMigrationJob" id="reverseFullMigrationJob"
+          parent="abstractMigrationJobPerformable">
+        <property name="migrationContext" ref="reverseIncrementalMigrationContext"/>
+    </bean>
 </beans>

commercedbsync/resources/impex/projectdata-commercemigration-jobs.impex

@@ -2,6 +2,8 @@
 INSERT_UPDATE ServicelayerJob;code[unique=true];springId[unique=true]
 ;incrementalMigrationJob;incrementalMigrationJob
 ;fullMigrationJob;fullMigrationJob
+;reverseIncrementalMigrationJob;reverseIncrementalMigrationJob
+;reverseFullMigrationJob;reverseFullMigrationJob
 ;migrationPrepJob;migrationPrepJob
 
 # Update details for incremental migration
@@ -15,24 +17,39 @@ INSERT_UPDATE FullMigrationCronJob;code[unique=true];job(code)[default=fullMigra
 ;fullDatabaseMigrationJob;;true;true;true;true;true;;mediaformatmapping,cat2attrrellp,categories,compositeentries,mediafolders,mediacontextlp,validationconstraintslp,validationconstraints,catalogslp,units,genericitems,pcp2wrtblecvrel,renderertemplate,dynamiccontent,userrightslp,backofficesearchcond,metainformations,unitslp,workflowactions,productprops,scripts,systemsetupaudit,gentestitems,cat2princrel,jalovelocityrenderer,paymentmodeslp,usergroupprops,orderprops,userrights,workflowactionitemsrel,parserproperty,productfeatures,productreferences,commentcompreadrels,languageslp,syncjob2pcplrel,commentitemrelations,jobs,themes,discounts,catalogversionsyncjob,cat2catrel,categorieslp,syncjob2langrel,currencieslp,impexdocumentids,userprofiles,stdpaymmodevals,links,workflowitematts,products,backofficesavedquery,productslp,workflowtemplatelinkrel,previewtickets,backofficecollections,props,retentionrule,syncjob2typerel,commentcompremoverels,genericitemslp,addresses,catalogs,languages,taxeslp,discountslp,distributedbatches,backofficesavedquerylp,searchrestrictions,aclentries,format2medforrel,keywords,paymentmodes,whereparts,commentassignrelations,commentattachments,discountrows,mediacontainerlp,commentdomains,synattcfg,mediacontext,impbatchcontent,classificationattrslp,commenttypes,globaldiscountrows,mediacontainer,searchrestrictionslp,mediaformatlp,catverdiffs,cmptype2covgrprels,workflowtemplprincrel,clattruntlp,jobslp,titles,pendingstepsrelation,themeslp,countries,commentcompwriterels,processedstepsrelation,slactions,productreferenceslp,usergroups,regionslp,userprops,exportslp,numberseries,distributedprocesses,catalogversions,externalimportkey,usergroupslp,cat2attrrel,medias,jobsearchrestriction,triggerscj,addressprops,openidexternalscopes,attr2valuerel,constraintgroup,renderertemplatelp,titleslp,indextestitem,workflowactionlinkrel,workflowactionslp,catalogversionslp,commentwatchrelations,configitems,pcpl2rdblecvrel,abstrcfgproductinfo,users,workflowitemattslp,commentcompcreaterels,derivedmedias,cat2medrel,scriptslp,regions,currencies,steps,deliverymodeslp,classattrvalueslp,mediaformat,zonedeliverymodevalues,configuratorsettings,prod2keywordrel,cat2prodrel,taxes,cat2keywordrel,classattrvalues,ydeployments,cstrgr2abscstrrel,mediaprops,pgrels,zone2country,classificationattrs,taxrows,renderersproperty,cronjobs,commentcomponents,exports,deliverymodes,comments,workflowactioncomments,countrieslp,commentusersettings,format2comtyprel,corsconfigproperty,backofficecollitemrefs,pricerows,agreements,workflowactionsrel,clattrunt,format,changedescriptors,formatlp,zones
 ;fullTableMigrationJob;;true;true;false;false;true;;products,paymentmodes
 
+INSERT_UPDATE IncrementalMigrationCronJob; code[unique = true]             ; migrationItems; active[default=true]; job(code)[default = reverseIncrementalMigrationJob]; sessionLanguage(isoCode)[default = en]
+                                         ; reverseIncrementalMigrationJob ; products
+
+INSERT_UPDATE FullMigrationCronJob; code[unique = true]         ; truncateEnabled; migrationItems; job(code)[default = reverseFullMigrationJob]; sessionLanguage(isoCode)[default = en]
+                                  ; reverseFullMigrationJob ; false           ; products
+
 INSERT_UPDATE CronJob;code[unique=true];job(code);useReadOnlyDatasource
                      ;migrationPrepCronJob;migrationPrepJob;false
 
 INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code)
                             ;prepForTableMigrationEntry;migrationPrepCronJob
                             ;prepForDbMigrationEntry;migrationPrepCronJob
                             ;prepForIncrementalMigrationEntry;migrationPrepCronJob
+                            ;prepForReverseTableMigrationEntry;migrationPrepCronJob
+                            ;prepForReverseIncrementalMigrationEntry;migrationPrepCronJob
                             ;migrateDatabaseEntry;fullDatabaseMigrationJob
                             ;migrateTableEntry;fullTableMigrationJob
                             ;migrateIncrementalEntry;incrementalMigrationJob
+                            ;reverseMigrateTableEntry;reverseFullMigrationJob
+                            ;reverseMigrateIncrementalEntry;reverseIncrementalMigrationJob
 
 INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default=en];
                               ;compositeDatabaseMigrationJob;compositeJobPerformable;
                               ;compositeTableMigrationJob;compositeJobPerformable;
                               ;compositeIncrementalMigrationJob;compositeJobPerformable;
+                              ;compositeReverseTableMigrationJob;compositeJobPerformable;
+                              ;compositeReverseIncrementalMigrationJob;compositeJobPerformable;
 
 INSERT_UPDATE CompositeCronJob;code[unique=true];compositeEntries(code)[mode = append]
                               ;compositeDatabaseMigrationJob;prepForDbMigrationEntry,migrateDatabaseEntry
                               ;compositeTableMigrationJob;prepForTableMigrationEntry,migrateTableEntry
                               ;compositeIncrementalMigrationJob;prepForIncrementalMigrationEntry,migrateIncrementalEntry
+                              ;compositeReverseTableMigrationjob;prepForReverseTableMigrationEntry,reverseMigrateTableEntry
+                              ;compositeReverseIncrementalMigrationJob;prepForReverseIncrementalMigrationEntry,reverseMigrateIncrementalEntry
+
 

commercedbsync/resources/sql/createSchedulerTablesHANA.sql

@@ -19,6 +19,12 @@ IF tablename = 'MIGRATIONTOOLKIT_TABLECOPYBATCHES' AND :found > 0
 	THEN
 DROP TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES;
 END IF;
+
+IF tablename = 'MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART' AND :found > 0
+	THEN
+DROP TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART;
+END IF;
+
 END;
 #
 CALL MIGRATION_PROCEDURE('MIGRATIONTOOLKIT_TABLECOPYTASKS');
@@ -28,6 +34,7 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYTASKS (
                                                  targetnodeId int NOT NULL,
                                                  migrationId NVARCHAR(255) NOT NULL,
                                                  pipelinename NVARCHAR(255) NOT NULL,
+                                                 itemorder int NOT NULL DEFAULT 0,
                                                  sourcetablename NVARCHAR(255) NOT NULL,
                                                  targettablename NVARCHAR(255) NOT NULL,
                                                  columnmap NVARCHAR(5000) NULL,
@@ -67,6 +74,21 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES (
 
 #
 
+CALL MIGRATION_PROCEDURE('MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART');
+#
+
+CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART (
+                                                   migrationId NVARCHAR(255) NOT NULL,
+                                                   batchId int NOT NULL DEFAULT 0,
+                                                   pipelinename NVARCHAR(255) NOT NULL,
+                                                   lowerBoundary NVARCHAR(255) NOT NULL,
+                                                   upperBoundary NVARCHAR(255) NULL,
+                                                   partition VARCHAR(128) NOT NULL,
+                                                   PRIMARY KEY (migrationid, batchId, pipelinename, partition)
+);
+
+#
+
 CALL MIGRATION_PROCEDURE('MIGRATIONTOOLKIT_TABLECOPYSTATUS');
 #
 

commercedbsync/resources/sql/createSchedulerTablesMSSQL.sql

@@ -5,6 +5,7 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYTASKS (
     targetnodeId int NOT NULL,
     migrationId NVARCHAR(255) NOT NULL,
     pipelinename NVARCHAR(255) NOT NULL,
+    itemorder int NOT NULL DEFAULT 0,
     sourcetablename NVARCHAR(255) NOT NULL,
     targettablename NVARCHAR(255) NOT NULL,
     columnmap NVARCHAR(MAX) NULL,
@@ -39,6 +40,18 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES (
     PRIMARY KEY (migrationid, batchId, pipelinename)
 );
 
+DROP TABLE IF EXISTS MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART;
+
+CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART (
+                                                   migrationId NVARCHAR(255) NOT NULL,
+                                                   batchId int NOT NULL DEFAULT 0,
+                                                   pipelinename NVARCHAR(255) NOT NULL,
+                                                   lowerBoundary NVARCHAR(255) NOT NULL,
+                                                   upperBoundary NVARCHAR(255) NULL,
+                                                   partition VARCHAR(128) NOT NULL,
+                                                   PRIMARY KEY (migrationid, batchId, pipelinename, partition)
+);
+
 DROP TABLE IF EXISTS MIGRATIONTOOLKIT_TABLECOPYSTATUS;
 
 CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYSTATUS (

commercedbsync/resources/sql/createSchedulerTablesMYSQL.sql

@@ -5,6 +5,7 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYTASKS
     targetnodeId           int          NOT NULL,
     migrationId            VARCHAR(255) NOT NULL,
     pipelinename           VARCHAR(255) NOT NULL,
+    itemorder              int NOT NULL DEFAULT 0,
     sourcetablename        VARCHAR(255) NOT NULL,
     targettablename        VARCHAR(255) NOT NULL,
     columnmap              TEXT NULL,
@@ -40,6 +41,19 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES
     PRIMARY KEY (migrationid, batchId, pipelinename)
 );
 #
+DROP TABLE IF EXISTS MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART;
+#
+CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART
+(
+    migrationId   VARCHAR(255) NOT NULL,
+    batchId       int          NOT NULL DEFAULT 0,
+    pipelinename  VARCHAR(255) NOT NULL,
+    lowerBoundary VARCHAR(255) NOT NULL,
+    upperBoundary VARCHAR(255) NULL,
+    partition     VARCHAR(128) NOT NULL,
+    PRIMARY KEY (migrationid, batchId, pipelinename, partition)
+);
+#
 DROP TABLE IF EXISTS MIGRATIONTOOLKIT_TABLECOPYSTATUS;
 #
 CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYSTATUS

commercedbsync/resources/sql/createSchedulerTablesORACLE.sql

@@ -11,6 +11,7 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYTASKS (
     targetnodeId number(10) NOT NULL,
     migrationId NVARCHAR2(255) NOT NULL,
     pipelinename NVARCHAR2(255) NOT NULL,
+    itemorder int DEFAULT 0 NOT NULL,
     sourcetablename NVARCHAR2(255) NOT NULL,
     targettablename NVARCHAR2(255) NOT NULL,
     columnmap CLOB NULL,
@@ -57,6 +58,27 @@ CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES (
 
 
 
+BEGIN
+    EXECUTE IMMEDIATE 'DROP TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART';
+EXCEPTION
+    WHEN OTHERS THEN NULL;
+END;
+/
+
+
+CREATE TABLE MIGRATIONTOOLKIT_TABLECOPYBATCHES_PART (
+    migrationId NVARCHAR2(255) NOT NULL,
+    batchId number(10) DEFAULT 0 NOT NULL,
+    pipelinename NVARCHAR2(255) NOT NULL,
+    lowerBoundary NVARCHAR2(255) NOT NULL,
+    upperBoundary NVARCHAR2(255) NULL,
+    partition VARCHAR(128) NOT NULL,
+    PRIMARY KEY (migrationid, batchId, pipelinename, partition)
+)
+/
+
+
+
 
 BEGIN
    EXECUTE IMMEDIATE 'DROP TABLE MIGRATIONTOOLKIT_TABLECOPYSTATUS';