[GOBBLIN-2225] Use random id for Workunit if recovery helper is not initialised (#4134)

* Add unique id to CopyEntity instead of computing it from serialised json string

* Add unique id to CopyEntity instead of computing it from serialised json string

Author: thisisArjit

gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopyEntity.java

@@ -118,7 +118,9 @@ public static List<CopyEntity> deserializeList(String serialized) {
    */
   public static String getSerializedWithNewPackage(String serialized) {
     serialized = serialized.replace("\"gobblin.data.management.", "\"org.apache.gobblin.data.management.");
-    log.debug("Serialized updated copy entity: " + serialized);
+    if (log.isDebugEnabled()) {
+      log.debug("Serialized updated copy entity: " + serialized);
+    }
     return serialized;
   }
 

gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/CopySource.java

@@ -19,10 +19,12 @@
 
 import java.io.IOException;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
@@ -54,6 +56,7 @@
 import org.apache.gobblin.data.management.copy.extractor.FileAwareInputStreamExtractor;
 import org.apache.gobblin.data.management.copy.prioritization.FileSetComparator;
 import org.apache.gobblin.data.management.copy.publisher.CopyEventSubmitterHelper;
+import org.apache.gobblin.data.management.copy.recovery.RecoveryHelper;
 import org.apache.gobblin.data.management.copy.replication.ConfigBasedDataset;
 import org.apache.gobblin.data.management.copy.splitter.DistcpFileSplitter;
 import org.apache.gobblin.data.management.copy.watermark.CopyableFileWatermarkGenerator;
@@ -372,6 +375,10 @@ public Void call() {
 
         Extract extract = new Extract(Extract.TableType.SNAPSHOT_ONLY, CopyConfiguration.COPY_PREFIX, extractId);
         List<WorkUnit> workUnitsForPartition = Lists.newArrayList();
+        // use random guid for workunits if recovery is disabled. Getting deterministic guid is expensive as it
+        // serialises entire workunit to generate guid
+        boolean useRandomGuidForWorkUnit = !isRecoveryEnabled(state);
+        log.info("Using " + (useRandomGuidForWorkUnit ? "random" : "deterministic") + " guids for workunits");
 
         long fileSize;
         for (CopyEntity copyEntity : fileSet.getFiles()) {
@@ -400,7 +407,7 @@ public Void call() {
           workUnit.setProp(SlaEventKeys.PARTITION_KEY, copyEntity.getFileSet());
           setWorkUnitWeight(workUnit, copyEntity, minWorkUnitWeight);
           setWorkUnitWatermark(workUnit, watermarkGenerator, copyEntity);
-          computeAndSetWorkUnitGuid(workUnit, copyEntity);
+          computeAndSetWorkUnitGuid(workUnit, copyEntity, useRandomGuidForWorkUnit);
           addLineageInfo(copyEntity, workUnit);
           if (copyEntity instanceof CopyableFile) {
             CopyableFile castedCopyEntity = (CopyableFile) copyEntity;
@@ -427,6 +434,10 @@ public Void call() {
       }
     }
 
+    private static boolean isRecoveryEnabled(State state) {
+      return state.contains(RecoveryHelper.PERSIST_DIR_KEY);
+    }
+
     private void setWorkUnitWatermark(WorkUnit workUnit, Optional<CopyableFileWatermarkGenerator> watermarkGenerator,
         CopyEntity copyEntity)
         throws IOException {
@@ -514,11 +525,12 @@ private static void setWorkUnitWeight(WorkUnit workUnit, CopyEntity copyEntity,
     workUnit.setProp(WORK_UNIT_WEIGHT, Long.toString(weight));
   }
 
-  private static void computeAndSetWorkUnitGuid(WorkUnit workUnit, CopyEntity copyEntity)
+  private static void computeAndSetWorkUnitGuid(WorkUnit workUnit, CopyEntity copyEntity, boolean useRandomGuidForWorkUnit)
       throws IOException {
     Guid guid = Guid.fromStrings(workUnit.contains(ConfigurationKeys.CONVERTER_CLASSES_KEY) ? workUnit
         .getProp(ConfigurationKeys.CONVERTER_CLASSES_KEY) : "");
-    setWorkUnitGuid(workUnit, guid.append(copyEntity.guid()));
+    Guid copyEntityGuid = useRandomGuidForWorkUnit ? new Guid(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)) : copyEntity.guid();
+    setWorkUnitGuid(workUnit, guid.append(copyEntityGuid));
   }
 
   /**