Removed redundant duplicate detection

Removed redundant duplicate detection by removing rownum window function

Author: sanjeev-k

templates/Copy Dataverse data into Azure SQL/DataverseToSQL_Orchestrator.json

@@ -343,7 +343,7 @@
         },
         "annotations": [
         ],
-        "lastPublishTime": "2022-10-25T21:59:00Z"
+        "lastPublishTime": "2023-01-20T06:39:09Z"
       },
       "dependsOn": [
         "[concat(variables('factoryId'), '/datasets/DataverseToSQLDestination')]",
@@ -1088,7 +1088,7 @@
         },
         "annotations": [
         ],
-        "lastPublishTime": "2022-10-24T21:31:28Z"
+        "lastPublishTime": "2022-10-28T09:11:04Z"
       },
       "dependsOn": [
         "[concat(variables('factoryId'), '/datasets/OptionsetMetadataSourceDataset')]",
@@ -1481,12 +1481,6 @@
             },
             {
               "name": "SelectDeleteLogColumns"
-            },
-            {
-              "name": "RankRowNum"
-            },
-            {
-              "name": "DeDupRowNum"
             }
           ],
           "scriptLines": [
@@ -1529,7 +1523,7 @@
             "     desc(SinkModifiedOn, false),",
             "     {_RowVersion} = denseRank()) ~> Rank",
             "Rank filter(toLong(byName(\"_RowVersion\"))==1) ~> Dedup",
-            "DeDupRowNum, getMaxVersion join(versionnumber > maxversion,",
+            "Dedup, getMaxVersion join(versionnumber > maxversion,",
             "     joinType:'left',",
             "     matchType:'exact',",
             "     ignoreSpaces: false,",
@@ -1544,12 +1538,6 @@
             "     ),",
             "     skipDuplicateMapInputs: false,",
             "     skipDuplicateMapOutputs: true) ~> SelectDeleteLogColumns",
-            "Dedup window(over(Id),",
-            "     desc(Id, false),",
-            "     desc(versionnumber, false),",
-            "     desc(SinkModifiedOn, false),",
-            "     {_RowNum} = rowNumber()) ~> RankRowNum",
-            "RankRowNum filter(toLong(byName(\"_RowNum\"))==1) ~> DeDupRowNum",
             "AlterRow sink(allowSchemaDrift: true,",
             "     validateSchema: false,",
             "     format: 'table',",