fix to decode and encode as utf8 with html decode utility.

fixing some schema validation rules
fixing some value converters

Author: dotnetprog

README.md

@@ -4,8 +4,9 @@ Custom .net cli tool to import configuration data into a dataverse environment
 
 ## Todo list:
 
-- Functionnal testing for self hierarchy import jobs.
-- Add support to map users to owner.
+- Functionnal testing for self hierarchy import jobs.?
+- Add support for special characters. ?
+- <del> Add support to map users to owner.</del>
 - Add support to map created on to OverrideCreatedOn.
 - Refactor Schema validations into multiple validation rules
 - Add configuration for tool reporting

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs

@@ -178,7 +178,9 @@ private UpsertRequest BuildUpsertRequest(EntityMetadata entityMD, EntityImport e
             foreach (var field in record.Field)
             {
                 var attrMD = entityMD.Attributes.FirstOrDefault(a => a.LogicalName == field.Name);
-                if ((attrMD.IsValidForCreate != true && attrMD.LogicalName != "statecode") || attrMD.IsValidForUpdate != true) continue;
+                if ((attrMD.IsValidForCreate != true && attrMD.LogicalName != "statecode")
+                    || attrMD.IsValidForUpdate != true ||
+                    attrMD.AttributeType == AttributeTypeCode.Owner) continue;
                 entity[field.Name] = _dataverseValueConverter.Convert(attrMD, field);
             }
             return new UpsertRequest

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs

@@ -20,6 +20,8 @@ public class FieldSchemaToAttributeTypeMapper : IMapper<FieldSchema, AttributeTy
                         return AttributeTypeCode.Customer;
                     }
                     return AttributeTypeCode.Lookup;
+                case "owner":
+                    return AttributeTypeCode.Owner;
                 case "state":
                     return AttributeTypeCode.State;
                 case "status":
@@ -30,6 +32,8 @@ public class FieldSchemaToAttributeTypeMapper : IMapper<FieldSchema, AttributeTy
                     return AttributeTypeCode.Picklist;
                 case "number":
                     return AttributeTypeCode.Integer;
+                case "bigint":
+                    return AttributeTypeCode.BigInt;
                 case "float":
                     return AttributeTypeCode.Double;
                 case "bool":

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Validators/SchemaValidator.cs

@@ -58,7 +58,7 @@ public async Task<ValidationResult> Validate(ImportSchema value)
                         });
                         continue;
                     }
-                    if (attributeMetadata is LookupAttributeMetadata lookupMD && !lookupMD.Targets.AreEnumerablesEqualIgnoreOrder(fieldSchema.LookupType?.Split('|') ?? Array.Empty<string>()))
+                    if (attributeMetadata is LookupAttributeMetadata lookupMD && lookupMD.AttributeType != AttributeTypeCode.Owner && !lookupMD.Targets.AreEnumerablesEqualIgnoreOrder(fieldSchema.LookupType?.Split('|') ?? Array.Empty<string>()))
                     {
                         failures.Add(new ValidationFailure
                         {

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ValueConverters/DataverseValueConverter.cs

@@ -1,6 +1,7 @@
 using Dataverse.ConfigurationMigrationTool.Console.Features.Import.Model;
 using Microsoft.Xrm.Sdk;
 using Microsoft.Xrm.Sdk.Metadata;
+using System.Web;
 
 namespace Dataverse.ConfigurationMigrationTool.Console.Features.Import.ValueConverters
 {
@@ -25,7 +26,7 @@ public object Convert(AttributeMetadata attributeMetadata, Field field)
             switch (attributeMetadata.AttributeType.Value)
             {
                 case AttributeTypeCode.String:
-                    return value;
+                    return string.IsNullOrWhiteSpace(value) ? value : HttpUtility.HtmlDecode(value);
                 case AttributeTypeCode.Picklist:
                 case AttributeTypeCode.State:
                 case AttributeTypeCode.Status:

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/Services/Filesystem/XmlFileDataReader.cs

@@ -1,12 +1,13 @@
-using System.Xml.Serialization;
+using System.Text;
+using System.Xml.Serialization;
 
 namespace Dataverse.ConfigurationMigrationTool.Console.Services.Filesystem
 {
     public class XmlFileDataReader : IFileDataReader
     {
         public async Task<T> ReadAsync<T>(string path)
         {
-            var xml = await File.ReadAllTextAsync(path);
+            var xml = await File.ReadAllTextAsync(path, Encoding.UTF8);
             XmlSerializer serializer = new XmlSerializer(typeof(T));
             using (StringReader reader = new StringReader(xml))
             {

src/Dataverse.ConfigurationMigrationTool.Console/Dataverse.ConfigurationMigrationTool.Console/appsettings.json

@@ -6,7 +6,7 @@
   },
   "Dataverse": {
     "MaxThreadCount": 100,
-    "MaxDegreeOfParallism": 10,
-    "BatchSize": 10
+    "MaxDegreeOfParallism": 1,
+    "BatchSize": 1
   }
 }