Feature/add ci pipeline

.github/workflows/ci-pipeline.yml

@@ -0,0 +1,87 @@
+# This workflow will build a .NET project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net
+
+name: CI - Pipeline
+permissions:
+  contents: read
+  issues: read
+  checks: write
+  pull-requests: write
+on:
+  push:
+    branches: [ "main","release/**/*" ]
+  pull_request:
+    branches: [ "main","release/*" ]
+env:
+  solutionPath: '${{ github.workspace }}/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.sln'
+  solutionFolder: '${{ github.workspace }}/src/Dataverse.ConfigurationMigrationTool'
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: 8.0.x
+    - name: Restore dependencies
+      run: dotnet restore ${{  env.solutionPath  }}
+    - name: Build
+      run: dotnet build ${{  env.solutionPath  }} --configuration Release --no-restore
+    - name: Execute unit tests
+      run: dotnet test ${{  env.solutionPath  }} --configuration Release --settings ${{  env.solutionFolder  }}/Console.Tests/CodeCoverage.runsettings --no-build --logger trx --no-restore --results-directory "TestResults" --collect:"XPlat code coverage"
+    - name: Publish Test Report
+      uses: phoenix-actions/test-reporting@v8
+      id: test-report               # Set ID reference for step
+      if: ${{ (success() || failure()) && (github.event_name == 'pull_request')   }}     # run this step even if previous step failed
+      with:
+        name: unit tests            # Name of the check run which will be created
+        path: TestResults/*.trx    # Path to test results
+        reporter: dotnet-trx       # Format of test results
+    - name: Code Coverage Report
+      uses: irongut/[email protected]
+      if: github.event_name == 'pull_request'
+      with:
+        filename: TestResults/*/coverage.cobertura.xml
+        badge: true
+        fail_below_min: true
+        format: markdown
+        hide_branch_rate: false
+        hide_complexity: true
+        indicators: true
+        output: both
+        thresholds: '60 60'
+    - name: Add Coverage PR Comment
+      uses: marocchino/sticky-pull-request-comment@v2
+      if: github.event_name == 'pull_request'
+      with:
+        recreate: true
+        path: code-coverage-results.md
+
+    - name: ReportGenerator
+      uses: danielpalme/[email protected]
+      if: github.event_name == 'pull_request'
+      with:
+        reports: TestResults/*/coverage.cobertura.xml # REQUIRED # The coverage reports that should be parsed (separated by semicolon). Globbing is supported.
+        targetdir: 'coveragereport' # REQUIRED # The directory where the generated report should be saved.
+        reporttypes: 'HtmlInline;Cobertura' # The output formats and scope (separated by semicolon) Values: Badges, Clover, Cobertura, OpenCover, CsvSummary, Html, Html_Dark, Html_Light, Html_BlueRed, HtmlChart, HtmlInline, HtmlInline_AzurePipelines, HtmlInline_AzurePipelines_Dark, HtmlInline_AzurePipelines_Light, HtmlSummary, Html_BlueRed_Summary, JsonSummary, Latex, LatexSummary, lcov, MarkdownSummary, MarkdownSummaryGithub, MarkdownDeltaSummary, MHtml, SvgChart, SonarQube, TeamCitySummary, TextSummary, TextDeltaSummary, Xml, XmlSummary
+        tag: '${{ github.run_number }}_${{ github.run_id }}' # Optional tag or build version.
+        toolpath: 'reportgeneratortool' # Default directory for installing the dotnet tool.
+
+    - name: Upload coverage report artifact
+      uses: actions/upload-artifact@v4
+      if: github.event_name == 'pull_request'
+      with:
+        name: CoverageReport # Artifact name        
+        path: coveragereport # Directory containing files to upload
+
+
+
+
+
+
+
+
+

src/Dataverse.ConfigurationMigrationTool/Console.Tests/CodeCoverage.runsettings

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<RunSettings>
+	<DataCollectionRunSettings>
+		<DataCollectors>
+			<DataCollector friendlyName="XPlat code coverage">
+				<Configuration>
+					<Format>cobertura</Format>
+					<Exclude>[*]Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse*</Exclude>
+					<ExcludeByFile>**/Program.cs,</ExcludeByFile>
+					<SkipAutoProps>true</SkipAutoProps>
+				</Configuration>
+			</DataCollector>
+		</DataCollectors>
+	</DataCollectionRunSettings>
+</RunSettings>

src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs

@@ -53,6 +53,7 @@ await ImportRelationships(entityMD, task, entityToImport) :
 
     private async Task<TaskResult> ImportRelationships(EntityMetadata entity, ImportDataTask task, EntityImport entityImport)
     {
+        this.logger.LogInformation("Importing relationship : {relationshipname} | for entity source : {entityname}", task.RelationshipSchema.Name, entityImport.Name);
         var relMD = entity.ManyToManyRelationships.FirstOrDefault(m => m.IntersectEntityName == task.RelationshipSchema.Name);
         if (relMD == null) return TaskResult.Failed;
 
@@ -95,15 +96,16 @@ private async Task<TaskResult> ImportRelationships(EntityMetadata entity, Import
     }
     private async Task<TaskResult> ImportRecords(EntityMetadata entity, ImportDataTask task, EntityImport entityImport)
     {
-
+        logger.LogInformation("Importing {entityname} records", entityImport.Name);
         var recordsWithNoSelfDependancies = entityImport.Records.Record.Where(r =>
         !r.Field.Any(f => f.Lookupentity == entityImport.Name &&
                          entityImport.Records.Record.Any(r2 => r2.Id != r.Id && r2.Id.ToString() == f.Value))).Select(r => BuildUpsertRequest(entity, entityImport, r)).ToList();
         var recordsWithSelfDependancies = entityImport.Records.Record.Where(r =>
         r.Field.Any(f => f.Lookupentity == entityImport.Name &&
                          entityImport.Records.Record.Any(r2 => r2.Id != r.Id && r2.Id.ToString() == f.Value))).ToList();
 
-
+        logger.LogInformation("records with no self dependancies: {count}", recordsWithNoSelfDependancies.Count);
+        logger.LogInformation("records with self dependancies: {count}", recordsWithSelfDependancies.Count);
         //See if upsert request keep ids
 
         //implement parallelism and batching
@@ -172,6 +174,12 @@ private async Task<IEnumerable<OrganizationResponseFaultedResult>> ProcessDepend
             }
 
         }
+        var maxretries = retries.Where(kv => kv.Value >= MAX_RETRIES).Select(kv => kv.Key).ToList();
+        if (maxretries.Any())
+        {
+            logger.LogWarning("The following records ({count}) were not processed due to circular dependencies: {ids}", maxretries.Count, string.Join(", ", maxretries));
+        }
+
         return results;
 
     }

src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs

@@ -2,51 +2,27 @@
 using Dataverse.ConfigurationMigrationTool.Console.Features.Shared;
 using Microsoft.Xrm.Sdk.Metadata;
 
-namespace Dataverse.ConfigurationMigrationTool.Console.Features.Import.Mappers
+namespace Dataverse.ConfigurationMigrationTool.Console.Features.Import.Mappers;
+
+public class FieldSchemaToAttributeTypeMapper : IMapper<FieldSchema, AttributeTypeCode?>
 {
-    public class FieldSchemaToAttributeTypeMapper : IMapper<FieldSchema, AttributeTypeCode?>
+    public AttributeTypeCode? Map(FieldSchema source) => source.Type switch
     {
-        public AttributeTypeCode? Map(FieldSchema source)
-        {
-            switch (source.Type)
-            {
-                case "string":
-                    return AttributeTypeCode.String;
-                case "guid":
-                    return AttributeTypeCode.Uniqueidentifier;
-                case "entityreference":
-                    if (source.LookupType == "account|contact")
-                    {
-                        return AttributeTypeCode.Customer;
-                    }
-                    return AttributeTypeCode.Lookup;
-                case "owner":
-                    return AttributeTypeCode.Owner;
-                case "state":
-                    return AttributeTypeCode.State;
-                case "status":
-                    return AttributeTypeCode.Status;
-                case "decimal":
-                    return AttributeTypeCode.Decimal;
-                case "optionsetvalue":
-                    return AttributeTypeCode.Picklist;
-                case "number":
-                    return AttributeTypeCode.Integer;
-                case "bigint":
-                    return AttributeTypeCode.BigInt;
-                case "float":
-                    return AttributeTypeCode.Double;
-                case "bool":
-                    return AttributeTypeCode.Boolean;
-                case "datetime":
-                    return AttributeTypeCode.DateTime;
-                case "money":
-                    return AttributeTypeCode.Money;
-                default:
-                    return null;
-
+        "string" => AttributeTypeCode.String,
+        "guid" => AttributeTypeCode.Uniqueidentifier,
+        "entityreference" => source.LookupType == "account|contact" ? AttributeTypeCode.Customer : AttributeTypeCode.Lookup,
+        "owner" => AttributeTypeCode.Owner,
+        "state" => AttributeTypeCode.State,
+        "status" => AttributeTypeCode.Status,
+        "decimal" => AttributeTypeCode.Decimal,
+        "optionsetvalue" => AttributeTypeCode.Picklist,
+        "number" => AttributeTypeCode.Integer,
+        "bigint" => AttributeTypeCode.BigInt,
+        "float" => AttributeTypeCode.Double,
+        "bool" => AttributeTypeCode.Boolean,
+        "datetime" => AttributeTypeCode.DateTime,
+        "money" => AttributeTypeCode.Money,
+        _ => null
+    };
 
-            }
-        }
-    }
 }

src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/appsettings.json

@@ -6,7 +6,7 @@
   },
   "Dataverse": {
     "MaxThreadCount": 100,
-    "MaxDegreeOfParallism": 1,
-    "BatchSize": 1
+    "MaxDegreeOfParallism": 5,
+    "BatchSize": 20
   }
 }