diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml new file mode 100644 index 0000000..6b5bdd3 --- /dev/null +++ b/.github/workflows/ci-pipeline.yml @@ -0,0 +1,87 @@ +# This workflow will build a .NET project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net + +name: CI - Pipeline +permissions: + contents: read + issues: read + checks: write + pull-requests: write +on: + push: + branches: [ "main","release/**/*" ] + pull_request: + branches: [ "main","release/*" ] +env: + solutionPath: '${{ github.workspace }}/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.sln' + solutionFolder: '${{ github.workspace }}/src/Dataverse.ConfigurationMigrationTool' +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 8.0.x + - name: Restore dependencies + run: dotnet restore ${{ env.solutionPath }} + - name: Build + run: dotnet build ${{ env.solutionPath }} --configuration Release --no-restore + - name: Execute unit tests + run: dotnet test ${{ env.solutionPath }} --configuration Release --settings ${{ env.solutionFolder }}/Console.Tests/CodeCoverage.runsettings --no-build --logger trx --no-restore --results-directory "TestResults" --collect:"XPlat code coverage" + - name: Publish Test Report + uses: phoenix-actions/test-reporting@v8 + id: test-report # Set ID reference for step + if: ${{ (success() || failure()) && (github.event_name == 'pull_request') }} # run this step even if previous step failed + with: + name: unit tests # Name of the check run which will be created + path: TestResults/*.trx # Path to test results + reporter: dotnet-trx # Format of test results + - name: Code Coverage Report + uses: irongut/CodeCoverageSummary@v1.3.0 + if: github.event_name == 'pull_request' + with: + filename: TestResults/*/coverage.cobertura.xml + badge: true + fail_below_min: true + format: markdown + hide_branch_rate: false + hide_complexity: true + indicators: true + output: both + thresholds: '60 60' + - name: Add Coverage PR Comment + uses: marocchino/sticky-pull-request-comment@v2 + if: github.event_name == 'pull_request' + with: + recreate: true + path: code-coverage-results.md + + - name: ReportGenerator + uses: danielpalme/ReportGenerator-GitHub-Action@5.2.0 + if: github.event_name == 'pull_request' + with: + reports: TestResults/*/coverage.cobertura.xml # REQUIRED # The coverage reports that should be parsed (separated by semicolon). Globbing is supported. + targetdir: 'coveragereport' # REQUIRED # The directory where the generated report should be saved. + reporttypes: 'HtmlInline;Cobertura' # The output formats and scope (separated by semicolon) Values: Badges, Clover, Cobertura, OpenCover, CsvSummary, Html, Html_Dark, Html_Light, Html_BlueRed, HtmlChart, HtmlInline, HtmlInline_AzurePipelines, HtmlInline_AzurePipelines_Dark, HtmlInline_AzurePipelines_Light, HtmlSummary, Html_BlueRed_Summary, JsonSummary, Latex, LatexSummary, lcov, MarkdownSummary, MarkdownSummaryGithub, MarkdownDeltaSummary, MHtml, SvgChart, SonarQube, TeamCitySummary, TextSummary, TextDeltaSummary, Xml, XmlSummary + tag: '${{ github.run_number }}_${{ github.run_id }}' # Optional tag or build version. + toolpath: 'reportgeneratortool' # Default directory for installing the dotnet tool. + + - name: Upload coverage report artifact + uses: actions/upload-artifact@v4 + if: github.event_name == 'pull_request' + with: + name: CoverageReport # Artifact name + path: coveragereport # Directory containing files to upload + + + + + + + + + diff --git a/src/Dataverse.ConfigurationMigrationTool/Console.Tests/CodeCoverage.runsettings b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/CodeCoverage.runsettings new file mode 100644 index 0000000..ad20e27 --- /dev/null +++ b/src/Dataverse.ConfigurationMigrationTool/Console.Tests/CodeCoverage.runsettings @@ -0,0 +1,15 @@ + + + + + + + cobertura + [*]Dataverse.ConfigurationMigrationTool.Console.Services.Dataverse* + **/Program.cs, + true + + + + + \ No newline at end of file diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs index 52c2b16..e796756 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/ImportTaskProcessorService.cs @@ -53,6 +53,7 @@ await ImportRelationships(entityMD, task, entityToImport) : private async Task ImportRelationships(EntityMetadata entity, ImportDataTask task, EntityImport entityImport) { + this.logger.LogInformation("Importing relationship : {relationshipname} | for entity source : {entityname}", task.RelationshipSchema.Name, entityImport.Name); var relMD = entity.ManyToManyRelationships.FirstOrDefault(m => m.IntersectEntityName == task.RelationshipSchema.Name); if (relMD == null) return TaskResult.Failed; @@ -95,7 +96,7 @@ private async Task ImportRelationships(EntityMetadata entity, Import } private async Task ImportRecords(EntityMetadata entity, ImportDataTask task, EntityImport entityImport) { - + logger.LogInformation("Importing {entityname} records", entityImport.Name); var recordsWithNoSelfDependancies = entityImport.Records.Record.Where(r => !r.Field.Any(f => f.Lookupentity == entityImport.Name && entityImport.Records.Record.Any(r2 => r2.Id != r.Id && r2.Id.ToString() == f.Value))).Select(r => BuildUpsertRequest(entity, entityImport, r)).ToList(); @@ -103,7 +104,8 @@ private async Task ImportRecords(EntityMetadata entity, ImportDataTa r.Field.Any(f => f.Lookupentity == entityImport.Name && entityImport.Records.Record.Any(r2 => r2.Id != r.Id && r2.Id.ToString() == f.Value))).ToList(); - + logger.LogInformation("records with no self dependancies: {count}", recordsWithNoSelfDependancies.Count); + logger.LogInformation("records with self dependancies: {count}", recordsWithSelfDependancies.Count); //See if upsert request keep ids //implement parallelism and batching @@ -172,6 +174,12 @@ private async Task> ProcessDepend } } + var maxretries = retries.Where(kv => kv.Value >= MAX_RETRIES).Select(kv => kv.Key).ToList(); + if (maxretries.Any()) + { + logger.LogWarning("The following records ({count}) were not processed due to circular dependencies: {ids}", maxretries.Count, string.Join(", ", maxretries)); + } + return results; } diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs index b4121e1..2f16db5 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/Features/Import/Mappers/FieldSchemaToAttributeTypeMapper.cs @@ -2,51 +2,27 @@ using Dataverse.ConfigurationMigrationTool.Console.Features.Shared; using Microsoft.Xrm.Sdk.Metadata; -namespace Dataverse.ConfigurationMigrationTool.Console.Features.Import.Mappers +namespace Dataverse.ConfigurationMigrationTool.Console.Features.Import.Mappers; + +public class FieldSchemaToAttributeTypeMapper : IMapper { - public class FieldSchemaToAttributeTypeMapper : IMapper + public AttributeTypeCode? Map(FieldSchema source) => source.Type switch { - public AttributeTypeCode? Map(FieldSchema source) - { - switch (source.Type) - { - case "string": - return AttributeTypeCode.String; - case "guid": - return AttributeTypeCode.Uniqueidentifier; - case "entityreference": - if (source.LookupType == "account|contact") - { - return AttributeTypeCode.Customer; - } - return AttributeTypeCode.Lookup; - case "owner": - return AttributeTypeCode.Owner; - case "state": - return AttributeTypeCode.State; - case "status": - return AttributeTypeCode.Status; - case "decimal": - return AttributeTypeCode.Decimal; - case "optionsetvalue": - return AttributeTypeCode.Picklist; - case "number": - return AttributeTypeCode.Integer; - case "bigint": - return AttributeTypeCode.BigInt; - case "float": - return AttributeTypeCode.Double; - case "bool": - return AttributeTypeCode.Boolean; - case "datetime": - return AttributeTypeCode.DateTime; - case "money": - return AttributeTypeCode.Money; - default: - return null; - + "string" => AttributeTypeCode.String, + "guid" => AttributeTypeCode.Uniqueidentifier, + "entityreference" => source.LookupType == "account|contact" ? AttributeTypeCode.Customer : AttributeTypeCode.Lookup, + "owner" => AttributeTypeCode.Owner, + "state" => AttributeTypeCode.State, + "status" => AttributeTypeCode.Status, + "decimal" => AttributeTypeCode.Decimal, + "optionsetvalue" => AttributeTypeCode.Picklist, + "number" => AttributeTypeCode.Integer, + "bigint" => AttributeTypeCode.BigInt, + "float" => AttributeTypeCode.Double, + "bool" => AttributeTypeCode.Boolean, + "datetime" => AttributeTypeCode.DateTime, + "money" => AttributeTypeCode.Money, + _ => null + }; - } - } - } } diff --git a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/appsettings.json b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/appsettings.json index dce876f..90d0ab1 100644 --- a/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/appsettings.json +++ b/src/Dataverse.ConfigurationMigrationTool/Dataverse.ConfigurationMigrationTool.Console/appsettings.json @@ -6,7 +6,7 @@ }, "Dataverse": { "MaxThreadCount": 100, - "MaxDegreeOfParallism": 1, - "BatchSize": 1 + "MaxDegreeOfParallism": 5, + "BatchSize": 20 } }