diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..c041836 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*/**.csproj] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/dotnet-build-debug.yml b/.github/workflows/dotnet-build-debug.yml new file mode 100644 index 0000000..f339a07 --- /dev/null +++ b/.github/workflows/dotnet-build-debug.yml @@ -0,0 +1,50 @@ +name: Build .NET (Debug) + +on: + workflow_dispatch: # Enable manual trigger to run workflow. + push: + branches: ["main"] + pull_request: + branches: ["main"] + schedule: + - cron: "0 0 * * *" # https://crontab.guru/#0_0_*_*_* + +jobs: + build-debug: + name: "Build Debug" + defaults: + run: + shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + - name: Restore + run: dotnet restore + - name: Build + run: | + dotnet build \ + ${{ github.workspace }}/CosmosDbDataMigrationTool.sln \ + --no-restore \ + --configuration Debug \ + --framework net6.0 \ + --maxcpucount:1 + - name: Test + run: | + dotnet test \ + --configuration Debug \ + --no-build \ + --verbosity normal + - name: Upload a Build Artifact + uses: actions/upload-artifact@v3 + with: + name: debug-build + path: ${{ github.workspace }}/Core/Microsoft.DataTransfer.Core/bin/Debug/net6.0 diff --git a/.github/workflows/dotnet-build-release.yml b/.github/workflows/dotnet-build-release.yml new file mode 100644 index 0000000..d2deaa8 --- /dev/null +++ b/.github/workflows/dotnet-build-release.yml @@ -0,0 +1,50 @@ +name: Build .NET (Release) + +on: + workflow_dispatch: # Enable manual trigger to run workflow. + push: + branches: ["main"] + pull_request: + branches: ["main"] + schedule: + - cron: "0 0 * * *" # https://crontab.guru/#0_0_*_*_* + +jobs: + build-release: + name: "Build Release" + defaults: + run: + shell: bash + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + - name: Restore + run: dotnet restore + - name: Build + run: | + dotnet build \ + ${{ github.workspace }}/CosmosDbDataMigrationTool.sln \ + --no-restore \ + --configuration Release \ + --framework net6.0 \ + --maxcpucount:1 + - name: Test + run: | + dotnet test \ + --configuration Release \ + --no-build \ + --verbosity normal + - name: Upload a Build Artifact + uses: actions/upload-artifact@v3 + with: + name: release-build + path: ${{ github.workspace }}/Core/Microsoft.DataTransfer.Core/bin/Release/net6.0 diff --git a/.github/workflows/dotnet-build-test.yml b/.github/workflows/dotnet-build-test.yml deleted file mode 100644 index 85d9927..0000000 --- a/.github/workflows/dotnet-build-test.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: .NET - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - schedule: - - cron: '0 0 * * *' - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Setup .NET - uses: actions/setup-dotnet@v2 - with: - dotnet-version: 6.0.x - - name: Restore dependencies - run: dotnet restore - - name: Build - run: dotnet build --no-restore - - name: Test - run: dotnet test --no-build --verbosity normal - - - name: Debug Build Artifact - uses: actions/upload-artifact@v3 - with: - name: debug-build - path: /home/runner/work/azure-documentdb-datamigrationtool/azure-documentdb-datamigrationtool/Core/Microsoft.DataTransfer.Core/bin/Debug/net6.0 #path/to/artifact/ # or path/to/artifact diff --git a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj index 8056ee9..294324e 100644 --- a/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj +++ b/Core/Microsoft.DataTransfer.Core/Microsoft.DataTransfer.Core.csproj @@ -18,7 +18,8 @@ - + diff --git a/Core/Microsoft.DataTransfer.Core/appsettings.json b/Core/Microsoft.DataTransfer.Core/appsettings.json index b9b5535..3427344 100644 --- a/Core/Microsoft.DataTransfer.Core/appsettings.json +++ b/Core/Microsoft.DataTransfer.Core/appsettings.json @@ -1,22 +1,16 @@ { - "JsonSourceSettings": { - }, - "JsonSinkSettings": { - }, + "JsonSourceSettings": {}, + "JsonSinkSettings": {}, "CosmosSourceSettings": { "ConnectionString": "" }, "CosmosSinkSettings": { "ConnectionString": "" }, - "MongoSourceSettings": { - }, - "MongoSinkSettings": { - }, - "AzureTableAPISourceSettings": { - }, - "AzureTableAPISinkSettings": { - }, + "MongoSourceSettings": {}, + "MongoSinkSettings": {}, + "AzureTableAPISourceSettings": {}, + "AzureTableAPISinkSettings": {}, "SqlServerSourceSettings": { "ConnectionString": "" }, diff --git a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests.csproj b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests.csproj index 586f064..5682d63 100644 --- a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests.csproj +++ b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests/Microsoft.DataTransfer.AzureTableAPIExtension.UnitTests.csproj @@ -17,8 +17,10 @@ - - + + diff --git a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Microsoft.DataTransfer.AzureTableAPIExtension.csproj b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Microsoft.DataTransfer.AzureTableAPIExtension.csproj index 9693d9f..0a303ea 100644 --- a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Microsoft.DataTransfer.AzureTableAPIExtension.csproj +++ b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Microsoft.DataTransfer.AzureTableAPIExtension.csproj @@ -1,24 +1,30 @@  - - net6.0 - enable - enable - Exe - + + net6.0 + enable + enable + Exe + - - - - - + + + + + - - - + + + - - - + + + + + + diff --git a/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml new file mode 100644 index 0000000..e7eaeb9 --- /dev/null +++ b/Extensions/AzureTableAPI/Microsoft.DataTransfer.AzureTableAPIExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + ..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions + FileSystem + <_TargetId>Folder + net6.0 + false + + diff --git a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension.UnitTests/Microsoft.DataTransfer.CosmosExtension.UnitTests.csproj b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension.UnitTests/Microsoft.DataTransfer.CosmosExtension.UnitTests.csproj index 1d91eb3..7f1e86a 100644 --- a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension.UnitTests/Microsoft.DataTransfer.CosmosExtension.UnitTests.csproj +++ b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension.UnitTests/Microsoft.DataTransfer.CosmosExtension.UnitTests.csproj @@ -18,7 +18,8 @@ - + diff --git a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Microsoft.DataTransfer.CosmosExtension.csproj b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Microsoft.DataTransfer.CosmosExtension.csproj index bf86e5a..ba03307 100644 --- a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Microsoft.DataTransfer.CosmosExtension.csproj +++ b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Microsoft.DataTransfer.CosmosExtension.csproj @@ -1,28 +1,35 @@  - - net6.0 - enable - enable - Exe - + + net6.0 + enable + enable + Exe + - - - - - - - - - - + + + + + + + + + + - - - + + + + + + + + + + - - - diff --git a/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml new file mode 100644 index 0000000..e7eaeb9 --- /dev/null +++ b/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + ..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions + FileSystem + <_TargetId>Folder + net6.0 + false + + diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Microsoft.DataTransfer.JsonExtension.UnitTests.csproj b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Microsoft.DataTransfer.JsonExtension.UnitTests.csproj index b81a402..b2de0f9 100644 --- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Microsoft.DataTransfer.JsonExtension.UnitTests.csproj +++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension.UnitTests/Microsoft.DataTransfer.JsonExtension.UnitTests.csproj @@ -18,7 +18,8 @@ - + diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Microsoft.DataTransfer.JsonExtension.csproj b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Microsoft.DataTransfer.JsonExtension.csproj index dbf3be7..2d3795f 100644 --- a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Microsoft.DataTransfer.JsonExtension.csproj +++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Microsoft.DataTransfer.JsonExtension.csproj @@ -1,23 +1,29 @@  - - net6.0 - enable - enable - Exe - + + net6.0 + enable + enable + Exe + - - - - + + + + - - - + + + - - - + + + + + + diff --git a/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml new file mode 100644 index 0000000..e7eaeb9 --- /dev/null +++ b/Extensions/Json/Microsoft.DataTransfer.JsonExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + ..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions + FileSystem + <_TargetId>Folder + net6.0 + false + + diff --git a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Microsoft.DataTransfer.MongoExtension.csproj b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Microsoft.DataTransfer.MongoExtension.csproj index 28383e0..e3bc2b3 100644 --- a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Microsoft.DataTransfer.MongoExtension.csproj +++ b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Microsoft.DataTransfer.MongoExtension.csproj @@ -1,24 +1,30 @@ - - net6.0 - enable - enable - Exe - + + net6.0 + enable + enable + Exe + - - - - - + + + + + - - - + + + - - - + + + + + + diff --git a/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml new file mode 100644 index 0000000..e7eaeb9 --- /dev/null +++ b/Extensions/Mongo/Microsoft.DataTransfer.MongoExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + ..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions + FileSystem + <_TargetId>Folder + net6.0 + false + + diff --git a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj index 79f76fc..91eb167 100644 --- a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj +++ b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension.UnitTests/Microsoft.DataTransfer.SqlServerExtension.UnitTests.csproj @@ -18,7 +18,8 @@ - + diff --git a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Microsoft.DataTransfer.SqlServerExtension.csproj b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Microsoft.DataTransfer.SqlServerExtension.csproj index 2493d91..82a7e62 100644 --- a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Microsoft.DataTransfer.SqlServerExtension.csproj +++ b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Microsoft.DataTransfer.SqlServerExtension.csproj @@ -1,27 +1,34 @@  - - net6.0 - enable - enable - Exe - + + net6.0 + enable + enable + Exe + - - - - - - - - - + + + + + + + + + - - - + + + + + + + + + + - - - diff --git a/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml new file mode 100644 index 0000000..e7eaeb9 --- /dev/null +++ b/Extensions/SqlServer/Microsoft.DataTransfer.SqlServerExtension/Properties/PublishProfiles/LocalReleaseFolder.pubxml @@ -0,0 +1,15 @@ + + + + + Release + Any CPU + ..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions + FileSystem + <_TargetId>Folder + net6.0 + false + + diff --git a/README.md b/README.md index 482ccc4..786d59c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Azure Cosmos DB data migration tool +# Azure Cosmos DB Data Migration Tool > **PLEASE NOTE** -> -> The Azure Cosmos DB data migration tool is undergoing a full refactor to restructure the project to accomplish the following: +> +> The Azure Cosmos DB Data Migration Tool is undergoing a full refactor to restructure the project to accomplish the following: > > - Provide a base cross-platform CLI that is lightweight and serves as the base (core) executable for hosting extensions > - The extensible framework enables the community to add source and destination targets without modifying the core executable @@ -14,13 +14,13 @@ To access the archived version of the tool, navigate to the [**Archive branch**] --- -## Cosmos DB data migration tool architecture +## Cosmos DB Data Migration Tool Architecture The Cosmos DB Data Migration Tool is a lightweight executable that leverages the [Managed Extensibility Framework (MEF)](https://docs.microsoft.com/en-us/dotnet/framework/mef/). MEF enables decoupled implementation of the core project and its extensions. The core application is a command-line executable responsible for composing the required extensions at runtime by automatically loading them from the Extensions folder of the application. An Extension is a class library that includes the implementation of a System as a Source and (optionally) Sink for data transfer. The core application project does not contain direct references to any extension implementation. Instead, these projects share a common interface. ![An extensions folder holds multiple extensions implementations.The application loads extensions from the extensions folder and executes functionality based on an interface implementation.](media/application_architecture.png "Application architecture") -## Cosmos DB project structure +## Cosmos DB Data Migration Tool Project Structure The Cosmos DB Data Migration Tool core project is a C# command-line executable. The core application serves as the composition container for the required Source and Sink extensions. Therefore, the application user needs to put only the desired Extension class library assembly into the Extensions folder before running the application. In addition, the core project has a unit test project to exercise the application's behavior, whereas extension projects contain concrete integration tests that rely on external systems. diff --git a/buildDebug.sh b/buildDebug.sh new file mode 100755 index 0000000..1f0a715 --- /dev/null +++ b/buildDebug.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail +shopt -s inherit_errexit + +SCRIPT_DIRECTORY=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +DOTNET_CONFIGURATION="Debug" +DOTNET_FRAMEWORK="net6.0" + +rm \ + --recursive \ + --force \ + "${SCRIPT_DIRECTORY}/Core/Microsoft.DataTransfer.Core/bin/${DOTNET_CONFIGURATION}" + +dotnet restore +dotnet build \ + "${SCRIPT_DIRECTORY}/CosmosDbDataMigrationTool.sln" \ + --no-restore \ + --configuration "${DOTNET_CONFIGURATION}" \ + --framework "${DOTNET_FRAMEWORK}" \ + --maxcpucount:1 + +cd "${SCRIPT_DIRECTORY}/Core/Microsoft.DataTransfer.Core/bin/${DOTNET_CONFIGURATION}/${DOTNET_FRAMEWORK}" + +# Launch Settings +# https://github.com/Azure/azure-documentdb-datamigrationtool/blob/48b5f42efa6df8b0003d158dc94be02bad110f89/Core/Microsoft.DataTransfer.Core/Properties/launchSettings.json + +# Echo to emulate press Enter, else program does not quit automatically. +# https://github.com/Azure/azure-documentdb-datamigrationtool/blob/48b5f42efa6df8b0003d158dc94be02bad110f89/Core/Microsoft.DataTransfer.Core/Program.cs#L50 + +# TODO: Enable following lines to run the tool. +# echo | dotnet \ +# Microsoft.DataTransfer.Core.dll \ +# --source cosmos \ +# --sink json \ +# --SourceSettingsPath="CosmosSourceSettings.json" \ +# --SinkSettingsPath="JsonSinkSettings.json" + +cd - diff --git a/buildRelease.sh b/buildRelease.sh new file mode 100755 index 0000000..121d566 --- /dev/null +++ b/buildRelease.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail +shopt -s inherit_errexit + +SCRIPT_DIRECTORY=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +DOTNET_CONFIGURATION="Release" +DOTNET_FRAMEWORK="net6.0" + +rm \ + --recursive \ + --force \ + "${SCRIPT_DIRECTORY}/Core/Microsoft.DataTransfer.Core/bin/${DOTNET_CONFIGURATION}" + +dotnet restore +dotnet build \ + "${SCRIPT_DIRECTORY}/CosmosDbDataMigrationTool.sln" \ + --no-restore \ + --configuration "${DOTNET_CONFIGURATION}" \ + --framework "${DOTNET_FRAMEWORK}" \ + --maxcpucount:1 + +cd "${SCRIPT_DIRECTORY}/Core/Microsoft.DataTransfer.Core/bin/${DOTNET_CONFIGURATION}/${DOTNET_FRAMEWORK}" + +# Launch Settings +# https://github.com/Azure/azure-documentdb-datamigrationtool/blob/48b5f42efa6df8b0003d158dc94be02bad110f89/Core/Microsoft.DataTransfer.Core/Properties/launchSettings.json + +# Echo to emulate press Enter, else program does not quit automatically. +# https://github.com/Azure/azure-documentdb-datamigrationtool/blob/48b5f42efa6df8b0003d158dc94be02bad110f89/Core/Microsoft.DataTransfer.Core/Program.cs#L50 + +# TODO: Enable following lines to run the tool. +# echo | dotnet \ +# Microsoft.DataTransfer.Core.dll \ +# --source cosmos \ +# --sink json \ +# --SourceSettingsPath="CosmosSourceSettings.json" \ +# --SinkSettingsPath="JsonSinkSettings.json" + +cd - diff --git a/docs/developer-guidance.md b/docs/developer-guidance.md index 9f8776f..27692a9 100644 --- a/docs/developer-guidance.md +++ b/docs/developer-guidance.md @@ -1,41 +1,46 @@ # Developer Guidance - [Developer Guidance](#developer-guidance) - - [Software prerequisites](#software-prerequisites) - - [Clone the source code repository](#clone-the-source-code-repository) - - [Build the solution](#build-the-solution) - - [Tutorial: JSON to Cosmos DB migration](#tutorial-json-to-cosmos-db-migration) - - [Task 1: Provision a sample database and container using the Azure Cosmos DB Emulator as the destination(sink)](#task-1-provision-a-sample-database-and-container-using-the-azure-cosmos-db-emulator-as-the-destinationsink) - - [Task 2: Prepare JSON source documents](#task-2-prepare-json-source-documents) - - [Task 3: Setup the data migration configuration](#task-3-setup-the-data-migration-configuration) + - [Prerequisites](#prerequisites) + - [Clone The Source Code Repository](#clone-the-source-code-repository) + - [Build The Solution](#build-the-solution) + - [Build Using Command Line / Terminal](#build-using-command-line--terminal) + - [Tutorial: JSON To Cosmos DB Migration](#tutorial-json-to-cosmos-db-migration) + - [Task 1: Provision A Sample Database And Container Using The Azure Cosmos Db Emulator As The Destination](#task-1-provision-a-sample-database-and-container-using-the-azure-cosmos-db-emulator-as-the-destination) + - [Task 2: Prepare JSON Source Documents](#task-2-prepare-json-source-documents) + - [Task 3: Setup The Data Migration Configuration](#task-3-setup-the-data-migration-configuration) - [Creating Extensions](#creating-extensions) -## Software prerequisites +## Prerequisites 1. [Visual Studio 2022](https://visualstudio.microsoft.com/vs/) -2. [.NET 6.0 SDK](https://github.com/Azure/azure-documentdb-datamigrationtool) -3. [Git client](https://github.com/git-guides/install-git) +2. [.NET 6.0 SDK](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) +3. [Git](https://github.com/git-guides/install-git) 4. [Azure Cosmos DB Emulator](https://learn.microsoft.com/azure/cosmos-db/local-emulator) or Azure Cosmos DB resource. -## Clone the source code repository +## Clone The Source Code Repository 1. From a command prompt, execute the following command in an empty working folder that will house the source code. ```shell -git clone --branch feature/begin-core https://github.com/Azure/azure-documentdb-datamigrationtool.git +git clone https://github.com/Azure/azure-documentdb-datamigrationtool.git ``` -## Build the solution +## Build The Solution 1. Using Visual Studio 2022, open `CosmosDbDataMigrationTool.sln`. -2. Build the project using the keyboard shortcut Ctrl+Shift+B (Cmd+Shift+B on a Mac). This will build all current extension projects as well as the command-line **Core** application. The extension projects build assemblies get written to the **Extensions** folder of the **Core** application build. This way all extension options are available when the application is run. +2. Build the project using the keyboard shortcut `Ctrl`+`Shift`+`B` (`Cmd`+`Shift`+`B` on macOS). This will build all current extension projects as well as the command-line **Core** application. The extension projects build assemblies get written to the **Extensions** folder of the **Core** application build. All extension options will be available when the application is run. -## Tutorial: JSON to Cosmos DB migration +### Build Using Command Line / Terminal -This tutorial outlines how to use the Cosmos DB Data Migration tool to move JSON data to Azure Cosmos DB. This tutorial uses the Azure Cosmos DB Emulator. +- See [build-debug.sh](../build-debug.sh) and [build-release.sh](../build-release.sh) scripts. -### Task 1: Provision a sample database and container using the Azure Cosmos DB Emulator as the destination(sink) +## Tutorial: JSON To Cosmos DB Migration + +This tutorial outlines how to use the Cosmos DB Data Migration Tool to move JSON data to Azure Cosmos DB. This tutorial uses the Azure Cosmos DB Emulator. + +### Task 1: Provision A Sample Database And Container Using The Azure Cosmos Db Emulator As The Destination 1. Launch the Azure Cosmos DB emulator application and open in a browser. @@ -59,13 +64,13 @@ This tutorial outlines how to use the Cosmos DB Data Migration tool to move JSON ![The New Container blade displays with btcdata entered in the Container id field and /id entered in the Partition key field. The OK button is highlighted.](media/cosmosdbemulator_newcontainerform.png "New Container blade") - >**Note**: When using the Cosmos DB Data Migration tool, the container doesn't have to previously exist, it will be created automatically using the partition key specified in the sink configuration. + >**Note**: When using the Cosmos DB Data Migration Tool, the container doesn't have to previously exist, it will be created automatically using the partition key specified in the sink configuration. -### Task 2: Prepare JSON source documents +### Task 2: Prepare JSON Source Documents 1. Locate the **docs/resources/sample-data.zip** file. Extract the files to any desired folder. These files serve as the JSON data that is to be migrated to Cosmos DB. -### Task 3: Setup the data migration configuration +### Task 3: Setup The Data Migration Configuration 1. Each extension contains a README document that outlines configuration for the data migration. In this case, locate the configuration for [JSON (Source)](https://github.com/Azure/azure-documentdb-datamigrationtool/tree/feature/begin-core/Extensions/Json/Microsoft.DataTransfer.JsonExtension) and [Cosmos DB (Sink)](https://github.com/Azure/azure-documentdb-datamigrationtool/tree/feature/begin-core/Extensions/Cosmos/Microsoft.DataTransfer.CosmosExtension). @@ -89,39 +94,46 @@ This tutorial outlines how to use the Cosmos DB Data Migration tool to move JSON ![The Cosmos DB Emulator Quickstart screen displays with the Primary Connection String value highlighted.](media/cosmosemulator_primaryconnectionstring.png "Cosmos DB Emulator Primary Connection String") - >**NOTE**: Currently the Cosmos DB Data migration tool expects a single JSON file containing an array of objects. + >**NOTE**: Currently the Cosmos DB Data Migration Tool expects a single JSON file containing an array of objects. -3. Ensure the **Microsoft.DataTransfer.Core** project is set as the startup project then press F5 to run the application. +3. Ensure the **Microsoft.DataTransfer.Core** project is set as the startup project then press `F5` to run the application. -4. The application will load the provided extensions, then prompt for a **Source**. Select **JSON**. When prompted to load settings from a file enter `n` and press Enter. The application provides the ability to configure extensions in a separate file versus having them located in the **appsettings.json** file. When a separate file is not specified, the settings are read from **appsettings.json**. When prompted to provide the Configuration section name, press Enter to accept the default: **JSONSourceSettings**. +4. The application will load the provided extensions, then prompt for a **Source**. Select **JSON**. When prompted to load settings from a file enter `n` and press `Enter`. The application provides the ability to configure extensions in a separate file versus having them located in the **appsettings.json** file. When a separate file is not specified, the settings are read from **appsettings.json**. When prompted to provide the Configuration section name, press `Enter` to accept the default: **JSONSourceSettings**. ![The Data Migration application displays with a menu prompt for Source.](media/app_source_prompt.png "Select Source") -5. The application will then prompt for a **Sink**, choose **Cosmos**. When prompted to load settings from a file enter `n` and press Enter. The application provides the ability to configure extensions in a separate file versus having them located in the **appsettings.json** file. When a separate file is not specified, the settings are read from **appsettings.json**. When prompted to provide the Configuration section name, press Enter to accept the default: **CosmosSinkSettings**. +5. The application will then prompt for a **Sink**, choose **Cosmos**. When prompted to load settings from a file enter `n` and press `Enter`. The application provides the ability to configure extensions in a separate file versus having them located in the **appsettings.json** file. When a separate file is not specified, the settings are read from **appsettings.json**. When prompted to provide the Configuration section name, press `Enter` to accept the default: **CosmosSinkSettings**. ![The Data migration application displays with a menu prompt for Sink.](media/app_sink_prompt.png "Select Sink") -6. The application then performs the data migration. After a few moments the process will indicate **Done.**. +6. The application then performs the data migration. After a few moments the process will indicate **Done**. ![The Data migration application displays with the full output of the migration including the Done message.](media/app_final.png "Data Migration completes") ## Creating Extensions -1. Add a new folder in the Extensions folder with the name of your extension. +1. Add a new folder in the `Extensions` folder with the name of your extension. 2. Create the extension project and an accompanying test project. - - The naming convention for extension projects is `Microsoft.DataTransfer.Extension`. - - Extension projects should use .NET 6 framework and **Console Application** type. A Program.cs file must be included in order to build the console project. A Console Application Project is required to have the build include NuGet referenced packages. - + - The naming convention for extension projects is `Microsoft.DataTransfer.`. + - Extension projects should use .NET 6 framework and **Console Application** type. A `Program.cs` file must be included in order to build the console project. A Console Application Project is required to have the build include NuGet referenced packages. + 3. Add the new projects to the `CosmosDbDataMigrationTool` solution. -4. In order to facilitate local debugging the extension build output along with any dependencies needs to be copied into the `Core\Microsoft.DataTransfer.Core\bin\Debug\net6.0\Extensions` folder. To set up the project to automatically copy add the following changes. - - Add a Publish Profile to Folder named `LocalDebugFolder` with a Target Location of `..\..\..\Core\Microsoft.DataTransfer.Core\bin\Debug\net6.0\Extensions` - - To publish every time the project builds, edit the .csproj file to add a new post-build step: +4. In order to facilitate local debugging the extension build output along with any dependencies needs to be copied into the `Core\Microsoft.DataTransfer.Core\bin\{Debug,Release}\net6.0\Extensions` folder. To set up the project to automatically copy the extensions, add the following changes. + + - Add a Publish Profile to Folder named `LocalDebugFolder` with a Target Location of `..\..\..\Core\Microsoft.DataTransfer.Core\bin\Debug\net6.0\Extensions`. + - Add a Publish Profile to Folder named `LocalReleaseFolder` with a Target Location of `..\..\..\Core\Microsoft.DataTransfer.Core\bin\Release\net6.0\Extensions`. + - To publish every time the project builds, edit the `.csproj` file to add a new post-build step: ```xml - + + + + ```