Redesigned lineage event recording

Added support for resetting ETL metadata via the API
Added performance test

Author: kristan66

.github/workflows/check-pull-request.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Verify format
         run: dotnet format ./working-dir/KeeperData.Bridge.sln --verify-no-changes --verbosity diagnostic
       - name: Test (unit and component)
-        run: dotnet test ./working-dir/KeeperData.Bridge.sln --filter Dependence!=docker --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" --results-directory ./coverage
+        run: dotnet test ./working-dir/KeeperData.Bridge.sln --filter "Dependence!=docker&testtype!=performance" --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" --results-directory ./coverage
       - name: Install ReportGenerator
         run: dotnet tool install --global dotnet-reportgenerator-globaltool
       - name: Generate Coverage Summary
@@ -87,7 +87,7 @@ jobs:
       - name: Wait for keeperdata_bridge to be ready
         run: ./working-dir/wait-for-docker-logs.sh keeperdata_bridge "Now listening on" 60
       - name: Run integration tests
-        run: dotnet test ./working-dir/KeeperData.Bridge.sln --filter Dependence=docker
+        run: dotnet test ./working-dir/KeeperData.Bridge.sln --filter "Dependence=docker&testtype!=performance"
 
     ## SonarCloud
     ## Uncomment to unable SonarCloud scan

KeeperData.Bridge.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
+# Visual Studio Version 18
+VisualStudioVersion = 18.0.11201.2 d18.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge", "src\KeeperData.Bridge\KeeperData.Bridge.csproj", "{7D935959-D3BE-4EDC-BAEC-541C72741633}"
 EndProject
@@ -37,6 +37,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Crypto.Tool", "s
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge.Worker", "src\KeeperData.Bridge.Worker\KeeperData.Bridge.Worker.csproj", "{C20E1970-BB6F-46BC-886E-7E856DAA763E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Bridge.PerformanceTests", "src\KeeperData.Bridge.PerformanceTests\KeeperData.Bridge.PerformanceTests.csproj", "{B4FCD69D-D7BB-4028-903D-31A009191703}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -59,7 +61,6 @@ Global
 		{7D935959-D3BE-4EDC-BAEC-541C72741633}.Release|x64.Build.0 = Release|Any CPU
 		{7D935959-D3BE-4EDC-BAEC-541C72741633}.Release|x86.ActiveCfg = Release|Any CPU
 		{7D935959-D3BE-4EDC-BAEC-541C72741633}.Release|x86.Build.0 = Release|Any CPU
-
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -72,7 +73,6 @@ Global
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Release|x64.Build.0 = Release|Any CPU
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Release|x86.ActiveCfg = Release|Any CPU
 		{4E96024E-0BCB-4C41-8EB2-876499B07311}.Release|x86.Build.0 = Release|Any CPU
-
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -85,7 +85,6 @@ Global
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Release|x64.Build.0 = Release|Any CPU
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Release|x86.ActiveCfg = Release|Any CPU
 		{D5BA4CDF-AABC-4FE6-B41A-D32381CDFB32}.Release|x86.Build.0 = Release|Any CPU
-
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -98,7 +97,6 @@ Global
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Release|x64.Build.0 = Release|Any CPU
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Release|x86.ActiveCfg = Release|Any CPU
 		{45E67021-5132-416B-92F0-AA2C1C7E65AD}.Release|x86.Build.0 = Release|Any CPU
-
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -111,7 +109,6 @@ Global
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Release|x64.Build.0 = Release|Any CPU
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Release|x86.ActiveCfg = Release|Any CPU
 		{28DA1EFD-BB2E-4AF5-B9BD-AD3B6A139DBE}.Release|x86.Build.0 = Release|Any CPU
-
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -124,7 +121,6 @@ Global
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Release|x64.Build.0 = Release|Any CPU
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Release|x86.ActiveCfg = Release|Any CPU
 		{324F5874-B7B9-4B87-9FAB-BDB970971919}.Release|x86.Build.0 = Release|Any CPU
-
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -137,7 +133,6 @@ Global
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Release|x64.Build.0 = Release|Any CPU
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Release|x86.ActiveCfg = Release|Any CPU
 		{E2B6D37D-2D17-4B43-83DC-36F1C4961526}.Release|x86.Build.0 = Release|Any CPU
-
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -150,7 +145,6 @@ Global
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Release|x64.Build.0 = Release|Any CPU
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Release|x86.ActiveCfg = Release|Any CPU
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A}.Release|x86.Build.0 = Release|Any CPU
-
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -163,20 +157,6 @@ Global
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Release|x64.Build.0 = Release|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Release|x86.ActiveCfg = Release|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Release|x86.Build.0 = Release|Any CPU
-
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x64.Build.0 = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x86.Build.0 = Debug|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x64.ActiveCfg = Release|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x64.Build.0 = Release|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.ActiveCfg = Release|Any CPU
-		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.Build.0 = Release|Any CPU
-
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -189,7 +169,6 @@ Global
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Release|x64.Build.0 = Release|Any CPU
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Release|x86.ActiveCfg = Release|Any CPU
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46}.Release|x86.Build.0 = Release|Any CPU
-
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -202,6 +181,30 @@ Global
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Release|x64.Build.0 = Release|Any CPU
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Release|x86.ActiveCfg = Release|Any CPU
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0}.Release|x86.Build.0 = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x64.Build.0 = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Debug|x86.Build.0 = Debug|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x64.ActiveCfg = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x64.Build.0 = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.ActiveCfg = Release|Any CPU
+		{C20E1970-BB6F-46BC-886E-7E856DAA763E}.Release|x86.Build.0 = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|x64.Build.0 = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Debug|x86.Build.0 = Debug|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|x64.ActiveCfg = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|x64.Build.0 = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|x86.ActiveCfg = Release|Any CPU
+		{B4FCD69D-D7BB-4028-903D-31A009191703}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -212,6 +215,7 @@ Global
 		{16AA106D-B4F1-46C9-92F3-548AE89FDA3A} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{389DD08B-1B37-4BF2-9F2B-3F4805B25A46} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{92F51802-9BF8-42A8-98C9-C25B27163DF0} = {25784FFF-FBA8-4398-8497-7BE1FF8F92A5}
+		{B4FCD69D-D7BB-4028-903D-31A009191703} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {235917AE-9AF5-49A5-AB06-472A5E5DB30D}

src/KeeperData.Bridge.PerformanceTests/KeeperData.Bridge.PerformanceTests.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.Core" Version="4.0.0.26" />
+    <PackageReference Include="AWSSDK.S3" Version="4.0.6.12" />
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Options" Version="9.0.9" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.28.0" />
+    <PackageReference Include="Testcontainers.LocalStack" Version="3.10.0" />
+    <PackageReference Include="Testcontainers.MongoDb" Version="3.10.0" />
+    <PackageReference Include="xunit" Version="2.5.3" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\KeeperData.Core\KeeperData.Core.csproj" />
+    <ProjectReference Include="..\KeeperData.Infrastructure\KeeperData.Infrastructure.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+</Project>