Skip to content

Commit af5270e

Browse files
MrHinshMrHinsh
authored
TfsExportUsersForMappingProcessor: export all users in source and target server to JSON file (#2527)
Our scenario is: - Source server is on-premise TFS 2018 connected to on-premise Active Directory. - Target server is Azure DevOps connected to Azure Entra ID (formerly Azure Active Directory). Even when the users are synchronized between the Active Directiories, the users' display names are often different and also quite a bunch of email addresses are different. So a lot of people are not matched at all and in exported mapping file, the target name is `null`. So the mapping file needs to be manually edited to add missing users. Usually (almost always), the user is in target server, but has different email and display name. To help find the corresponding user, this PR allows to export all users in source and target server into separate JSON. We can than find the user we need and use his correct display name in mapping file. Two properties are added to `TfsExportUsersForMappingProcessorOptions`: - `ExportAllUsers` – turns this feature on. - `UserExportFile` – path to file, where users will be exported. ## Example of export ``` json { "SourceUsers": [ { "Sid": "24b644ca-b413-4bd5-bdea-466534a69c72:Build:1cf84e9e-e068-4a9a-b12e-bf058294c1f5", "DisplayName": "Lorem Ipsum", "Domain": "Build", "AccountName": "lorem", "MailAddress": "[email protected]" }, ... "TargetUsers": [ { "Sid": "751af7c1-c13a-4f1b-888e-da6f6db86a01:Build:0132d9af-7fab-4f97-8170-b28466e07427", "DisplayName": "Dolor Sit", "Domain": "Build", "AccountName": "dolor", "MailAddress": "[email protected]" }, ... } ```
2 parents 9135fee + 4c98820 commit af5270e

File tree

5 files changed

+83
-32
lines changed

5 files changed

+83
-32
lines changed

‎src/MigrationTools.Clients.TfsObjectModel/Processors/TfsExportUsersForMappingProcessor.cs‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Diagnostics;
4+
using System.IO;
45
using System.Linq;
56
using Microsoft.Extensions.Logging;
67
using Microsoft.Extensions.Options;
@@ -42,30 +43,26 @@ protected override void InternalExecute()
4243
{
4344
Stopwatch stopwatch = Stopwatch.StartNew();
4445

45-
if (string.IsNullOrEmpty(CommonTools.UserMapping.Options.UserMappingFile))
46-
{
47-
Log.LogError("UserMappingFile is not set");
48-
throw new ArgumentNullException("UserMappingFile must be set on the TfsUserMappingToolOptions in CommonEnrichersConfig.");
49-
}
46+
CheckOptions();
5047

51-
List<IdentityMapData> usersToMap = new List<IdentityMapData>();
48+
IdentityMapResult data;
5249
if (Options.OnlyListUsersInWorkItems)
5350
{
5451
Log.LogInformation("OnlyListUsersInWorkItems is true, only users in work items will be listed");
5552
List<WorkItemData> sourceWorkItems = Source.WorkItems.GetWorkItems(Options.WIQLQuery);
5653
Log.LogInformation("Processed {0} work items from Source", sourceWorkItems.Count);
5754

58-
usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
59-
Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count);
55+
data = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
56+
Log.LogInformation("Found {usersToMap} total mapped", data.IdentityMap.Count);
6057
}
6158
else
6259
{
6360
Log.LogInformation("OnlyListUsersInWorkItems is false, all users will be listed");
64-
usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTarget(this);
65-
Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count);
61+
data = CommonTools.UserMapping.GetUsersInSourceMappedToTarget(this);
62+
Log.LogInformation("Found {usersToMap} total mapped", data.IdentityMap.Count);
6663
}
6764

68-
usersToMap = usersToMap.Where(x => x.Source.DisplayName != x.Target?.DisplayName).ToList();
65+
List<IdentityMapData> usersToMap = data.IdentityMap.Where(x => x.Source.DisplayName != x.Target?.DisplayName).ToList();
6966
Log.LogInformation("Filtered to {usersToMap} total viable mappings", usersToMap.Count);
7067
Dictionary<string, string> usermappings = [];
7168
foreach (IdentityMapData userMapping in usersToMap)
@@ -74,11 +71,40 @@ protected override void InternalExecute()
7471
// it would throw with duplicate key. This way we just overwrite the value – last item in source wins.
7572
usermappings[userMapping.Source.DisplayName] = userMapping.Target?.DisplayName;
7673
}
77-
System.IO.File.WriteAllText(CommonTools.UserMapping.Options.UserMappingFile, JsonConvert.SerializeObject(usermappings, Formatting.Indented));
78-
Log.LogInformation("Writen to: {LocalExportJsonFile}", CommonTools.UserMapping.Options.UserMappingFile);
74+
File.WriteAllText(CommonTools.UserMapping.Options.UserMappingFile, JsonConvert.SerializeObject(usermappings, Formatting.Indented));
75+
Log.LogInformation("User mappings writen to: {LocalExportJsonFile}", CommonTools.UserMapping.Options.UserMappingFile);
76+
if (Options.ExportAllUsers)
77+
{
78+
ExportAllUsers(data);
79+
}
7980

8081
stopwatch.Stop();
8182
Log.LogInformation("DONE in {Elapsed} seconds", stopwatch.Elapsed);
8283
}
84+
85+
private void ExportAllUsers(IdentityMapResult data)
86+
{
87+
var allUsers = new
88+
{
89+
data.SourceUsers,
90+
data.TargetUsers
91+
};
92+
File.WriteAllText(Options.UserExportFile, JsonConvert.SerializeObject(allUsers, Formatting.Indented));
93+
Log.LogInformation("All user writen to: {exportFile}", Options.UserExportFile);
94+
}
95+
96+
private void CheckOptions()
97+
{
98+
if (string.IsNullOrEmpty(CommonTools.UserMapping.Options.UserMappingFile))
99+
{
100+
Log.LogError("UserMappingFile is not set");
101+
throw new ArgumentNullException("UserMappingFile must be set on the TfsUserMappingToolOptions in CommonTools.");
102+
}
103+
if (Options.ExportAllUsers && string.IsNullOrEmpty(Options.UserExportFile))
104+
{
105+
Log.LogError($"Flag ExportAllUsers is set but export file UserExportFile is not set.");
106+
throw new ArgumentNullException("UserExportFile must be set on the TfsExportUsersForMappingProcessorOptions in Processors.");
107+
}
108+
}
83109
}
84110
}

‎src/MigrationTools.Clients.TfsObjectModel/Processors/TfsExportUsersForMappingProcessorOptions.cs‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
using System.Collections.Generic;
2-
using MigrationTools._EngineV1.Configuration;
3-
using MigrationTools.Enrichers;
4-
using MigrationTools.Processors.Infrastructure;
1+
using MigrationTools.Processors.Infrastructure;
52

63
namespace MigrationTools.Processors
74
{
@@ -16,5 +13,17 @@ public class TfsExportUsersForMappingProcessorOptions : ProcessorOptions
1613
/// <default>true</default>
1714
public bool OnlyListUsersInWorkItems { get; set; } = true;
1815

16+
/// <summary>
17+
/// Set to <see langword="true"/>, if you want to export all users in source and target server.
18+
/// The lists of user can be useful, if you need tu manually edit mapping file.
19+
/// Users will be exported to file set in <see cref="UserExportFile"/>.
20+
/// </summary>
21+
public bool ExportAllUsers { get; set; }
22+
23+
/// <summary>
24+
/// Path to export file where all source and target servers' users will be exported.
25+
/// Users are exported only if <see cref="ExportAllUsers"/> is set to <see langword="true"/>.
26+
/// </summary>
27+
public string UserExportFile { get; set; }
1928
}
2029
}

‎src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -227,15 +227,15 @@ protected override void InternalExecute()
227227

228228
private void ValidateAllUsersExistOrAreMapped(List<WorkItemData> sourceWorkItems)
229229
{
230-
231230
contextLog.Information("Validating::Check that all users in the source exist in the target or are mapped!");
232-
List<IdentityMapData> usersToMap = new List<IdentityMapData>();
233-
usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
234-
if (usersToMap != null && usersToMap?.Count > 0)
231+
IdentityMapResult usersToMap = CommonTools.UserMapping.GetUsersInSourceMappedToTargetForWorkItems(this, sourceWorkItems);
232+
if (usersToMap.IdentityMap != null && usersToMap.IdentityMap.Count > 0)
235233
{
236-
Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist in the target. This will not cause any errors, but may result in disconnected users that could have been mapped. Use the ExportUsersForMapping processor to create a list of mappable users. Then Import using ", usersToMap.Count);
234+
Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist "
235+
+ "in the target. This will not cause any errors, but may result in disconnected users that could have "
236+
+ "been mapped. Use the ExportUsersForMapping processor to create a list of mappable users.",
237+
usersToMap.IdentityMap.Count);
237238
}
238-
239239
}
240240

241241
//private void ValidateAllNodesExistOrAreMapped(List<WorkItemData> sourceWorkItems)

‎src/MigrationTools.Clients.TfsObjectModel/Tools/TfsUserMappingTool.cs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
using System;
1+
using System;
22
using System.Collections.Generic;
33
using System.Linq;
44
using Microsoft.Extensions.Logging;
@@ -121,11 +121,12 @@ private List<IdentityItemData> GetUsersListFromServer(IGroupSecurityService gss)
121121
Log.LogWarning("TfsUserMappingTool::GetUsersListFromServer::[user:{user}] Failed With {Exception}", sid, ex.Message);
122122
}
123123
}
124+
foundUsers.Sort((x, y) => x.AccountName.CompareTo(y.AccountName));
124125
Log.LogInformation("TfsUserMappingTool::GetUsersListFromServer {count} user identities are applicable for mapping", foundUsers.Count);
125126
return foundUsers;
126127
}
127128

128-
public List<IdentityMapData> GetUsersInSourceMappedToTarget(TfsProcessor processor)
129+
public IdentityMapResult GetUsersInSourceMappedToTarget(TfsProcessor processor)
129130
{
130131
Log.LogDebug("TfsUserMappingTool::GetUsersInSourceMappedToTarget");
131132
if (Options.Enabled)
@@ -162,25 +163,31 @@ public List<IdentityMapData> GetUsersInSourceMappedToTarget(TfsProcessor process
162163
targetUser ??= targetUsers.SingleOrDefault(x => x.DisplayName == sourceUser.DisplayName);
163164
identityMap.Add(new IdentityMapData { Source = sourceUser, Target = targetUser });
164165
}
165-
return identityMap;
166+
return new()
167+
{
168+
IdentityMap = identityMap,
169+
SourceUsers = sourceUsers,
170+
TargetUsers = targetUsers
171+
};
166172
}
167173
else
168174
{
169175
Log.LogWarning("TfsUserMappingTool is disabled in settings. You may have users in the source that are not mapped to the target. ");
170-
return [];
176+
return new();
171177
}
172178
}
173179

174-
public List<IdentityMapData> GetUsersInSourceMappedToTargetForWorkItems(TfsProcessor processor, List<WorkItemData> sourceWorkItems)
180+
public IdentityMapResult GetUsersInSourceMappedToTargetForWorkItems(TfsProcessor processor, List<WorkItemData> sourceWorkItems)
175181
{
176182
if (Options.Enabled)
177183
{
178184
Dictionary<string, string> result = new Dictionary<string, string>();
179185
HashSet<string> workItemUsers = GetUsersFromWorkItems(sourceWorkItems, Options.IdentityFieldsToCheck);
180186
Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [workItemUsers|{workItemUsers.Count}]");
181-
List<IdentityMapData> mappedUsers = GetUsersInSourceMappedToTarget(processor);
182-
Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.Count}]");
183-
return mappedUsers.Where(x => workItemUsers.Contains(x.Source.DisplayName)).ToList();
187+
IdentityMapResult mappedUsers = GetUsersInSourceMappedToTarget(processor);
188+
Log.LogDebug($"TfsUserMappingTool::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.IdentityMap.Count}]");
189+
mappedUsers.IdentityMap = mappedUsers.IdentityMap.Where(x => workItemUsers.Contains(x.Source.DisplayName)).ToList();
190+
return mappedUsers;
184191
}
185192
else
186193
{

‎src/MigrationTools/DataContracts/IdentityItemData.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
namespace MigrationTools.DataContracts
1+
using System.Collections.Generic;
2+
3+
namespace MigrationTools.DataContracts
24
{
35
public class IdentityItemData
46
{
@@ -14,4 +16,11 @@ public class IdentityMapData
1416
public IdentityItemData Source { get; set; }
1517
public IdentityItemData Target { get; set; }
1618
}
19+
20+
public class IdentityMapResult
21+
{
22+
public List<IdentityMapData> IdentityMap { get; set; } = [];
23+
public List<IdentityItemData> SourceUsers { get; set; } = [];
24+
public List<IdentityItemData> TargetUsers { get; set; } = [];
25+
}
1726
}

0 commit comments

Comments
 (0)