Skip to content

Commit f81f9a5

Browse files
sambettssambetts
authored
Optional filtering by Entra ID Groups (#54)
Merge with dev for next test release.
2 parents dd7dc12 + 83b708f commit f81f9a5

39 files changed

+590
-142
lines changed

‎src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using WebJob.Office365ActivityImporter.Engine;
2222
using WebJob.Office365ActivityImporter.Engine.ActivityAPI;
2323
using WebJob.Office365ActivityImporter.Engine.Graph.UsageReports;
24+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
2425
using static App.ControlPanel.Engine.Models.AutodetectedSqlAndFtpDetails;
2526

2627
namespace App.ControlPanel.Engine
@@ -308,7 +309,10 @@ async Task VerifyTeamsAndUserActivityImport(string clientId, string tenantId, st
308309

309310
var graphClient = new Microsoft.Graph.GraphServiceClient(auth.Creds);
310311

311-
var teamsUserUsageLoader = new TeamsUserUsageLoader(new WebJob.Office365ActivityImporter.Engine.Graph.ManualGraphCallClient(auth, telemetry), telemetry);
312+
var teamsUserUsageLoader = new TeamsUserUsageLoader(new WebJob.Office365ActivityImporter.Engine.Graph.ManualGraphCallClient(auth, telemetry),
313+
new NoUsersHaveGroupsUserGroupsCache(_logger),
314+
new Common.Entities.Config.UserGroupsFilterModel(string.Empty),
315+
telemetry);
312316

313317
// Usage reports
314318
if (Config.SolutionConfig.ImportTaskSettings.GraphUsageReports)

‎src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
using Microsoft.Extensions.Logging;
22
using System;
3-
using System.Configuration;
43

54
namespace DataUtils
65
{
@@ -35,34 +34,16 @@ public static void BombOut(bool error)
3534
}
3635
}
3736

38-
public static void PrintStartupAndLoggingConfig(ILogger logger)
37+
public static void PrintStartupAndLoggingConfig(string efConnectionString, string buildLabel, string userGroupsFilterString, ILogger logger)
3938
{
40-
if (logger is null)
41-
{
42-
throw new ArgumentNullException(nameof(logger));
43-
}
4439

45-
var buildLabel = ConfigurationManager.AppSettings["BuildLabel"];
4640
logger.LogInformation($"Office 365 Advanced Analytics engine START: '{buildLabel}'.");
47-
48-
string efConnectionString = ConfigurationManager.ConnectionStrings["SPOInsightsEntities"].ConnectionString;
4941
var sqlConnectionInfo = new System.Data.SqlClient.SqlConnectionStringBuilder(efConnectionString);
5042
logger.LogInformation($"Destination SQL Server='{sqlConnectionInfo.DataSource}', DB='{sqlConnectionInfo.InitialCatalog}'.");
51-
52-
bool loggingEnabled = ConfigurationManager.AppSettings["ImportLogging"] == "True";
53-
#if DEBUG
54-
loggingEnabled = true;
55-
#endif
56-
57-
if (loggingEnabled)
43+
if (!string.IsNullOrEmpty(userGroupsFilterString))
5844
{
59-
logger.LogInformation("Import logging is ENABLED.");
60-
}
61-
else
62-
{
63-
logger.LogInformation("Import logging is disabled. Add key 'ImportLogging' value 'True' to configuration to enable full logging.");
45+
logger.LogWarning($"WARNING: User groups import filter configured: '{userGroupsFilterString}'. Will not import data for users not in those groups");
6446
}
6547
}
66-
6748
}
6849
}

‎src/AnalyticsEngine/Common/DataUtils/JobTimer.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System;
1+
using Microsoft.Extensions.Logging;
2+
using System;
23
using System.Collections.Generic;
34
using System.Diagnostics;
45
using static DataUtils.AnalyticsLogger;

‎src/AnalyticsEngine/Common/Entities/Config/AppConfig.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public AppConfig()
2020
this.AppInsightsApiKey = ConfigurationManager.AppSettings["AppInsightsApiKey"];
2121
this.AppInsightsAppId = ConfigurationManager.AppSettings["AppInsightsAppId"];
2222

23+
this.BuildLabel = ConfigurationManager.AppSettings["BuildLabel"];
2324

2425
this.ClientID = ConfigurationManager.AppSettings.Get("ClientID");
2526
this.ClientSecret = ConfigurationManager.AppSettings.Get("ClientSecret");
@@ -28,6 +29,9 @@ public AppConfig()
2829
this.AADInstance = ConfigurationManager.AppSettings.Get("AADInstance");
2930
this.KeyVaultUrl = ConfigurationManager.AppSettings.Get("KeyVaultUrl");
3031

32+
// New: UserGroupsFilter (optional)
33+
this.UserGroupsFilter = ConfigurationManager.AppSettings.Get("UserGroupsFilter");
34+
3135
var useClientCertificate = ConfigurationManager.AppSettings.Get("UseClientCertificate");
3236
if (!string.IsNullOrEmpty(useClientCertificate))
3337
{
@@ -73,7 +77,7 @@ public AppConfig()
7377
}
7478
}
7579

76-
80+
public string BuildLabel { get; set; }
7781
public string AppInsightsContainerName { get; set; }
7882
public string AppInsightsApiKey { get; set; }
7983
public string AppInsightsAppId { get; set; }
@@ -132,5 +136,10 @@ public List<string> ContentTypesToRead
132136
public string StatsApiUrl { get; set; } = null;
133137

134138
public AppConnectionStrings ConnectionStrings { get; set; } = null;
139+
140+
/// <summary>
141+
/// Optional filter for user groups
142+
/// </summary>
143+
public string UserGroupsFilter { get; set; }
135144
}
136145
}

‎src/AnalyticsEngine/Common/Entities/Config/UserGroupsFilterModel.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text.RegularExpressions;
5+
6+
namespace Common.Entities.Config
7+
{
8+
/// <summary>
9+
/// Model to represent and match Entra ID group names from a filter string.
10+
/// </summary>
11+
public class UserGroupsFilterModel
12+
{
13+
public List<string> Patterns { get; }
14+
15+
public UserGroupsFilterModel() : this(string.Empty) { }
16+
public UserGroupsFilterModel(string filterString)
17+
{
18+
if (string.IsNullOrWhiteSpace(filterString))
19+
{
20+
Patterns = new List<string>();
21+
}
22+
else
23+
{
24+
Patterns = filterString.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries)
25+
.Select(p => p.Trim())
26+
.Where(p => !string.IsNullOrEmpty(p))
27+
.ToList();
28+
}
29+
}
30+
31+
/// <summary>
32+
/// Checks if the given group name matches any filter pattern (supports * wildcard).
33+
/// </summary>
34+
public bool Matches(string groupName)
35+
{
36+
if (string.IsNullOrEmpty(groupName) || Patterns.Count == 0)
37+
return false;
38+
39+
foreach (var pattern in Patterns)
40+
{
41+
var regexPattern = "^" + Regex.Escape(pattern).Replace("\\*", ".*") + "$";
42+
if (Regex.IsMatch(groupName, regexPattern, RegexOptions.IgnoreCase))
43+
return true;
44+
}
45+
return false;
46+
}
47+
}
48+
}

‎src/AnalyticsEngine/Common/Entities/Entities.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
<Compile Include="Config\AppConfig.cs" />
7979
<Compile Include="Config\AppConnectionStrings.cs" />
8080
<Compile Include="Config\ImportConfig.cs" />
81+
<Compile Include="Config\UserGroupsFilterModel.cs" />
8182
<Compile Include="Entities\AuditLog\CopilotEvents.cs" />
8283
<Compile Include="Entities\OnlineMeeting.cs" />
8384
<Compile Include="Entities\UsageReports\AppPlatformUserActivityLog.cs" />

‎src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using WebJob.Office365ActivityImporter.Engine.ActivityAPI.Loaders;
2222
using WebJob.Office365ActivityImporter.Engine.Entities;
2323
using WebJob.Office365ActivityImporter.Engine.Entities.Serialisation;
24+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
2425

2526
namespace Tests.UnitTests
2627
{
@@ -146,7 +147,8 @@ public async Task OneDriveAppearsAsSPEventTests()
146147
var oneDriveEvent = DataGenerators.GetRandomSharePointLog();
147148
oneDriveEvent.Workload = ActivityImportConstants.WORKLOAD_OD;
148149
hits.Add(oneDriveEvent);
149-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
150+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
151+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
150152

151153
await hits.CommitAllToSQL(sqlPersist);
152154

@@ -181,7 +183,8 @@ public async Task RealSPActivityImportTests()
181183

182184
// Save
183185
int preSPLogsInsertSPEventsCount = db.sharepoint_events.Count();
184-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
186+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
187+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
185188
await sharePointLogs.CommitAllToSQL(sqlPersist);
186189

187190
// Validate new count
@@ -248,7 +251,8 @@ public async Task RealOtherActivityImportTests()
248251

249252
// Save
250253
int preSPLogsInsertSPEventsCount = db.AuditEventsCommon.Count();
251-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
254+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
255+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
252256
await otherLogs.CommitAllToSQL(sqlPersist);
253257

254258
// Validate new events count
@@ -442,7 +446,9 @@ public async Task DuplicateActivitiesTest()
442446
using (var db = new AnalyticsEntitiesContext())
443447
{
444448
int preInsertCount = db.sharepoint_events.Count();
445-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
449+
450+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
451+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
446452

447453
// Create content-set for two different-but-same-id activities
448454
TestActivityReportSet duplicateContent = new TestActivityReportSet() { randomActivity, duplicateIdRandomActivity };
@@ -557,7 +563,9 @@ async Task InsertAndTestSPEvents(int count, bool allRandomLookups)
557563

558564
// Save
559565
var tempCache = ActivityImportCache.GetEmptyCache();
560-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
566+
567+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
568+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
561569

562570
var s = await hitsActivity.CommitAllToSQL(sqlPersist);
563571

@@ -657,7 +665,9 @@ public async Task FakeActivityTests()
657665
var importer = new ActivityWebImporter(fakeClient, s, telemetry);
658666

659667
// Download all the things & get stats.
660-
var stats = await importer.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), telemetry, s));
668+
669+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
670+
var stats = await importer.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig()));
661671

662672
var contentMetaDataLoader = new WebContentMetaDataLoader(telemetry, fakeClient, s);
663673

‎src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/MockUserGroupsCache.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
using Common.Entities.Config;
2+
using Microsoft.Extensions.Logging;
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Threading.Tasks;
6+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
7+
using Microsoft.VisualStudio.TestTools.UnitTesting;
8+
9+
namespace Tests.UnitTests.FakeLoaderClasses
10+
{
11+
public class MockUserGroupsCache : UserGroupsCache
12+
{
13+
private readonly Dictionary<string, List<string>> _mockGroups;
14+
15+
public MockUserGroupsCache(Dictionary<string, List<string>> mockGroups, ILogger logger = null)
16+
: base(logger)
17+
{
18+
_mockGroups = mockGroups ?? new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
19+
}
20+
21+
protected override Task<List<string>> LoadGroupsFromExternalAsync(string upn)
22+
{
23+
if (_mockGroups.TryGetValue(upn, out var groups))
24+
return Task.FromResult(groups);
25+
return Task.FromResult(new List<string>());
26+
}
27+
}
28+
}

‎src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public async Task UserAppLoaderFakeTest()
3535
{
3636
const int users = 10000;
3737
var l = new FakeUserAppLoader(AnalyticsLogger.ConsoleOnlyTracer(), users);
38-
var updates = await l.LoadAndSave();
38+
var updates = await l.LoadAndSave(new NoUsersHaveGroupsUserGroupsCache(AnalyticsLogger.ConsoleOnlyTracer()), new UserGroupsFilterModel());
3939
Assert.IsTrue(updates == users);
4040
}
4141

@@ -56,7 +56,7 @@ public async Task UserAppLoaderRealTest()
5656
await userUpdater.InsertAndUpdateDatabaseUsersFromGraph();
5757

5858
var updater = new UserAppLogUpdater(telemetry, new AppConfig());
59-
var sucess = await updater.UpdateUserInstalledApps(graphClient);
59+
var sucess = await updater.UpdateUserInstalledApps(graphClient, new NoUsersHaveGroupsUserGroupsCache(telemetry), new UserGroupsFilterModel());
6060
Assert.IsTrue(sucess);
6161
}
6262

‎src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using WebJob.Office365ActivityImporter.Engine;
1010
using WebJob.Office365ActivityImporter.Engine.Graph;
1111
using WebJob.Office365ActivityImporter.Engine.Graph.UsageReports.Aggregate;
12+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
1213

1314
namespace Tests.UnitTests
1415
{
@@ -58,10 +59,14 @@ public async Task AllO365ActivityTests()
5859
var telemetry = AnalyticsLogger.ConsoleOnlyTracer();
5960
var authConfig = new AppConfig();
6061

61-
var graphImporter = new GraphImporter(telemetry, authConfig);
6262
var graphAppIndentityOAuthContext = new GraphAppIndentityOAuthContext(telemetry, authConfig.ClientID, authConfig.TenantGUID.ToString(), authConfig.ClientSecret, authConfig.KeyVaultUrl, authConfig.UseClientCertificate);
63+
await graphAppIndentityOAuthContext.InitClientCredential();
64+
65+
var graphClient = new Microsoft.Graph.GraphServiceClient(graphAppIndentityOAuthContext.Creds);
66+
var graphImporter = new GraphImporter(telemetry, new NoUsersHaveGroupsUserGroupsCache(telemetry), graphAppIndentityOAuthContext, graphClient, authConfig);
6367

64-
await graphImporter.GetAndSaveActivityReportsMultiThreaded(1, new ManualGraphCallClient(graphAppIndentityOAuthContext, telemetry));
68+
await graphImporter.GetAndSaveActivityReportsMultiThreaded(1, new ManualGraphCallClient(graphAppIndentityOAuthContext, telemetry),
69+
new NoUsersHaveGroupsUserGroupsCache(telemetry), new UserGroupsFilterModel());
6570
}
6671

6772
[TestMethod]

0 commit comments

Comments
 (0)