Skip to content

Commit db7f292

Browse files
sambettssambetts
committed
All changes made for filtering. Pending testing
1 parent cb60118 commit db7f292

28 files changed

+273
-103
lines changed

‎src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstallVerifier.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using WebJob.Office365ActivityImporter.Engine;
2222
using WebJob.Office365ActivityImporter.Engine.ActivityAPI;
2323
using WebJob.Office365ActivityImporter.Engine.Graph.UsageReports;
24+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
2425
using static App.ControlPanel.Engine.Models.AutodetectedSqlAndFtpDetails;
2526

2627
namespace App.ControlPanel.Engine
@@ -308,7 +309,10 @@ async Task VerifyTeamsAndUserActivityImport(string clientId, string tenantId, st
308309

309310
var graphClient = new Microsoft.Graph.GraphServiceClient(auth.Creds);
310311

311-
var teamsUserUsageLoader = new TeamsUserUsageLoader(new WebJob.Office365ActivityImporter.Engine.Graph.ManualGraphCallClient(auth, telemetry), telemetry);
312+
var teamsUserUsageLoader = new TeamsUserUsageLoader(new WebJob.Office365ActivityImporter.Engine.Graph.ManualGraphCallClient(auth, telemetry),
313+
new NoUsersHaveGroupsUserGroupsCache(_logger),
314+
new Common.Entities.Config.UserGroupsFilterModel(string.Empty),
315+
telemetry);
312316

313317
// Usage reports
314318
if (Config.SolutionConfig.ImportTaskSettings.GraphUsageReports)

‎src/AnalyticsEngine/Common/DataUtils/JobTimer.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using System;
1+
using Microsoft.Extensions.Logging;
2+
using System;
23
using System.Collections.Generic;
34
using System.Diagnostics;
45
using static DataUtils.AnalyticsLogger;

‎src/AnalyticsEngine/Common/Entities/Config/UserGroupsFilterModel.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ public class UserGroupsFilterModel
1212
{
1313
public List<string> Patterns { get; }
1414

15+
public UserGroupsFilterModel() : this(string.Empty) { }
1516
public UserGroupsFilterModel(string filterString)
1617
{
1718
if (string.IsNullOrWhiteSpace(filterString))

‎src/AnalyticsEngine/Tests.UnitTests/ActivityImporterTests.cs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using WebJob.Office365ActivityImporter.Engine.ActivityAPI.Loaders;
2222
using WebJob.Office365ActivityImporter.Engine.Entities;
2323
using WebJob.Office365ActivityImporter.Engine.Entities.Serialisation;
24+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
2425

2526
namespace Tests.UnitTests
2627
{
@@ -146,7 +147,8 @@ public async Task OneDriveAppearsAsSPEventTests()
146147
var oneDriveEvent = DataGenerators.GetRandomSharePointLog();
147148
oneDriveEvent.Workload = ActivityImportConstants.WORKLOAD_OD;
148149
hits.Add(oneDriveEvent);
149-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
150+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
151+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
150152

151153
await hits.CommitAllToSQL(sqlPersist);
152154

@@ -181,7 +183,8 @@ public async Task RealSPActivityImportTests()
181183

182184
// Save
183185
int preSPLogsInsertSPEventsCount = db.sharepoint_events.Count();
184-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
186+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
187+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
185188
await sharePointLogs.CommitAllToSQL(sqlPersist);
186189

187190
// Validate new count
@@ -248,7 +251,8 @@ public async Task RealOtherActivityImportTests()
248251

249252
// Save
250253
int preSPLogsInsertSPEventsCount = db.AuditEventsCommon.Count();
251-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
254+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
255+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
252256
await otherLogs.CommitAllToSQL(sqlPersist);
253257

254258
// Validate new events count
@@ -442,7 +446,9 @@ public async Task DuplicateActivitiesTest()
442446
using (var db = new AnalyticsEntitiesContext())
443447
{
444448
int preInsertCount = db.sharepoint_events.Count();
445-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
449+
450+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
451+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
446452

447453
// Create content-set for two different-but-same-id activities
448454
TestActivityReportSet duplicateContent = new TestActivityReportSet() { randomActivity, duplicateIdRandomActivity };
@@ -557,7 +563,9 @@ async Task InsertAndTestSPEvents(int count, bool allRandomLookups)
557563

558564
// Save
559565
var tempCache = ActivityImportCache.GetEmptyCache();
560-
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), AnalyticsLogger.ConsoleOnlyTracer(), new AppConfig());
566+
567+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
568+
var sqlPersist = new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig());
561569

562570
var s = await hitsActivity.CommitAllToSQL(sqlPersist);
563571

@@ -657,7 +665,9 @@ public async Task FakeActivityTests()
657665
var importer = new ActivityWebImporter(fakeClient, s, telemetry);
658666

659667
// Download all the things & get stats.
660-
var stats = await importer.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), telemetry, s));
668+
669+
var logger = AnalyticsLogger.ConsoleOnlyTracer();
670+
var stats = await importer.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(logger), logger, new AppConfig()));
661671

662672
var contentMetaDataLoader = new WebContentMetaDataLoader(telemetry, fakeClient, s);
663673

‎src/AnalyticsEngine/Tests.UnitTests/FakeLoaderClasses/MockUserGroupsCache.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public MockUserGroupsCache(Dictionary<string, List<string>> mockGroups, ILogger
1818
_mockGroups = mockGroups ?? new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
1919
}
2020

21-
protected override Task<List<string>> LoadGroupsFromGraphAsync(string upn)
21+
protected override Task<List<string>> LoadGroupsFromExternalAsync(string upn)
2222
{
2323
if (_mockGroups.TryGetValue(upn, out var groups))
2424
return Task.FromResult(groups);

‎src/AnalyticsEngine/Tests.UnitTests/GraphImportTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public async Task UserAppLoaderFakeTest()
3535
{
3636
const int users = 10000;
3737
var l = new FakeUserAppLoader(AnalyticsLogger.ConsoleOnlyTracer(), users);
38-
var updates = await l.LoadAndSave();
38+
var updates = await l.LoadAndSave(new NoUsersHaveGroupsUserGroupsCache(AnalyticsLogger.ConsoleOnlyTracer()), new UserGroupsFilterModel());
3939
Assert.IsTrue(updates == users);
4040
}
4141

@@ -56,7 +56,7 @@ public async Task UserAppLoaderRealTest()
5656
await userUpdater.InsertAndUpdateDatabaseUsersFromGraph();
5757

5858
var updater = new UserAppLogUpdater(telemetry, new AppConfig());
59-
var sucess = await updater.UpdateUserInstalledApps(graphClient);
59+
var sucess = await updater.UpdateUserInstalledApps(graphClient, new NoUsersHaveGroupsUserGroupsCache(telemetry), new UserGroupsFilterModel());
6060
Assert.IsTrue(sucess);
6161
}
6262

‎src/AnalyticsEngine/Tests.UnitTests/GraphUsageReportImportTests.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using WebJob.Office365ActivityImporter.Engine;
1010
using WebJob.Office365ActivityImporter.Engine.Graph;
1111
using WebJob.Office365ActivityImporter.Engine.Graph.UsageReports.Aggregate;
12+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
1213

1314
namespace Tests.UnitTests
1415
{
@@ -58,10 +59,14 @@ public async Task AllO365ActivityTests()
5859
var telemetry = AnalyticsLogger.ConsoleOnlyTracer();
5960
var authConfig = new AppConfig();
6061

61-
var graphImporter = new GraphImporter(telemetry, authConfig);
6262
var graphAppIndentityOAuthContext = new GraphAppIndentityOAuthContext(telemetry, authConfig.ClientID, authConfig.TenantGUID.ToString(), authConfig.ClientSecret, authConfig.KeyVaultUrl, authConfig.UseClientCertificate);
63+
await graphAppIndentityOAuthContext.InitClientCredential();
64+
65+
var graphClient = new Microsoft.Graph.GraphServiceClient(graphAppIndentityOAuthContext.Creds);
66+
var graphImporter = new GraphImporter(telemetry, new NoUsersHaveGroupsUserGroupsCache(telemetry), graphAppIndentityOAuthContext, graphClient, authConfig);
6367

64-
await graphImporter.GetAndSaveActivityReportsMultiThreaded(1, new ManualGraphCallClient(graphAppIndentityOAuthContext, telemetry));
68+
await graphImporter.GetAndSaveActivityReportsMultiThreaded(1, new ManualGraphCallClient(graphAppIndentityOAuthContext, telemetry),
69+
new NoUsersHaveGroupsUserGroupsCache(telemetry), new UserGroupsFilterModel());
6570
}
6671

6772
[TestMethod]

‎src/AnalyticsEngine/Tests.UnitTests/Program.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using Tests.UnitTests.FakeLoaderClasses;
77
using WebJob.Office365ActivityImporter.Engine;
88
using WebJob.Office365ActivityImporter.Engine.ActivityAPI;
9+
using WebJob.Office365ActivityImporter.Engine.Graph.User;
910

1011
namespace Tests.UnitTests
1112
{
@@ -36,7 +37,7 @@ private static async Task DoThing()
3637
var t = new JobTimer(telemetry, "Soyve");
3738
t.Start();
3839
Console.WriteLine("Saving data...");
39-
await testLoader.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), telemetry, new AppConfig()));
40+
await testLoader.LoadReportsAndSave(new ActivityReportSqlPersistenceManager(new AllowAllFilterConfig(), new NoUsersHaveGroupsUserGroupsCache(telemetry), telemetry, new AppConfig()));
4041

4142
t.StopAndPrintElapsed();
4243
}

‎src/AnalyticsEngine/Tests.UnitTests/UserGroupsCacheTests.cs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
using Common.Entities.Config;
2-
using Microsoft.Extensions.Logging;
32
using System.Collections.Generic;
43
using System.Threading.Tasks;
54
using Tests.UnitTests.FakeLoaderClasses;
65
using Microsoft.VisualStudio.TestTools.UnitTesting;
7-
using WebJob.Office365ActivityImporter.Engine.Graph.User;
86

97
namespace Tests.UnitTests
108
{
@@ -46,5 +44,26 @@ public async Task IsInGroupsFilter_ReturnsFalse_WhenUserNotInMatchingGroup()
4644
// Assert
4745
Microsoft.VisualStudio.TestTools.UnitTesting.Assert.IsFalse(result);
4846
}
47+
48+
[TestMethod]
49+
public async Task IsInGroupsFilter_ReturnsTrue_WhenNoFilterConfigured()
50+
{
51+
// Arrange
52+
var mockGroups = new Dictionary<string, List<string>>
53+
{
54+
{ "user1@contoso.com", new List<string> { "Finance", "HR" } },
55+
{ "user2@contoso.com", new List<string> { "Marketing", "Sales" } },
56+
{ "user3@contoso.com", new List<string> { "IT", "Support" } }
57+
};
58+
var cache = new MockUserGroupsCache(mockGroups);
59+
var filter = new UserGroupsFilterModel(); // No filter configured
60+
61+
// Act & Assert
62+
foreach (var user in mockGroups.Keys)
63+
{
64+
var result = await cache.IsInGroupsFilter(user, filter);
65+
Microsoft.VisualStudio.TestTools.UnitTesting.Assert.IsTrue(result, $"Expected true for user {user} when no filter is configured.");
66+
}
67+
}
4968
}
5069
}

‎src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Abstract.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using Common.Entities.Config;
22
using DataUtils;
33
using DataUtils.Http;
4+
using Microsoft.Extensions.Logging;
45

56
namespace WebJob.Office365ActivityImporter.Engine
67
{

0 commit comments

Comments
 (0)