Optional redis fixes + import fixes for audit logs

Author: sambetts

src/AnalyticsEngine/Common/DataUtils/ConsoleApp.cs

@@ -1,6 +1,5 @@
 using Microsoft.Extensions.Logging;
 using System;
-using System.Configuration;
 
 namespace DataUtils
 {
@@ -35,34 +34,16 @@ public static void BombOut(bool error)
             }
         }
 
-        public static void PrintStartupAndLoggingConfig(ILogger logger)
+        public static void PrintStartupAndLoggingConfig(string efConnectionString, string buildLabel, string userGroupsFilterString, ILogger logger)
         {
-            if (logger is null)
-            {
-                throw new ArgumentNullException(nameof(logger));
-            }
 
-            var buildLabel = ConfigurationManager.AppSettings["BuildLabel"];
             logger.LogInformation($"Office 365 Advanced Analytics engine START: '{buildLabel}'.");
-
-            string efConnectionString = ConfigurationManager.ConnectionStrings["SPOInsightsEntities"].ConnectionString;
             var sqlConnectionInfo = new System.Data.SqlClient.SqlConnectionStringBuilder(efConnectionString);
             logger.LogInformation($"Destination SQL Server='{sqlConnectionInfo.DataSource}', DB='{sqlConnectionInfo.InitialCatalog}'.");
-
-            bool loggingEnabled = ConfigurationManager.AppSettings["ImportLogging"] == "True";
-#if DEBUG
-            loggingEnabled = true;
-#endif
-
-            if (loggingEnabled)
+            if (!string.IsNullOrEmpty(userGroupsFilterString))
             {
-                logger.LogInformation("Import logging is ENABLED.");
-            }
-            else
-            {
-                logger.LogInformation("Import logging is disabled. Add key 'ImportLogging' value 'True' to configuration to enable full logging.");
+                logger.LogWarning($"WARNING: User groups import filter configured: '{userGroupsFilterString}'. Will not import data for users not in those groups");
             }
         }
-
     }
 }

src/AnalyticsEngine/Common/Entities/Config/AppConfig.cs

@@ -20,6 +20,7 @@ public AppConfig()
             this.AppInsightsApiKey = ConfigurationManager.AppSettings["AppInsightsApiKey"];
             this.AppInsightsAppId = ConfigurationManager.AppSettings["AppInsightsAppId"];
 
+            this.BuildLabel = ConfigurationManager.AppSettings["BuildLabel"];
 
             this.ClientID = ConfigurationManager.AppSettings.Get("ClientID");
             this.ClientSecret = ConfigurationManager.AppSettings.Get("ClientSecret");
@@ -76,7 +77,7 @@ public AppConfig()
             }
         }
 
-
+        public string BuildLabel { get; set; }
         public string AppInsightsContainerName { get; set; }
         public string AppInsightsApiKey { get; set; }
         public string AppInsightsAppId { get; set; }

src/AnalyticsEngine/WebJob.AppInsightsImporter/Program.cs

@@ -155,25 +155,15 @@ private static void BombOut()
         /// Output the config and check it's good.
         /// </summary>
         /// <returns>If the config is valid</returns>
-        private static bool ValidateAndPrintConfig(AppConfig config, ILogger telemetry)
+        private static bool ValidateAndPrintConfig(AppConfig settings, ILogger telemetry)
         {
-            ConsoleApp.PrintStartupAndLoggingConfig(telemetry);
+            ConsoleApp.PrintStartupAndLoggingConfig(settings.ConnectionStrings.DatabaseConnectionString, settings.BuildLabel, settings.UserGroupsFilter, telemetry);
 
             // Have config object test
 
-            var accountName = StringUtils.FindValueForProp(config.ConnectionStrings.DatabaseConnectionString, "AccountName");
+            settings.ConnectionStrings.TestSQLSettings(telemetry);
 
-            string efConnectionString = System.Configuration.ConfigurationManager.ConnectionStrings["SPOInsightsEntities"].ConnectionString;
-            var sqlConnectionInfo = new System.Data.SqlClient.SqlConnectionStringBuilder(efConnectionString);
-
-            telemetry.LogInformation("Configured values:");
-            //pIIConfig.PrintConfig(telemetry);     // PII anonisation is broken for the time being in this project, but nobody uses it anyway...
-
-            telemetry.LogInformation($"Destination SQL Server configuration: \ndata source='{sqlConnectionInfo.DataSource}', initial catalog='{sqlConnectionInfo.InitialCatalog}'");
-
-            config.ConnectionStrings.TestSQLSettings(telemetry);
-
-            if (string.IsNullOrEmpty(config.AppInsightsApiKey))
+            if (string.IsNullOrEmpty(settings.AppInsightsApiKey))
             {
                 telemetry.LogInformation("Critical: no Application Insights API key found - can't continue. Run the latest installer again to reset configuration.");
                 return false;

src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/ActivityAPI/ActivityReportSqlPersistenceManager.cs

@@ -128,14 +128,15 @@ private async Task<ImportStat> SaveToSqlAllTheThings(ActivityReportSet activitie
                         }
                         else
                         {
+                            result = SaveResultEnum.UserOutOfScope;
                             _telemetry.LogInformation($"Skipping activity report for user '{abtractLog.UserId}' - not in user groups filter");
                         }
                     }
                     else
                     {
                         // No URL
                         cache.RememberNewlyIgnoredEvent(abtractLog);
-                        result = SaveResultEnum.OutOfScope;
+                        result = SaveResultEnum.UrlOutOfScope;
                     }
 
                     // Update stats
@@ -145,8 +146,9 @@ private async Task<ImportStat> SaveToSqlAllTheThings(ActivityReportSet activitie
                         listOfActivitiesSavedToSQL.Add(abtractLog);
                     }
                     else if (result == SaveResultEnum.ProcessedAlready) stats.ProcessedAlready++;
-                    else if (result == SaveResultEnum.OutOfScope) stats.URLsOutOfScope++;
-                    else throw new InvalidOperationException("Unexpected save result");
+                    else if (result == SaveResultEnum.UrlOutOfScope) stats.URLsOutOfScope++;
+                    else if (result == SaveResultEnum.UserOutOfScope) stats.UsersOutOfScope++;
+                    else _telemetry.LogError($"Unexpected log result for log {abtractLog.Id}");
 
                     processedIds.Add(abtractLog.Id);
                 }

src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/ImportStat.cs

@@ -12,6 +12,7 @@ public class ImportStat
         public int Imported { get; set; }
         public int ProcessedAlready { get; set; }
         public int URLsOutOfScope { get; set; }
+        public int UsersOutOfScope { get; set; }
         public int DownloadErrors { get; set; }
         public int Total { get; set; }
 
@@ -36,6 +37,7 @@ public override string ToString()
                 $"Imported successfully: {this.Imported.ToString("n0")}, " +
                 $"already processed: {this.ProcessedAlready.ToString("n0")}, " +
                 $"URLs out of scope (orgs table): {this.URLsOutOfScope.ToString("n0")}, " +
+                $"users out of scope: {this.UsersOutOfScope.ToString("n0")}, " +
                 $"errors: {this.DownloadErrors.ToString("n0")}, " +
                 $"total: {this.Total.ToString("n0")}";
         }

src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Entities/Serialisation/CommonAuditLogContent.cs

@@ -127,6 +127,7 @@ public enum SaveResultEnum
         NotSaved = 0,           // Default
         ProcessedAlready = 1,   // Event ignored previously
         Imported = 2,           // Already imported
-        OutOfScope = 3          // Not to be imported. Usually because the SharePoint URL is for a site we don't care about.
+        UrlOutOfScope = 3,          // Not to be imported. Usually because the SharePoint URL is for a site we don't care about.
+        UserOutOfScope = 4,          // Not to be imported. Usually because the user is outside group filter
     }
 }

src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/Teams/TeamTokenManager.cs

@@ -14,13 +14,13 @@ public class TeamTokenManager
     {
         public TeamTokenManager(O365Team team, AppConfig appConfig, ILogger logger)
         {
-            if (appConfig.ConnectionStrings.RedisConnectionString != null)
+            if (!string.IsNullOrEmpty(appConfig.ConnectionStrings.RedisConnectionString))
             {
                 this.CacheConnectionManager = CacheConnectionManager.GetConnectionManager(appConfig.ConnectionStrings.RedisConnectionString);
             }
             else
             {
-                logger.LogWarning("No redis connection string found in app config. No caching will be done.");
+                logger.LogWarning("No redis connection string found in config. No deep Teams analytics will be possible.");
             }
             this.Team = team;
         }

src/AnalyticsEngine/WebJob.Office365ActivityImporter.Engine/Graph/User/UserMetadataUpdater.cs

@@ -37,7 +37,7 @@ public UserMetadataUpdater(AnalyticsLogger telemetry, AppConfig settings, TokenC
 
 
             IDeltaValueProvider deltaProvider = null;
-            if (settings.ConnectionStrings.RedisConnectionString != null)
+            if (!string.IsNullOrEmpty(settings.ConnectionStrings.RedisConnectionString))
             {
                 deltaProvider = new RedisProcessDeltaValueProvider(settings, telemetry);
                 telemetry.LogInformation($"User import - using Redis for delta token cache.");

src/AnalyticsEngine/WebJob.Office365ActivityImporter/Program.cs

@@ -205,7 +205,7 @@ static async Task MainAsync(string[] args)
                 using (var db = new AnalyticsEntitiesContext())
                 {
                     var sqlUsageBuilder = new SqlUsageStatsBuilder(db, telemetry, configuredSettings.TenantGUID);
-                    if (configuredSettings.ConnectionStrings.RedisConnectionString != null)
+                    if (!string.IsNullOrEmpty(configuredSettings.ConnectionStrings.RedisConnectionString))
                     {
                         var redisDatesAdaptor = new RedisStatsDatesLoader(configuredSettings);
 
@@ -259,7 +259,7 @@ private static void TestDB(ILogger debugTracer)
         /// </summary>
         private static void PrintStartupDetails(AppConfig settings, ILogger telemetry)
         {
-            ConsoleApp.PrintStartupAndLoggingConfig(telemetry);
+            ConsoleApp.PrintStartupAndLoggingConfig(settings.ConnectionStrings.DatabaseConnectionString, settings.BuildLabel, settings.UserGroupsFilter, telemetry);
 
             var efConnectionString = ConfigurationManager.ConnectionStrings["SPOInsightsEntities"].ConnectionString;
             var sqlConnectionInfo = new System.Data.SqlClient.SqlConnectionStringBuilder(efConnectionString);

src/AnalyticsEngine/WebJob.Office365ActivityImporter/ProgramTasks.cs

@@ -91,6 +91,8 @@ async Task InitAuth()
             _graphClient.HttpProvider.OverallTimeout = TimeSpan.FromHours(1);
             _manualGraphCallClient = new ManualGraphCallClient(_graphAppIndentityOAuthContext, _telemetry);
             _graphUserGroupsCache = new GraphUserGroupsCache(_manualGraphCallClient, _telemetry);
+
+
             _isInitialized = true;
 
         }