[in-proc] Backport log level changes and failure handling from #11100 (#11156)

* Set errors to warning

* Check for _hostId first so we avoid lookups.

* Disable service when there's any failure (even if transient) as part of flush logs, purge events or execute batch.

Author: cjaliaga

src/WebJobs.Script.WebHost/Diagnostics/DiagnosticEventTableStorageRepository.Log.cs

@@ -38,16 +38,16 @@ private static class Logger
                 LoggerMessage.Define<string>(LogLevel.Debug, new EventId(6, nameof(DeletingTableWithOutdatedEventVersion)), "Deleting table '{tableName}' as it contains records with an outdated EventVersion.");
 
             private static readonly Action<ILogger, Exception> _errorPurgingDiagnosticEventVersions =
-                LoggerMessage.Define(LogLevel.Error, new EventId(7, nameof(ErrorPurgingDiagnosticEventVersions)), "Error occurred when attempting to purge previous diagnostic event versions.");
+                LoggerMessage.Define(LogLevel.Warning, new EventId(7, nameof(ErrorPurgingDiagnosticEventVersions)), "Error occurred when attempting to purge previous diagnostic event versions.");
 
             private static readonly Action<ILogger, Exception> _unableToGetTableReference =
-                LoggerMessage.Define(LogLevel.Error, new EventId(8, nameof(UnableToGetTableReference)), "Unable to get table reference. Aborting write operation.");
+                LoggerMessage.Define(LogLevel.Warning, new EventId(8, nameof(UnableToGetTableReference)), "Unable to get table reference. Aborting write operation.");
 
             private static readonly Action<ILogger, Exception> _unableToGetTableReferenceOrCreateTable =
-                LoggerMessage.Define(LogLevel.Error, new EventId(9, nameof(UnableToGetTableReferenceOrCreateTable)), "Unable to get table reference or create table. Aborting write operation.");
+                LoggerMessage.Define(LogLevel.Warning, new EventId(9, nameof(UnableToGetTableReferenceOrCreateTable)), "Unable to get table reference or create table. Aborting write operation.");
 
             private static readonly Action<ILogger, Exception> _unableToWriteDiagnosticEvents =
-                LoggerMessage.Define(LogLevel.Error, new EventId(10, nameof(UnableToWriteDiagnosticEvents)), "Unable to write diagnostic events to table storage.");
+                LoggerMessage.Define(LogLevel.Warning, new EventId(10, nameof(UnableToWriteDiagnosticEvents)), "Unable to write diagnostic events to table storage.");
 
             private static readonly Action<ILogger, Exception> _primaryHostStateProviderNotAvailable =
                 LoggerMessage.Define(LogLevel.Debug, new EventId(11, nameof(PrimaryHostStateProviderNotAvailable)), "PrimaryHostStateProvider is not available. Skipping the check for primary host.");

src/WebJobs.Script.WebHost/Diagnostics/DiagnosticEventTableStorageRepository.cs

@@ -107,7 +107,7 @@ internal string HostId
         {
             get
             {
-                if (!_environment.IsPlaceholderModeEnabled() && string.IsNullOrEmpty(_hostId))
+                if (string.IsNullOrEmpty(_hostId) && !_environment.IsPlaceholderModeEnabled())
                 {
                     _hostId = _hostIdProvider?.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
                 }
@@ -192,9 +192,23 @@ await Utility.InvokeWithRetriesAsync(async () =>
 
                     _purged = true;
                 }
+                catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.Forbidden)
+                {
+                    // If we reach this point, we already checked for permissions on TableClient initialization.
+                    // It is possible that the permissions changed after the initialization, any firewall/network rules were changed or it's a custom role where we don't have permissions to query entities.
+                    // We will log the error and disable the service.
+                    Logger.ErrorPurgingDiagnosticEventVersions(_logger, ex);
+                    DisableService();
+                    Logger.ServiceDisabledUnauthorizedClient(_logger, ex);
+                }
                 catch (Exception ex)
                 {
+                    // We failed to connect to the table storage account. This could be due to a transient error or a configuration issue (e.g., network problems).
+                    // To avoid repeatedly retrying in a potentially unhealthy state, we will disable the service.
+                    // The operation may succeed in a future instance if the underlying issue is resolved.
                     Logger.ErrorPurgingDiagnosticEventVersions(_logger, ex);
+                    DisableService();
+                    Logger.ServiceDisabledUnableToConnectToStorage(_logger, ex);
                 }
             }, maxRetries: 5, retryInterval: TimeSpan.FromSeconds(5));
 
@@ -245,12 +259,16 @@ internal virtual async Task FlushLogs(TableClient table = null)
                 Logger.UnableToGetTableReferenceOrCreateTable(_logger, ex);
                 DisableService();
                 Logger.ServiceDisabledUnauthorizedClient(_logger, ex);
+                return;
             }
             catch (Exception ex)
             {
+                // We failed to connect to the table storage account. This could be due to a transient error or a configuration issue (e.g., network problems).
+                // To avoid repeatedly retrying in a potentially unhealthy state, we will disable the service.
+                // The operation may succeed in a future instance if the underlying issue is resolved.
                 Logger.UnableToGetTableReferenceOrCreateTable(_logger, ex);
-                // Clearing the memory cache to avoid memory build up.
-                _events.Clear();
+                DisableService();
+                Logger.ServiceDisabledUnableToConnectToStorage(_logger, ex);
                 return;
             }
 
@@ -292,12 +310,14 @@ internal async Task ExecuteBatchAsync(ConcurrentDictionary<string, DiagnosticEve
             catch (Exception ex)
             {
                 Logger.UnableToWriteDiagnosticEvents(_logger, ex);
+                DisableService();
+                Logger.ServiceDisabledUnableToConnectToStorage(_logger, ex);
             }
         }
 
         public void WriteDiagnosticEvent(DateTime timestamp, string errorCode, LogLevel level, string message, string helpLink, Exception exception)
         {
-            if (TableClient is null || string.IsNullOrEmpty(HostId))
+            if (TableClient is null || string.IsNullOrEmpty(HostId) || !IsEnabled())
             {
                 return;
             }