EventHub + ServiceHub with EventProducerIdentifier (#174)

Author: snakefoot

src/NLog.Extensions.AzureBlobStorage/BlobStorageTarget.cs

@@ -348,7 +348,7 @@ protected override Task WriteAsyncTask(IList<LogEventInfo> logEvents, Cancellati
                 }
             }
 
-            return Task.WhenAll(multipleTasks ?? new Task[0]);
+            return multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask;
         }
 
         private byte[] CreateBlobPayload(IList<LogEventInfo> logEvents)
@@ -474,7 +474,7 @@ public override int GetHashCode()
             }
         }
 
-        class CloudBlobService : ICloudBlobService
+        private sealed class CloudBlobService : ICloudBlobService
         {
             private IDictionary<string, string> _blobMetadata;
             private IDictionary<string, string> _blobTags;

src/NLog.Extensions.AzureBlobStorage/NLog.Extensions.AzureBlobStorage.csproj

@@ -24,9 +24,7 @@
     <RepositoryUrl>https://github.com/JDetmar/NLog.Extensions.AzureStorage.git</RepositoryUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageReleaseNotes>
-- Updated Azure.Identity ver. 1.14.2 to fix security issue
-- Updated NLog ver. 5.2.5 to support build-triming
-- Updated Azure.Storage.Blobs ver. 12.24.1
+- Added support for Proxy-options: ProxyAddress, ProxyLogin, ProxyPassword, NoProxy, UseDefaultCredentialsForProxy
 
 Docs: https://github.com/JDetmar/NLog.Extensions.AzureStorage/blob/master/src/NLog.Extensions.AzureBlobStorage/README.md
     </PackageReleaseNotes>

src/NLog.Extensions.AzureBlobStorage/README.md

@@ -25,7 +25,7 @@
 </targets>
 ```
 
-### Parameters
+### General Options
 
 _name_ - Name of the target.
 
@@ -37,9 +37,11 @@ _container_ - Azure blob container name. [Layout](https://github.com/NLog/NLog/w
 
 _contentType_ - Azure blob ContentType (Default = text/plain)
 
-_connectionString_ - Azure storage connection string. Ex. `UseDevelopmentStorage=true;`
+_connectionString_ - Azure Blob Storage connection string from your storage account. Required unless using `ServiceUri`.
 
-_serviceUri_ - Uri to reference the blob service (e.g. https://{account_name}.blob.core.windows.net). Input for `BlobServiceClient`. Required, when `connectionString` is not configured. Overrides `connectionString` when both are set.
+_serviceUri_ - Uri to reference the blob service (e.g. https://{account_name}.blob.core.windows.net). Alternative to ConnectionString, where Managed Identiy is applied from DefaultAzureCredential.
+
+### Authentication Options
 
 _managedIdentityClientId_ - Sets `ManagedIdentityClientId` on `DefaultAzureCredentialOptions`. Requires `serviceUri`.
 
@@ -57,6 +59,8 @@ _clientAuthId_ - clientId for `ClientSecretCredential` authentication. Requires
 
 _clientAuthSecret_ - clientSecret for `ClientSecretCredential` authentication. Requires `serviceUri`,`tenantIdentity` and `clientAuthId`.
 
+### Proxy Options
+
 _noProxy_ - Bypasses any system proxy and proxy in `ProxyAddress` when set to `true`.
 
 _proxyAddress_ - Address of the proxy server to use (e.g. http://proxyserver:8080).
@@ -65,7 +69,7 @@ _proxyLogin_ - Login to use for the proxy server. Requires `proxyPassword`.
 
 _proxyPassword_ - Password to use for the proxy server. Requires `proxyLogin`.
 
-_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server, overriding any values that may have been set in `proxyLogin` and `proxyPassword`.
+_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server.
 
 ### Batching Policy
 

src/NLog.Extensions.AzureDataTables/DataTablesTarget.cs

@@ -356,7 +356,7 @@ protected override Task WriteAsyncTask(IList<LogEventInfo> logEvents, Cancellati
                 }
             }
 
-            return Task.WhenAll(multipleTasks ?? new Task[0]);
+            return multipleTasks?.Count > 0 ? Task.WhenAll(multipleTasks) : Task.CompletedTask;
         }
 
         private async Task WriteMultipleBatchesAsync(IEnumerable<IEnumerable<TableTransactionAction>> batchCollection, string tableName, CancellationToken cancellationToken)
@@ -485,7 +485,7 @@ private static string TryLookupValue(Func<string> lookupFunc, string lookupType)
             }
         }
 
-        class CloudTableService : ICloudTableService
+        private sealed class CloudTableService : ICloudTableService
         {
             private TableServiceClient _client;
             private TableClient _table;

src/NLog.Extensions.AzureDataTables/NLog.Extensions.AzureDataTables.csproj

@@ -24,9 +24,8 @@
     <RepositoryUrl>https://github.com/JDetmar/NLog.Extensions.AzureStorage.git</RepositoryUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageReleaseNotes>
-- Updated Azure.Identity ver. 1.14.2 to fix security issue
-- Updated NLog ver. 5.2.5 to support build-triming
-- Updated Azure.Data.Tables ver. 12.11.0
+- Added support for Proxy-options: ProxyAddress, ProxyLogin, ProxyPassword, NoProxy, UseDefaultCredentialsForProxy
+- Added support for Authentication-options: ClientAuthId + ClientAuthSecret
 
 Docs: https://github.com/JDetmar/NLog.Extensions.AzureStorage/blob/master/src/NLog.Extensions.AzureDataTables/README.md
     </PackageReleaseNotes>

src/NLog.Extensions.AzureDataTables/README.md

@@ -22,16 +22,26 @@ Supports both Azure Storage Tables and CosmosDB Tables.
           logTimeStampFormat="O" />
 </targets>
 ```
-### Parameters
+### General Options
 
 _name_ - Name of the target.
 
 _layout_ - Text to be rendered. [Layout](https://github.com/NLog/NLog/wiki/Layouts) Required. 
 
-_connectionString_ - Azure storage connection string. [Layout](https://github.com/NLog/NLog/wiki/Layouts)
+_connectionString_ - Azure storage connection string. [Layout](https://github.com/NLog/NLog/wiki/Layouts). Required unless using `serviceUri`.
 
 _serviceUri_ - Alternative to ConnectionString, where Managed Identiy is acquired from DefaultAzureCredential.
 
+_tableName_ - Azure table name. [Layout](https://github.com/NLog/NLog/wiki/Layouts)
+
+_rowKey_ - Azure Table RowKey. [Layout](https://github.com/NLog/NLog/wiki/Layouts). Default = "InverseTicks_${guid}"
+
+_partitionKey_ - Azure PartitionKey. [Layout](https://github.com/NLog/NLog/wiki/Layouts). Default = `${logger}`
+
+_logTimeStampFormat_ - Default Log TimeStamp is set to 'O' for [Round-trip](https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings#the-round-trip-o-o-format-specifier) format if not specified.
+
+## Authentication Options
+
 _managedIdentityClientId_ - Sets `ManagedIdentityClientId` on `DefaultAzureCredentialOptions`. Requires `serviceUri`
 
 _managedIdentityResourceId_ - resourceId for `ManagedIdentityResourceId` on `DefaultAzureCredentialOptions`, do not use together with `ManagedIdentityClientId`. Requires `serviceUri`.
@@ -48,6 +58,8 @@ _clientAuthId_ - clientId for `ClientSecretCredential` authentication. Requires
 
 _clientAuthSecret_ - clientSecret for `ClientSecretCredential` authentication. Requires `serviceUri`,`tenantIdentity` and `clientAuthId`.
 
+## Proxy Options
+
 _noProxy_ - Bypasses any system proxy and proxy in `ProxyAddress` when set to `true`.
 
 _proxyAddress_ - Address of the proxy server to use (e.g. http://proxyserver:8080).
@@ -56,15 +68,7 @@ _proxyLogin_ - Login to use for the proxy server. Requires `proxyPassword`.
 
 _proxyPassword_ - Password to use for the proxy server. Requires `proxyLogin`.
 
-_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server, overriding any values that may have been set in `proxyLogin` and `proxyPassword`.
-
-_tableName_ - Azure table name. [Layout](https://github.com/NLog/NLog/wiki/Layouts)
-
-_rowKey_ - Azure Table RowKey. [Layout](https://github.com/NLog/NLog/wiki/Layouts). Default = "InverseTicks_${guid}"
-
-_partitionKey_ - Azure PartitionKey. [Layout](https://github.com/NLog/NLog/wiki/Layouts). Default = `${logger}`
-
-_logTimeStampFormat_ - Default Log TimeStamp is set to 'O' for [Round-trip](https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-date-and-time-format-strings#the-round-trip-o-o-format-specifier) format if not specified.
+_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server.
 
 ### Dynamic TableEntity
 Instead of using the predefined NLogEntity-properties, then one can specify wanted properties:

src/NLog.Extensions.AzureEventGrid/NLog.Extensions.AzureEventGrid.csproj

@@ -24,9 +24,8 @@
     <RepositoryUrl>https://github.com/JDetmar/NLog.Extensions.AzureStorage.git</RepositoryUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageReleaseNotes>
-- Updated Azure.Identity ver. 1.14.2 to fix security issue
-- Updated NLog ver. 5.2.5 to support build-triming
-- Updated Azure.Messaging.EventGrid ver. 5.0.0
+- Added support for Proxy-options: ProxyAddress, ProxyLogin, ProxyPassword, NoProxy, UseDefaultCredentialsForProxy
+- Added support for Authentication-options: ClientAuthId + ClientAuthSecret
 
 Docs: https://github.com/JDetmar/NLog.Extensions.AzureStorage/blob/master/src/NLog.Extensions.AzureEventGrid/README.md
     </PackageReleaseNotes>

src/NLog.Extensions.AzureEventGrid/README.md

@@ -28,7 +28,7 @@
 </targets>
 ```
 
-### Parameters
+### General Options
 
 _name_ - Name of the target.
 
@@ -48,6 +48,8 @@ _dataFormat_ - Format of the data-payload (Binary / Json). Default Binary. `Stri
 
 _dataSchema_ - Schema version of the data-payload. [Layout](https://github.com/NLog/NLog/wiki/Layouts)
 
+### Authentication Options
+
 _managedIdentityClientId_ - Sets `ManagedIdentityClientId` on `DefaultAzureCredentialOptions`. Requires `serviceUri`.
 
 _managedIdentityResourceId_ - resourceId for `ManagedIdentityResourceId` on `DefaultAzureCredentialOptions`, do not use together with `ManagedIdentityClientId`. Requires `serviceUri`.
@@ -62,6 +64,8 @@ _clientAuthId_ - clientId for `ClientSecretCredential` authentication. Requires
 
 _clientAuthSecret_ - clientSecret for `ClientSecretCredential` authentication. Requires `tenantIdentity` and `clientAuthId`.
 
+### Proxy Options
+
 _noProxy_ - Bypasses any system proxy and proxy in `ProxyAddress` when set to `true`.
 
 _proxyAddress_ - Address of the proxy server to use (e.g. http://proxyserver:8080).
@@ -70,8 +74,7 @@ _proxyLogin_ - Login to use for the proxy server. Requires `proxyPassword`.
 
 _proxyPassword_ - Password to use for the proxy server. Requires `proxyLogin`.
 
-_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server, overriding any values that may have been set in `proxyLogin` and `proxyPassword`.
-Only applies if `noProxy` is not set to `true`.
+_useDefaultCredentialsForProxy_ - Uses the default credentials (`System.Net.CredentialCache.DefaultCredentials`) for the proxy server.
 
 ### Retry Policy
 

src/NLog.Extensions.AzureEventHub/EventHubTarget.cs

@@ -7,6 +7,7 @@
 using Azure.Messaging.EventHubs;
 using NLog.Common;
 using NLog.Config;
+using NLog.Extensions.AzureBlobStorage;
 using NLog.Extensions.AzureStorage;
 using NLog.Layouts;
 
@@ -161,13 +162,50 @@ public class EventHubTarget : AsyncTaskTarget
         /// <summary>
         /// The proxy to use for communication over web sockets.
         /// </summary>
-        public Layout WebSocketProxyAddress { get; set; }
+        [Obsolete("Replaced by ProxyAddress to align with other NLog Targets for Azure")]
+        public Layout WebSocketProxyAddress { get => ProxyAddress; set => ProxyAddress = value; }
 
         /// <summary>
         /// Custom endpoint address that can be used when establishing the connection.
         /// </summary>
         public Layout CustomEndpointAddress { get; set; }
 
+        /// <summary>
+        /// A unique name used to identify the producer. If <c>null</c> or empty, a GUID will be used as the identifier.
+        /// </summary>
+        public Layout EventProducerIdentifier { get; set; }
+
+        /// <summary>
+        /// Bypasses any system proxy and proxy in <see cref="ProxyAddress"/> when set to <see langword="true"/>.
+        /// Overrides <see cref="ProxyAddress"/>.
+        /// </summary>
+        /// <remarks>Only applies when <see cref="UseWebSockets"/> = <see langword="true"/></remarks>
+        public bool NoProxy { get; set; }
+
+        /// <summary>
+        /// Address of the proxy server to use (e.g. http://proxyserver:8080).
+        /// </summary>
+        /// <remarks>Only applies when <see cref="UseWebSockets"/> = <see langword="true"/></remarks>
+        public Layout ProxyAddress { get; set; }
+
+        /// <summary>
+        /// Login to use for the proxy server. Requires <see cref="ProxyPassword"/>.
+        /// </summary>
+        /// <remarks>Only applies when <see cref="UseWebSockets"/> = <see langword="true"/></remarks>
+        public Layout ProxyLogin { get; set; }
+
+        /// <summary>
+        /// Password to use for the proxy server. Requires <see cref="ProxyLogin"/>.
+        /// </summary>
+        /// <remarks>Only applies when <see cref="UseWebSockets"/> = <see langword="true"/></remarks>
+        public Layout ProxyPassword { get; set; }
+
+        /// <summary>
+        /// Uses the default credentials (<see cref="System.Net.CredentialCache.DefaultCredentials"/>) for the proxy server, overriding any values that may have been set in <see cref="ProxyLogin"/> and <see cref="ProxyPassword"/>.
+        /// </summary>
+        /// <remarks>Only applies when <see cref="UseWebSockets"/> = <see langword="true"/></remarks>
+        public bool UseDefaultCredentialsForProxy { get; set; }
+
         /// <summary>
         /// Gets a list of user properties (aka custom properties) to add to the AMQP message
         /// </summary>
@@ -225,6 +263,7 @@ protected override void InitializeTarget()
             string useWebSockets = string.Empty;
             string webSocketProxyAddress = string.Empty;
             string customEndPointAddress = string.Empty;
+            string eventProducerIdentifier = string.Empty;
 
             var defaultLogEvent = LogEventInfo.CreateNullEvent();
 
@@ -245,15 +284,26 @@ protected override void InitializeTarget()
                     clientAuthSecret = ClientAuthSecret?.Render(defaultLogEvent);
                 }
 
+                eventProducerIdentifier = EventProducerIdentifier?.Render(defaultLogEvent) ?? string.Empty;
+
                 useWebSockets = UseWebSockets?.Render(defaultLogEvent) ?? string.Empty;
                 if (!string.IsNullOrEmpty(useWebSockets) && (string.Equals(useWebSockets.Trim(), bool.TrueString, StringComparison.OrdinalIgnoreCase) || string.Equals(useWebSockets.Trim(), "1", StringComparison.OrdinalIgnoreCase)))
                 {
                     useWebSockets = bool.TrueString;
                 }
                 customEndPointAddress = CustomEndpointAddress?.Render(defaultLogEvent) ?? string.Empty;
-                webSocketProxyAddress = WebSocketProxyAddress?.Render(defaultLogEvent) ?? string.Empty;
 
-                _eventHubService.Connect(connectionString, eventHubName, serviceUri, tenantIdentity, managedIdentityResourceId, managedIdentityClientId, sharedAccessSignature, storageAccountName, storageAccountAccessKey, clientAuthId, clientAuthSecret, bool.TrueString == useWebSockets, webSocketProxyAddress, customEndPointAddress);
+                var proxySettings = new ProxySettings
+                {
+                    NoProxy = NoProxy,
+                    UseDefaultCredentials = UseDefaultCredentialsForProxy,
+                    Address = ProxyAddress?.Render(defaultLogEvent),
+                    Login = ProxyLogin?.Render(defaultLogEvent),
+                    Password = ProxyPassword?.Render(defaultLogEvent)
+                };
+                proxySettings = proxySettings.RequiresManualProxyConfiguration ? proxySettings : null;
+
+                _eventHubService.Connect(connectionString, eventHubName, serviceUri, tenantIdentity, managedIdentityResourceId, managedIdentityClientId, sharedAccessSignature, storageAccountName, storageAccountAccessKey, clientAuthId, clientAuthSecret, eventProducerIdentifier, bool.TrueString == useWebSockets, customEndPointAddress, proxySettings);
                 InternalLogger.Debug("AzureEventHubTarget(Name={0}): Initialized", Name);
             }
             catch (Exception ex)
@@ -556,17 +606,21 @@ private sealed class EventHubService : IEventHubService
 
             public string EventHubName { get; private set; }
 
-            public void Connect(string connectionString, string eventHubName, string serviceUri, string tenantIdentity, string managedIdentityResourceId, string managedIdentityClientId, string sharedAccessSignature, string storageAccountName, string storageAccountAccessKey, string clientAuthId, string clientAuthSecret, bool useWebSockets, string webSocketsProxyAddress, string endPointAddress)
+            public void Connect(string connectionString, string eventHubName, string serviceUri, string tenantIdentity, string managedIdentityResourceId, string managedIdentityClientId, string sharedAccessSignature, string storageAccountName, string storageAccountAccessKey, string clientAuthId, string clientAuthSecret, string eventProducerIdentifier, bool useWebSockets, string endPointAddress, ProxySettings proxySettings)
             {
                 EventHubName = eventHubName;
 
                 Azure.Messaging.EventHubs.Producer.EventHubProducerClientOptions options = default;
-                if (useWebSockets || !string.IsNullOrEmpty(webSocketsProxyAddress) || !string.IsNullOrEmpty(endPointAddress))
+                if (useWebSockets || !string.IsNullOrEmpty(endPointAddress) || !string.IsNullOrEmpty(eventProducerIdentifier))
                 {
                     options = new Azure.Messaging.EventHubs.Producer.EventHubProducerClientOptions();
                     options.ConnectionOptions.TransportType = useWebSockets ? EventHubsTransportType.AmqpWebSockets : options.ConnectionOptions.TransportType;
-                    options.ConnectionOptions.Proxy = !string.IsNullOrEmpty(webSocketsProxyAddress) ? new System.Net.WebProxy(webSocketsProxyAddress, true) : options.ConnectionOptions.Proxy;
-                    options.ConnectionOptions.CustomEndpointAddress = !string.IsNullOrEmpty(endPointAddress) ? new Uri(endPointAddress) : options.ConnectionOptions.CustomEndpointAddress;
+                    if (useWebSockets && proxySettings != null)
+                        options.ConnectionOptions.Proxy = proxySettings.CreateWebProxy(options.ConnectionOptions.Proxy);
+                    if (!string.IsNullOrEmpty(endPointAddress))
+                        options.ConnectionOptions.CustomEndpointAddress = new Uri(endPointAddress);
+                    if (!string.IsNullOrEmpty(eventProducerIdentifier))
+                        options.Identifier = eventProducerIdentifier;
                 }
 
                 if (string.IsNullOrWhiteSpace(serviceUri))

src/NLog.Extensions.AzureEventHub/IEventHubService.cs

@@ -2,13 +2,14 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Messaging.EventHubs;
+using NLog.Extensions.AzureBlobStorage;
 
 namespace NLog.Extensions.AzureStorage
 {
     internal interface IEventHubService
     {
         string EventHubName { get; }
-        void Connect(string connectionString, string eventHubName, string serviceUri, string tenantIdentity, string managedIdentityResourceId, string managedIdentityClientId, string sharedAccessSignature, string storageAccountName, string storageAccountAccessKey, string clientAuthId, string clientAuthSecret, bool useWebSockets, string webSocketsProxyAddress, string endPointAddress);
+        void Connect(string connectionString, string eventHubName, string serviceUri, string tenantIdentity, string managedIdentityResourceId, string managedIdentityClientId, string sharedAccessSignature, string storageAccountName, string storageAccountAccessKey, string clientAuthId, string clientAuthSecret, string eventProducerIdentifier, bool useWebSockets, string endPointAddress, ProxySettings proxySettings);
         Task CloseAsync();
         Task SendAsync(IEnumerable<EventData> eventDataBatch, string partitionKey, CancellationToken cancellationToken);
     }