Assign WorkloadIdentityClientId from ClientIdentity (#156)

* Assign WorkloadIdentityClientId from ClientIdentity

* Updated unit-test dependencies

* AzureDataTables - NLogEntity - Skip AggregateException.Flatten when not needed

* AzureDataTables  - Override Layout default value to just ${message}

* AzureDataTables - Added documentation about Column Value size limits

* Updated unit-test dependencies

* Explicit specify RetryDelayMilliseconds to avoid default of 500ms

* Removed NLog.Extensions.AzureAccessToken from solution-file

* BlobStorageTarget - Handle invalid empty BlobName + ContainerName

* DataTablesTarget - Handle invalid empty TableName

* QueueStorageTarget - Handle invalid empty QueueName

* Updated to Azure.Messaging.EventGrid ver. 4.24.1

* Removed User delegation SAS since it confuses

* Microsoft.NET.Test.Sdk ver. 17.13.0

Author: snakefoot

README.md

@@ -5,11 +5,11 @@
 | Package Name                          | NuGet                 | Description | Documentation |
 | ------------------------------------- | :-------------------: | ----------- | ------------- |
 | **NLog.Extensions.AzureBlobStorage**  | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureBlobStorage.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureBlobStorage/) | Azure Blob Storage | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureBlobStorage/README.md) | 
-| **NLog.Extensions.AzureDataTables**  | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureDataTables.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureDataTables/) | Azure Table Storage or Azure CosmosDb Tables | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureDataTables/README.md) | 
+| **NLog.Extensions.AzureDataTables**   | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureDataTables.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureDataTables/) | Azure Table Storage or Azure CosmosDb Tables | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureDataTables/README.md) | 
 | **NLog.Extensions.AzureEventHub**     | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureEventHub.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureEventHub/) | Azure EventHubs | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureEventHub/README.md) | 
-| **NLog.Extensions.AzureEventGrid**     | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureEventGrid.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureEventGrid/) | Azure Event Grid | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureEventGrid/README.md) | 
+| **NLog.Extensions.AzureEventGrid**    | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureEventGrid.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureEventGrid/) | Azure Event Grid | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureEventGrid/README.md) | 
 | **NLog.Extensions.AzureQueueStorage** | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureQueueStorage.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureQueueStorage/) | Azure Queue Storage | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureQueueStorage/README.md) | 
-| **NLog.Extensions.AzureServiceBus** | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureServiceBus.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureServiceBus/) | Azure Service Bus | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureServiceBus/README.md) | 
+| **NLog.Extensions.AzureServiceBus**   | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureServiceBus.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureServiceBus/) | Azure Service Bus | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureServiceBus/README.md) | 
 | **NLog.Extensions.AzureAccessToken**  | [![NuGet](https://img.shields.io/nuget/v/NLog.Extensions.AzureAccessToken.svg)](https://www.nuget.org/packages/NLog.Extensions.AzureAccessToken/) | Azure App Authentication Access Token for Managed Identity | [![](https://img.shields.io/badge/Readme-Docs-blue)](src/NLog.Extensions.AzureAccessToken/README.md) | 
 
 Initially all NLog targets was bundled into a single nuget-package called [NLog.Extensions.AzureStorage](https://www.nuget.org/packages/NLog.Extensions.AzureStorage/).
@@ -27,7 +27,6 @@ and so [NLog.Extensions.AzureCosmosTable](https://www.nuget.org/packages/NLog.Ex
   <add assembly="NLog.Extensions.AzureEventHub" /> 
   <add assembly="NLog.Extensions.AzureEventGrid" /> 
   <add assembly="NLog.Extensions.AzureServiceBus" /> 
-  <add assembly="NLog.Extensions.AzureAccessToken" /> 
 </extensions>
 
 <targets async="true">

appveyor.yml

@@ -10,7 +10,6 @@ build_script:
 - cmd: msbuild /t:Pack src\NLog.Extensions.AzureEventGrid /p:Configuration=Release /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg /p:ContinuousIntegrationBuild=true /p:EmbedUntrackedSources=true /p:PublishRepositoryUrl=true /verbosity:minimal
 - cmd: msbuild /t:Pack src\NLog.Extensions.AzureEventHub /p:Configuration=Release /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg /p:ContinuousIntegrationBuild=true /p:EmbedUntrackedSources=true /p:PublishRepositoryUrl=true /verbosity:minimal
 - cmd: msbuild /t:Pack src\NLog.Extensions.AzureServiceBus /p:Configuration=Release /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg /p:ContinuousIntegrationBuild=true /p:EmbedUntrackedSources=true /p:PublishRepositoryUrl=true /verbosity:minimal
-- cmd: msbuild /t:Pack src\NLog.Extensions.AzureAccessToken /p:Configuration=Release /p:IncludeSymbols=true /p:SymbolPackageFormat=snupkg /p:ContinuousIntegrationBuild=true /p:EmbedUntrackedSources=true /p:PublishRepositoryUrl=true /verbosity:minimal
 
 test_script:
 - cmd: dotnet test test\NLog.Extensions.AzureBlobStorage.Tests /p:Configuration=Release --verbosity minimal
@@ -19,7 +18,6 @@ test_script:
 - cmd: dotnet test test\NLog.Extensions.AzureEventGrid.Tests /p:Configuration=Release --verbosity minimal
 - cmd: dotnet test test\NLog.Extensions.AzureEventHub.Tests /p:Configuration=Release --verbosity minimal
 - cmd: dotnet test test\NLog.Extensions.AzureServiceBus.Tests /p:Configuration=Release --verbosity minimal
-- cmd: dotnet test test\NLog.Extensions.AzureAccessToken.Tests /p:Configuration=Release --verbosity minimal
 
 artifacts:
 - path: '**\NLog.Extensions.Azure*.nupkg'

src/NLog.Extensions.AzureBlobStorage/BlobStorageTarget.cs

@@ -49,7 +49,7 @@ public sealed class BlobStorageTarget : AsyncTaskTarget
         public Layout ResourceIdentity { get; set; }
 
         /// <summary>
-        /// Alternative to ConnectionString, when using <see cref="ServiceUri"/> with ManagedIdentityClientId
+        /// Alternative to ConnectionString, when using <see cref="ServiceUri"/> with ManagedIdentityClientId / WorkloadIdentityClientId
         /// </summary>
         public Layout ClientIdentity { get; set; }
 
@@ -95,6 +95,7 @@ internal BlobStorageTarget(ICloudBlobService cloudBlobService)
         {
             TaskDelayMilliseconds = 200;
             BatchSize = 100;
+            RetryDelayMilliseconds = 100;
 
             BlobMetadata = new List<TargetPropertyWithContext>();
             BlobTags = new List<TargetPropertyWithContext>();
@@ -405,7 +406,7 @@ public Task AppendFromByteArrayAsync(string containerName, string blobName, stri
 
                 var blob = _appendBlob;
                 var container = _container;
-                if (containerName == null || container?.Name != containerName || blobName == null || blob?.Name != blobName)
+                if (string.IsNullOrEmpty(containerName) || container?.Name != containerName || string.IsNullOrEmpty(blobName) || blob?.Name != blobName)
                 {
                     return InitializeAndCacheBlobAsync(containerName, blobName, contentType, cancellationToken).ContinueWith(async (t, s) => await t.Result.AppendBlockAsync((System.IO.Stream)s, null).ConfigureAwait(false), stream, cancellationToken);
                 }
@@ -420,7 +421,7 @@ async Task<AppendBlobClient> InitializeAndCacheBlobAsync(string containerName, s
                 try
                 {
                     var container = _container;
-                    if (containerName == null || container?.Name != containerName)
+                    if (string.IsNullOrEmpty(containerName) || container?.Name != containerName)
                     {
                         container = await InitializeContainer(containerName, cancellationToken).ConfigureAwait(false);
                         if (container != null)
@@ -431,7 +432,7 @@ async Task<AppendBlobClient> InitializeAndCacheBlobAsync(string containerName, s
                     }
 
                     var blob = _appendBlob;
-                    if (blobName == null || blob?.Name != blobName)
+                    if (string.IsNullOrEmpty(blobName) || blob?.Name != blobName)
                     {
                         blob = await InitializeBlob(blobName, container, contentType, cancellationToken).ConfigureAwait(false);
                         if (blob != null && ReferenceEquals(container, _container))

src/NLog.Extensions.AzureBlobStorage/NLog.Extensions.AzureBlobStorage.csproj

@@ -18,8 +18,7 @@
     <RepositoryUrl>https://github.com/JDetmar/NLog.Extensions.AzureStorage.git</RepositoryUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageReleaseNotes>
-- Updated dependency Azure.Identity v1.11.4 because of security issues.
-- Updated dependency Azure.Storage.Blobs v12.19.1
+- Assign WorkloadIdentityClientId from ClientIdentity
 
       Docs: https://github.com/JDetmar/NLog.Extensions.AzureStorage/blob/master/src/NLog.Extensions.AzureBlobStorage/README.md
     </PackageReleaseNotes>

src/NLog.Extensions.AzureBlobStorage/README.md

@@ -31,21 +31,21 @@ _name_ - Name of the target.
 
 _layout_ - Text to be rendered. [Layout](https://github.com/NLog/NLog/wiki/Layouts) Required. 
 
-_blobName_ - BlobName. [Layout](https://github.com/NLog/NLog/wiki/Layouts)  
+_blobName_ - BlobName. [Layout](https://github.com/NLog/NLog/wiki/Layouts) Required. 
 
-_container_ - Azure blob container name. [Layout](https://github.com/NLog/NLog/wiki/Layouts)
+_container_ - Azure blob container name. [Layout](https://github.com/NLog/NLog/wiki/Layouts) Required.
 
 _contentType_ - Azure blob ContentType (Default = text/plain)
 
 _connectionString_ - Azure storage connection string. Ex. `UseDevelopmentStorage=true;`
 
-_serviceUri_ - Alternative to ConnectionString, where Managed Identiy is acquired from DefaultAzureCredential for User delegation SAS.
+_serviceUri_ - Alternative to ConnectionString, where Managed Identiy is acquired from DefaultAzureCredential.
 
-_tenantIdentity_ - Alternative to ConnectionString. Used together with ServiceUri. Input for DefaultAzureCredential.
+_clientIdentity_ - Alternative to ConnectionString. Used together with ServiceUri. Input for DefaultAzureCredential as ManagedIdentityClientId.
 
 _resourceIdentity_ - Alternative to ConnectionString. Used together with ServiceUri. Input for DefaultAzureCredential as ManagedIdentityResourceId.
 
-_clientIdentity_ - Alternative to ConnectionString. Used together with ServiceUri. Input for DefaultAzureCredential as ManagedIdentityClientId.
+_tenantIdentity_ - Alternative to ConnectionString. Used together with ServiceUri. Input for DefaultAzureCredential.
 
 _sharedAccessSignature_ - Alternative to ConnectionString. Used together with ServiceUri. Input for AzureSasCredential
 
@@ -81,6 +81,13 @@ Azure.RequestFailedException: This feature is not currently supported by the Sto
 
 Instead one can try an alternative Azure Storage Emulator like [Azurite](https://github.com/azure/azurite)
 
+## Azure Identity Environment
+When using `ServiceUri` (Instead of ConnectionString), then `DefaultAzureCredential` is used for Azure Identity which supports environment variables:
+- `AZURE_CLIENT_ID` - For ManagedIdentityClientId / WorkloadIdentityClientId
+- `AZURE_TENANT_ID` - For TenantId
+
+See also: [Set up Your Environment for Authentication](https://github.com/Azure/azure-sdk-for-go/wiki/Set-up-Your-Environment-for-Authentication)
+
 ## Azure ConnectionString
 
 NLog Layout makes it possible to retrieve settings from [many locations](https://nlog-project.org/config/?tab=layout-renderers).

src/NLog.Extensions.AzureDataTables/DataTablesTarget.cs

@@ -17,6 +17,8 @@ namespace NLog.Targets
     [Target("AzureDataTables")]
     public sealed class DataTablesTarget : AsyncTaskTarget
     {
+        internal const int ColumnStringValueMaxSize = 32768;
+
         private readonly ICloudTableService _cloudTableService;
         private string _machineName;
         private readonly AzureStorageNameCache _containerNameCache = new AzureStorageNameCache();
@@ -79,7 +81,7 @@ public override int GetHashCode()
         public Layout ResourceIdentity { get; set; }
 
         /// <summary>
-        /// Alternative to ConnectionString, when using <see cref="ServiceUri"/> with ManagedIdentityClientId
+        /// Alternative to ConnectionString, when using <see cref="ServiceUri"/> with ManagedIdentityClientId / WorkloadIdentityClientId
         /// </summary>
         public Layout ClientIdentity { get; set; }
 
@@ -117,12 +119,14 @@ public override int GetHashCode()
         public DataTablesTarget()
             :this(new CloudTableService())
         {
+            Layout = "${message}";  // Override default Layout, since splitting into columns
         }
 
         internal DataTablesTarget(ICloudTableService cloudTableService)
         {
             TaskDelayMilliseconds = 200;
             BatchSize = 100;
+            RetryDelayMilliseconds = 100;
 
             RowKey = Layout.FromMethod(l => string.Concat((DateTime.MaxValue.Ticks - l.TimeStamp.Ticks).ToString("d19"), "__", Guid.NewGuid().ToString()), LayoutRenderOptions.ThreadAgnostic);
 
@@ -298,18 +302,32 @@ private ITableEntity CreateTableEntity(LogEventInfo logEvent, string partitionKe
                     if (string.IsNullOrEmpty(contextproperty.Name))
                         continue;
 
-                    var propertyValue = contextproperty.Layout != null ? RenderLogEvent(contextproperty.Layout, logEvent) : string.Empty;
+                    var propertyValue = RenderLogEvent(contextproperty.Layout, logEvent) ?? string.Empty;
                     if (logTimeStampOverridden && i == 0 && string.IsNullOrEmpty(propertyValue))
                         continue;
 
+                    if (!contextproperty.IncludeEmptyValue && string.IsNullOrEmpty(propertyValue))
+                        continue;
+
+                    if (propertyValue.Length >= ColumnStringValueMaxSize)
+                    {
+                        InternalLogger.Debug("AzureDataTablesTarget(Name={0}): Truncating value from column '{1}', because string-length above 32K", Name, contextproperty.Name);
+                        propertyValue = propertyValue.Substring(0, ColumnStringValueMaxSize - 1);
+                    }
+
                     entity.Add(contextproperty.Name, propertyValue);
                 }
 
                 return entity;
             }
             else
             {
-                var layoutMessage = RenderLogEvent(Layout, logEvent);
+                var layoutMessage = RenderLogEvent(Layout, logEvent) ?? string.Empty;
+                if (layoutMessage.Length >= ColumnStringValueMaxSize)
+                {
+                    layoutMessage = layoutMessage.Substring(0, ColumnStringValueMaxSize - 1);
+                    InternalLogger.Debug("AzureDataTablesTarget(Name={0}): Truncating value from Layout, because string-length above 32K", Name);
+                }
                 return new NLogEntity(logEvent, layoutMessage, _machineName, partitionKey, rowKey, LogTimeStampFormat);
             }
         }
@@ -390,7 +408,7 @@ public void Connect(string connectionString, string serviceUri, string tenantIde
             public Task SubmitTransactionAsync(string tableName, IEnumerable<TableTransactionAction> tableTransaction, CancellationToken cancellationToken)
             {
                 var table = _table;
-                if (tableName == null || table?.Name != tableName)
+                if (string.IsNullOrEmpty(tableName) || table?.Name != tableName)
                 {
                     return InitializeAndCacheTableAsync(tableName, cancellationToken).ContinueWith(async (t, operation) => await t.Result.SubmitTransactionAsync((IEnumerable<TableTransactionAction>)operation).ConfigureAwait(false), tableTransaction, cancellationToken);
                 }

src/NLog.Extensions.AzureDataTables/NLog.Extensions.AzureDataTables.csproj

@@ -18,8 +18,10 @@
     <RepositoryUrl>https://github.com/JDetmar/NLog.Extensions.AzureStorage.git</RepositoryUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageReleaseNotes>
-- Updated dependency Azure.Identity v1.11.4 because of security issues.
-- Updated dependency Azure.Data.Tables v12.8.3
+- Assign WorkloadIdentityClientId from ClientIdentity
+- Changed Layout default-value to ${message} since LogEvent is split into columns
+- Added automatic truncate when column-values has string-length above 32K
+- Skips adding column values when empty string and configured IncludeEmptyValue = false
 
 Docs: https://github.com/JDetmar/NLog.Extensions.AzureStorage/blob/master/src/NLog.Extensions.AzureDataTables/README.md
     </PackageReleaseNotes>

src/NLog.Extensions.AzureDataTables/NLogEntity.cs

@@ -22,10 +22,10 @@ public class NLogEntity: ITableEntity
 
         public NLogEntity(LogEventInfo logEvent, string layoutMessage, string machineName, string partitionKey, string rowKey, string logTimeStampFormat)
         {
-            FullMessage = layoutMessage;
+            FullMessage = TruncateWhenTooBig(layoutMessage);
             Level = logEvent.Level.Name;
             LoggerName = logEvent.LoggerName;
-            Message = logEvent.Message;
+            Message = TruncateWhenTooBig(logEvent.Message);
             LogTimeStamp = logEvent.TimeStamp.ToString(logTimeStampFormat);
             MachineName = machineName;
             if(logEvent.Exception != null)
@@ -34,28 +34,44 @@ public NLogEntity(LogEventInfo logEvent, string layoutMessage, string machineNam
                 var innerException = exception.InnerException;
                 if (exception is AggregateException aggregateException)
                 {
-                    var innerExceptions = aggregateException.Flatten();
-                    if (innerExceptions.InnerExceptions?.Count == 1)
+                    if (aggregateException.InnerExceptions?.Count == 1 && !(aggregateException.InnerExceptions[0] is AggregateException))
                     {
-                        exception = innerExceptions.InnerExceptions[0];
-                        innerException = null;
+                        exception = aggregateException.InnerExceptions[0];
+                        innerException = exception.InnerException;
                     }
                     else
                     {
-                        innerException = innerExceptions;
+                        var flatten = aggregateException.Flatten();
+                        if (flatten.InnerExceptions?.Count == 1)
+                        {
+                            exception = flatten.InnerExceptions[0];
+                            innerException = exception.InnerException;
+                        }
+                        else
+                        {
+                            innerException = flatten;
+                        }
                     }
                 }
 
                 Exception = string.Concat(exception.Message, " - ", exception.GetType().ToString());
-                StackTrace = exception.StackTrace;
+                StackTrace = TruncateWhenTooBig(exception.StackTrace);
                 if (innerException != null)
                 {
-                    InnerException = innerException.ToString();
+                    var innerExceptionText = innerException.ToString();
+                    InnerException = TruncateWhenTooBig(innerExceptionText);
                 }
             }
             RowKey = rowKey;
             PartitionKey = partitionKey;
         }
+
+        private static string TruncateWhenTooBig(string stringValue)
+        {
+             return stringValue?.Length >= Targets.DataTablesTarget.ColumnStringValueMaxSize ?
+                stringValue.Substring(0, Targets.DataTablesTarget.ColumnStringValueMaxSize - 1) : stringValue;
+        }
+
         public NLogEntity() { }
     }
 }