Updated logging to push exceptions to Application Insights exceptions itemType. This means with sampling turned on requests and exceptions always get logged.

Author: Niall Langley

DataPipelineTools.Functions/DataLake/DataLakeConfigFactory.cs

@@ -66,7 +66,8 @@ public DataLakeGetItemsConfig GetItemsConfig (HttpRequest req)
             int.TryParse(req.Query[LimitParam] != StringValues.Empty ? (string)req.Query[LimitParam] : data?.orderByDesc, out limit);
 
             string directory = req.Query[DirectoryParam] != StringValues.Empty || data?.directory  == null ? (string)req.Query[DirectoryParam] : data?.directory;
-            config.Directory = directory?.TrimStart('/');
+            config.Directory = string.IsNullOrWhiteSpace(directory?.TrimStart('/')) ? "/" : directory?.TrimStart('/');
+
             config.IgnoreDirectoryCase = ignoreDirectoryCase;
             config.Recursive = recursive;
             config.OrderByColumn = req.Query[OrderByColumnParam] != StringValues.Empty ? (string)req.Query[OrderByColumnParam] : data?.orderBy;

DataPipelineTools.Functions/DataLake/DataLakeFunctions.cs

@@ -43,8 +43,8 @@ public async Task<IActionResult> DataLakeGetItems(
                 if (string.IsNullOrWhiteSpace(dataLakeConfig.AccountUri))
                     throw new ArgumentException($"Parameter 'accountUri' with value '{dataLakeConfig.AccountUri}' not found. Check the URI is correct.");
 
-                if (getItemsConfig.Directory == null)
-                    throw new ArgumentException($"Parameter 'directory' is required.");
+                //if (getItemsConfig.Directory == null)
+                //    throw new ArgumentException($"Parameter 'directory' is required.");
 
                 var client = _clientFactory.GetDataLakeClient(dataLakeConfig);
                 var controller = _serviceFactory.CreateDataLakeService(client);
@@ -63,7 +63,7 @@ public async Task<IActionResult> DataLakeGetItems(
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex.ToString());
+                _logger.LogError(ex, ex.Message); // The simple message goes in the trace, but the full exception details are in the exception logging in Application Insights
                 return new BadRequestObjectResult("An error occurred, see the Azure Function logs for more details");
             }
         }
@@ -111,7 +111,7 @@ public async Task<IActionResult> DataLakeCheckPathCase(
             }
             catch (Exception ex)
             {
-                _logger.LogError(ex.ToString());
+                _logger.LogError(ex, ex.Message); // The simple message goes in the trace, but the full exception details are in the exception logging in Application Insights
                 return new BadRequestObjectResult("An error occurred, see the Azure Function logs for more details");
             }
         }

DataPipelineTools.Functions/host.json

@@ -5,8 +5,8 @@
     "logLevel": {
       "default": "Error",
       "Host.Results": "Information",
-      "Function.DataLakeGetItems": "Information",
-      "Function.DataLakeCheckPathCase": "Information"
+      "Function": "Information",
+      "SqlCollaborative.Azure.DataPipelineTools": "Information"
     },
     "applicationInsights": {
       "samplingSettings": {

DataPipelineTools/DataLake/DataLakeService.cs

@@ -106,28 +106,12 @@ public async Task<JObject> GetItemsAsync(DataLakeConfig dataLakeConfig, DataLake
         {
             // Check the directory exists. If multiple directories match (ie different casing), it will throw an error, as we don't know
             // which one we wanted the files from.
-            /* TODO: When the path does not exist, this is throwing an exception:
-                       Azure.RequestFailedException: Service request failed.
-                       Status: 400 (The requested URI does not represent any resource on the server.)
-                       ErrorCode: InvalidUri
-
-                           at Azure.Storage.Blobs.BlobRestClient.Blob.GetPropertiesAsync_CreateResponse(ClientDiagnostics clientDiagnostics, Response response)
-                           at Azure.Storage.Blobs.BlobRestClient.Blob.GetPropertiesAsync(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, Uri resourceUri, String version, String snapshot, String versionId, Nullable`1 timeout, String leaseId, String encryptionKey, String encryptionKeySha256, Nullable`1 encryptionAlgorithm, Nullable`1 ifModifiedSince, Nullable`1 ifUnmodifiedSince, Nullable`1 ifMatch, Nullable`1 ifNoneMatch, String ifTags, String requestId, Boolean async, String operationName, CancellationToken cancellationToken)
-                           at Azure.Storage.Blobs.Specialized.BlobBaseClient.GetPropertiesInternal(BlobRequestConditions conditions, Boolean async, CancellationToken cancellationToken, String operationName)
-                           at Azure.Storage.Blobs.Specialized.BlobBaseClient.ExistsInternal(Boolean async, CancellationToken cancellationToken)
-                           at Azure.Storage.Blobs.Specialized.BlobBaseClient.Exists(CancellationToken cancellationToken)
-                           at Azure.Storage.Files.DataLake.DataLakePathClient.Exists(CancellationToken cancellationToken)
-                           at SqlCollaborative.Azure.DataPipelineTools.DataLake.DataLakeService.GetItemsAsync(DataLakeConfig dataLakeConfig, DataLakeGetItemsConfig getItemsConfig) 
-                             in /home/runner/work/AzureDataPipelineTools/AzureDataPipelineTools/DataPipelineTools/DataLake/DataLakeService.cs:line 109
-                           at SqlCollaborative.Azure.DataPipelineTools.Functions.DataLake.DataLakeFunctions.DataLakeGetItems(HttpRequest req) 
-                             in /home/runner/work/AzureDataPipelineTools/AzureDataPipelineTools/DataPipelineTools.Functions/DataLake/DataLakeFunctions.cs:line 54
-            */
             var directory = getItemsConfig.IgnoreDirectoryCase ?
                                 await CheckPathAsync(getItemsConfig.Directory, true) :
                                 getItemsConfig.Directory;
 
-            
-            if (!_client.GetDirectoryClient(directory).Exists())
+            // Only check the path if it is not the root. Checking is the root exists throws, and if the container is valid the root will always be valid
+            if (directory != "/" && !await _client.GetDirectoryClient(directory).ExistsAsync() )
                 throw new DirectoryNotFoundException($"Directory '{directory} could not be found'");
 
             var paths = _client