update character validation for InvokeCompatabilityCheckCmdlet

Author: jlindamood

src/StorageSync/StorageSync.Test/UnitTests/FilenamesCharactersValidationTest.cs

@@ -45,7 +45,6 @@ public void ItReturnsErrorOnDirectoryWithInvalidCodePoint()
                 0x7A
             };
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(codePointBlacklist);
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(this._configuration.WhitelistOfCodePointRanges);
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 
             var directoryInfoMockFactory = new Moq.Mock<IDirectoryInfo>();
@@ -69,7 +68,6 @@ public void ItReturnsErrorOnFileWithInvalidCodePoint()
                 0x7A
             };
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(codePointBlacklist);
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(this._configuration.WhitelistOfCodePointRanges);
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 
             var fileInfoMockFactory = new Moq.Mock<IFileInfo>();
@@ -98,7 +96,6 @@ public void ItReturnsErrorWhenCodePointIsInBoundsOfRange()
                     End = 0x10FFFF
                 },
             };
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(whitelist);
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(new List<int>());
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 
@@ -131,7 +128,6 @@ public void ItReturnsErrorWhenCodePointIsInMiddleOfRange()
                 },
             };
 
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(whitelist);
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(new List<int>());
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 

src/StorageSync/StorageSync/Config.json

@@ -49,97 +49,6 @@
     62,
     63,
     92,
-    124,
-    127
-  ],
-  "WhitelistOfCodePointRanges": [
-    {
-      "Start": 0,
-      "End": 127
-    },
-    {
-      "Start": 160,
-      "End": 55295
-    },
-    {
-      "Start": 57344,
-      "End": 63743
-    },
-    {
-      "Start": 63744,
-      "End": 64975
-    },
-    {
-      "Start": 65008,
-      "End": 65519
-    },
-    {
-      "Start": 65536,
-      "End": 131069
-    },
-    {
-      "Start": 131072,
-      "End": 196605
-    },
-    {
-      "Start": 196608,
-      "End": 262141
-    },
-    {
-      "Start": 262144,
-      "End": 327677
-    },
-    {
-      "Start": 327680,
-      "End": 393213
-    },
-    {
-      "Start": 393216,
-      "End": 458749
-    },
-    {
-      "Start": 458752,
-      "End": 524285
-    },
-    {
-      "Start": 524288,
-      "End": 589821
-    },
-    {
-      "Start": 589824,
-      "End": 655357
-    },
-    {
-      "Start": 655360,
-      "End": 720893
-    },
-    {
-      "Start": 720896,
-      "End": 786429
-    },
-    {
-      "Start": 786432,
-      "End": 851965
-    },
-    {
-      "Start": 851968,
-      "End": 917501
-    },
-    {
-      "Start": 917504,
-      "End": 921599
-    },
-    {
-      "Start": 921600,
-      "End": 983037
-    },
-    {
-      "Start": 983040,
-      "End": 1048573
-    },
-    {
-      "Start": 1048576,
-      "End": 1114109
-    }
+    124
   ]
 }

src/StorageSync/StorageSync/Configuration.cs

@@ -54,10 +54,6 @@ public class ValidationsConfiguration
             /// </summary>
             [DataMember] public List<string> InvalidFilenames;
             /// <summary>
-            /// The whitelist of code point ranges
-            /// </summary>
-            [DataMember] public List<CodePointRange> WhitelistOfCodePointRanges;
-            /// <summary>
             /// The blacklist of code points
             /// </summary>
             [DataMember] public List<int> BlacklistOfCodePoints;
@@ -198,15 +194,6 @@ public IEnumerable<string> InvalidFileNames()
             return _validationsConfiguration.InvalidFilenames;
         }
 
-        /// <summary>
-        /// Whitelists the of code point ranges.
-        /// </summary>
-        /// <returns>IEnumerable&lt;Configuration.CodePointRange&gt;.</returns>
-        public IEnumerable<CodePointRange> WhitelistOfCodePointRanges()
-        {
-            return _validationsConfiguration.WhitelistOfCodePointRanges;
-        }
-
         /// <summary>
         /// Blacklists the of code points.
         /// </summary>

src/StorageSync/StorageSync/Interfaces/IConfiguration.cs

@@ -37,11 +37,6 @@ public interface IConfiguration
         /// <returns>IEnumerable&lt;System.String&gt;.</returns>
         IEnumerable<string> ValidFilesystems();
         /// <summary>
-        /// Whitelists the of code point ranges.
-        /// </summary>
-        /// <returns>IEnumerable&lt;Configuration.CodePointRange&gt;.</returns>
-        IEnumerable<Configuration.CodePointRange> WhitelistOfCodePointRanges();
-        /// <summary>
         /// Blacklists the of code points.
         /// </summary>
         /// <returns>IEnumerable&lt;System.Int32&gt;.</returns>

src/StorageSync/StorageSync/Validations/NamespaceValidations/FilenamesCharactersValidation.cs

@@ -18,6 +18,7 @@ namespace Microsoft.Azure.Commands.StorageSync.Evaluation.Validations.NamespaceV
     using System;
     using System.Collections;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using System.Linq;
 
     /// <summary>
@@ -49,14 +50,13 @@ public FilenamesCharactersValidation(
                 "File/Directory names with unsupported characters",
                 ValidationType.FilenameCharacters)
         {
-            var whitelistOfCodePointRanges = configuration.WhitelistOfCodePointRanges().ToList();
             var blacklistOfCodePoints = new HashSet<int>(configuration.BlacklistOfCodePoints());
 
             _codePointBlackList = new bool[NumberOfCodePoints];
 
             for (int i = 0; i < NumberOfCodePoints; ++i)
             {
-                _codePointBlackList[i] = blacklistOfCodePoints.Contains(i) || !whitelistOfCodePointRanges.Any(range => range.Includes(i));
+                _codePointBlackList[i] = blacklistOfCodePoints.Contains(i);
             }
         }
         #endregion
@@ -98,16 +98,34 @@ protected override IValidationResult DoValidate(IDirectoryInfo directoryInfo)
         /// <param name="node">The node.</param>
         /// <param name="isDirectory">if set to <c>true</c> [is directory].</param>
         /// <returns>IValidationResult.</returns>
-        private IValidationResult ValidateInternal (INamedObjectInfo node, bool isDirectory)
+        private IValidationResult ValidateInternal(INamedObjectInfo node, bool isDirectory)
         {
             string name = node.Name;
             List<int> positions = new List<int>();
+            string message = string.Empty;
+            char prevChar = 'a';
 
             for (int i = 0, codepointIndex = 0; i < name.Length; ++i, ++codepointIndex)
             {
                 int codePoint;
+                message = string.Empty;
 
-                if (char.IsSurrogatePair(name, i))
+                if (char.IsHighSurrogate(name[i]) && name.Length == 1)
+                {
+                    codePoint = name[i];
+                    message = $"Invalid surrogate char found in the file name at location {i + 1}";
+                }
+                else if (!char.IsHighSurrogate(prevChar) && char.IsLowSurrogate(name[i]))
+                {
+                    codePoint = name[i];
+                    message = $"Found a low surrogate character without a preceding high surrogate character at location {i + 1}";
+                }
+                else if (char.IsHighSurrogate(prevChar) && !char.IsLowSurrogate(name[i]))
+                {
+                    codePoint = name[i];
+                    message = $"Invalid hight surrogate char found in the file name at location {i + 1}";
+                }
+                else if (char.IsSurrogatePair(name, i))
                 {
                     codePoint = char.ConvertToUtf32(name, i);
                     i += 1;
@@ -117,14 +135,22 @@ private IValidationResult ValidateInternal (INamedObjectInfo node, bool isDirect
                     codePoint = name[i];
                 }
 
-                if (codePoint < 0 || 
-                    codePoint >= NumberOfCodePoints || 
-                    _codePointBlackList[codePoint])
+                prevChar = name[i];
+
+                if (codePoint < 0 ||
+                    codePoint >= NumberOfCodePoints ||
+                    _codePointBlackList[codePoint] ||
+                    !string.IsNullOrEmpty(message))
                 {
                     // adding +1 so that positions are 1-based
                     // to make them more human friendly
                     positions.Add(codepointIndex + 1);
                 }
+
+                if (!string.IsNullOrEmpty(message))
+                {
+                    Trace.TraceInformation(message);
+                }
             }
 
             if (positions.Count > 0)

src/StorageSync/StorageSync/debug.Netcore.ps1

@@ -39,14 +39,12 @@ function Build-CharacterTable
     param ($Configuration)
 
     $blacklistOfCodePoints = $Configuration.BlacklistOfCodePoints
-    $whitelistOfCodePointRanges = $Configuration.WhitelistOfCodePointRanges
 
     $arraySize = 0x10FFFF + 1
     $blacklistedCodePointsTable = new-object bool[] $arraySize
     for ($i = 0; $i -lt $arraySize; $i += 1)
     {
-        $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i) -or
-                    ($whitelistOfCodePointRanges.Where({ ($_.Start -le $i) -and ($_.End -ge $i) }).Count -eq 0);
+        $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i);
     }
     return $blacklistedCodePointsTable
 }