StorageSync: Update character validation for InvokeCompatabilityCheckCmdlet (#23298)

VeryEarly · web-flow · commit ccf8cfda0de7 · 2023-11-15T14:00:54.000+08:00
diff --git a/src/StorageSync/StorageSync.Test/UnitTests/FilenamesCharactersValidationTest.cs b/src/StorageSync/StorageSync.Test/UnitTests/FilenamesCharactersValidationTest.cs
@@ -45,7 +45,6 @@ public void ItReturnsErrorOnDirectoryWithInvalidCodePoint()
                 0x7A
             };
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(codePointBlacklist);
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(this._configuration.WhitelistOfCodePointRanges);
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 
             var directoryInfoMockFactory = new Moq.Mock<IDirectoryInfo>();
@@ -69,7 +68,6 @@ public void ItReturnsErrorOnFileWithInvalidCodePoint()
                 0x7A
             };
             configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(codePointBlacklist);
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(this._configuration.WhitelistOfCodePointRanges);
             FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
 
             var fileInfoMockFactory = new Moq.Mock<IFileInfo>();
@@ -80,69 +78,6 @@ public void ItReturnsErrorOnFileWithInvalidCodePoint()
             Assert.StrictEqual<Result>(Result.Fail, validationResult.Result);
         }
 
-        /// <summary>
-        /// Defines the test method ItReturnsErrorWhenCodePointIsInBoundsOfRange.
-        /// </summary>
-        [Fact]
-        [Trait(Category.AcceptanceType, Category.CheckIn)]
-        public void ItReturnsErrorWhenCodePointIsInBoundsOfRange()
-        {
-            var configurationMockFactory = new Moq.Mock<IConfiguration>();
-            List<Configuration.CodePointRange> whitelist = new List<Configuration.CodePointRange> {
-                new Configuration.CodePointRange {
-                    Start = 0x00, 
-                    End = 0x77    // x - 1
-                },
-                new Configuration.CodePointRange {
-                    Start = 0x7B, // z + 1
-                    End = 0x10FFFF
-                },
-            };
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(whitelist);
-            configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(new List<int>());
-            FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
-
-            var fileInfoMockFactory = new Moq.Mock<IFileInfo>();
-            fileInfoMockFactory.SetupGet(fileInfo => fileInfo.Name).Returns("AAAxAAAzAAA");
-            IValidationResult validationResult = validation.Validate(fileInfoMockFactory.Object);
-
-            Assert.StrictEqual<Result>(Result.Fail, validationResult.Result);
-            Assert.True(validationResult.Positions.Count == 2, $"Unexpected number of error positions");
-            Assert.True(validationResult.Positions[0] == 4, $"Unexpected position of first error");
-            Assert.True(validationResult.Positions[1] == 8, $"Unexpected position of second error");
-        }
-
-        /// <summary>
-        /// Defines the test method ItReturnsErrorWhenCodePointIsInMiddleOfRange.
-        /// </summary>
-        [Fact]
-        [Trait(Category.AcceptanceType, Category.CheckIn)]
-        public void ItReturnsErrorWhenCodePointIsInMiddleOfRange()
-        {
-            var configurationMockFactory = new Moq.Mock<IConfiguration>();
-            List<Configuration.CodePointRange> whitelist = new List<Configuration.CodePointRange> {
-                new Configuration.CodePointRange {
-                    Start = 0x00,
-                    End = 0x77    // x - 1
-                },
-                new Configuration.CodePointRange {
-                    Start = 0x7B, // z + 1
-                    End = 0x10FFFF
-                },
-            };
-
-            configurationMockFactory.Setup(configuration => configuration.WhitelistOfCodePointRanges()).Returns(whitelist);
-            configurationMockFactory.Setup(configuration => configuration.BlacklistOfCodePoints()).Returns(new List<int>());
-            FilenamesCharactersValidation validation = new FilenamesCharactersValidation(configurationMockFactory.Object);
-
-            var fileInfoMockFactory = new Moq.Mock<IFileInfo>();
-            fileInfoMockFactory.SetupGet(fileInfo => fileInfo.Name).Returns("AAAyAAA");
-            IValidationResult validationResult = validation.Validate(fileInfoMockFactory.Object);
-
-            Assert.StrictEqual<Result>(Result.Fail, validationResult.Result);
-            Assert.True(validationResult.Positions.Count == 1, $"Unexpected number of error positions");
-            Assert.True(validationResult.Positions[0] == 4, $"Unexpected position of first error");
-        }
 
         /// <summary>
         /// Defines the test method TestWithRealConfigReturnsSuccessForValidSurrogateCodePoint.
@@ -175,16 +110,20 @@ public void TestWithRealConfigReturnsErrorForInvalidCodePoint()
             fileInfoMockFactory.SetupGet(fileInfo => fileInfo.Name).Returns(new string(new char[] {
                 'a',
                 (char)0xD834, (char)0xDD1E, // valid surrogate pair
-                (char)0xFFF0, // invalid codepoint
+                (char)0xD834, 'a', // invalid surrogate pair,
+                'a', (char)0xDD1E, // invalid surrogate pair,
+                (char)0xD834, 'a', // invalid surrogate pair,
                 'z',
                 (char)0x007C // blacklisted codepoint
             }));
             IValidationResult validationResult = validation.Validate(fileInfoMockFactory.Object);
 
             Assert.StrictEqual<Result>(Result.Fail, validationResult.Result);
-            Assert.True(validationResult.Positions.Count == 2, $"Unexpected number of error positions");
-            Assert.True(validationResult.Positions[0] == 3, $"Unexpected position of first error");
-            Assert.True(validationResult.Positions[1] == 5, $"Unexpected position of second error");
+            Assert.True(validationResult.Positions.Count == 4, $"Unexpected number of error positions");
+            Assert.True(validationResult.Positions[0] == 4, $"Unexpected position of first error");
+            Assert.True(validationResult.Positions[1] == 7, $"Unexpected position of second error");
+            Assert.True(validationResult.Positions[2] == 8, $"Unexpected position of third error");
+            Assert.True(validationResult.Positions[3] == 11, $"Unexpected position of fourth error");
         }
 
     }
diff --git a/src/StorageSync/StorageSync/ChangeLog.md b/src/StorageSync/StorageSync/ChangeLog.md
@@ -21,6 +21,7 @@
 
 ## Version 2.1.0
 * Fixed minor issues.
+* Updated supported character sets in `Invoke-AzStorageSyncCompatibilityCheck`.
 
 ## Version 2.0.0
 * Deprecated "RegisteredServer" alias for InputObject parameter for Set-AzStorageSyncServerEndpoint
diff --git a/src/StorageSync/StorageSync/Config.json b/src/StorageSync/StorageSync/Config.json
@@ -49,97 +49,6 @@
     62,
     63,
     92,
-    124,
-    127
-  ],
-  "WhitelistOfCodePointRanges": [
-    {
-      "Start": 0,
-      "End": 127
-    },
-    {
-      "Start": 160,
-      "End": 55295
-    },
-    {
-      "Start": 57344,
-      "End": 63743
-    },
-    {
-      "Start": 63744,
-      "End": 64975
-    },
-    {
-      "Start": 65008,
-      "End": 65519
-    },
-    {
-      "Start": 65536,
-      "End": 131069
-    },
-    {
-      "Start": 131072,
-      "End": 196605
-    },
-    {
-      "Start": 196608,
-      "End": 262141
-    },
-    {
-      "Start": 262144,
-      "End": 327677
-    },
-    {
-      "Start": 327680,
-      "End": 393213
-    },
-    {
-      "Start": 393216,
-      "End": 458749
-    },
-    {
-      "Start": 458752,
-      "End": 524285
-    },
-    {
-      "Start": 524288,
-      "End": 589821
-    },
-    {
-      "Start": 589824,
-      "End": 655357
-    },
-    {
-      "Start": 655360,
-      "End": 720893
-    },
-    {
-      "Start": 720896,
-      "End": 786429
-    },
-    {
-      "Start": 786432,
-      "End": 851965
-    },
-    {
-      "Start": 851968,
-      "End": 917501
-    },
-    {
-      "Start": 917504,
-      "End": 921599
-    },
-    {
-      "Start": 921600,
-      "End": 983037
-    },
-    {
-      "Start": 983040,
-      "End": 1048573
-    },
-    {
-      "Start": 1048576,
-      "End": 1114109
-    }
+    124
   ]
 }
diff --git a/src/StorageSync/StorageSync/Configuration.cs b/src/StorageSync/StorageSync/Configuration.cs
@@ -54,10 +54,6 @@ public class ValidationsConfiguration
             /// </summary>
             [DataMember] public List<string> InvalidFilenames;
             /// <summary>
-            /// The whitelist of code point ranges
-            /// </summary>
-            [DataMember] public List<CodePointRange> WhitelistOfCodePointRanges;
-            /// <summary>
             /// The blacklist of code points
             /// </summary>
             [DataMember] public List<int> BlacklistOfCodePoints;
@@ -198,15 +194,6 @@ public IEnumerable<string> InvalidFileNames()
             return _validationsConfiguration.InvalidFilenames;
         }
 
-        /// <summary>
-        /// Whitelists the of code point ranges.
-        /// </summary>
-        /// <returns>IEnumerable&lt;Configuration.CodePointRange&gt;.</returns>
-        public IEnumerable<CodePointRange> WhitelistOfCodePointRanges()
-        {
-            return _validationsConfiguration.WhitelistOfCodePointRanges;
-        }
-
         /// <summary>
         /// Blacklists the of code points.
         /// </summary>
diff --git a/src/StorageSync/StorageSync/Interfaces/IConfiguration.cs b/src/StorageSync/StorageSync/Interfaces/IConfiguration.cs
@@ -37,11 +37,6 @@ public interface IConfiguration
         /// <returns>IEnumerable&lt;System.String&gt;.</returns>
         IEnumerable<string> ValidFilesystems();
         /// <summary>
-        /// Whitelists the of code point ranges.
-        /// </summary>
-        /// <returns>IEnumerable&lt;Configuration.CodePointRange&gt;.</returns>
-        IEnumerable<Configuration.CodePointRange> WhitelistOfCodePointRanges();
-        /// <summary>
         /// Blacklists the of code points.
         /// </summary>
         /// <returns>IEnumerable&lt;System.Int32&gt;.</returns>
diff --git a/src/StorageSync/StorageSync/Validations/NamespaceValidations/FilenamesCharactersValidation.cs b/src/StorageSync/StorageSync/Validations/NamespaceValidations/FilenamesCharactersValidation.cs
@@ -18,6 +18,7 @@ namespace Microsoft.Azure.Commands.StorageSync.Evaluation.Validations.NamespaceV
     using System;
     using System.Collections;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using System.Linq;
 
     /// <summary>
@@ -49,14 +50,13 @@ public FilenamesCharactersValidation(
                 "File/Directory names with unsupported characters",
                 ValidationType.FilenameCharacters)
         {
-            var whitelistOfCodePointRanges = configuration.WhitelistOfCodePointRanges().ToList();
             var blacklistOfCodePoints = new HashSet<int>(configuration.BlacklistOfCodePoints());
 
             _codePointBlackList = new bool[NumberOfCodePoints];
 
             for (int i = 0; i < NumberOfCodePoints; ++i)
             {
-                _codePointBlackList[i] = blacklistOfCodePoints.Contains(i) || !whitelistOfCodePointRanges.Any(range => range.Includes(i));
+                _codePointBlackList[i] = blacklistOfCodePoints.Contains(i);
             }
         }
         #endregion
@@ -98,16 +98,35 @@ protected override IValidationResult DoValidate(IDirectoryInfo directoryInfo)
         /// <param name="node">The node.</param>
         /// <param name="isDirectory">if set to <c>true</c> [is directory].</param>
         /// <returns>IValidationResult.</returns>
-        private IValidationResult ValidateInternal (INamedObjectInfo node, bool isDirectory)
+        private IValidationResult ValidateInternal(INamedObjectInfo node, bool isDirectory)
         {
             string name = node.Name;
             List<int> positions = new List<int>();
+            string message = string.Empty;
+            char prevChar = 'a';
 
             for (int i = 0, codepointIndex = 0; i < name.Length; ++i, ++codepointIndex)
             {
                 int codePoint;
+                message = string.Empty;
 
-                if (char.IsSurrogatePair(name, i))
+                if (char.IsHighSurrogate(name[i]) && name.Length == 1)
+                {
+                    codePoint = name[i];
+                    message = $"Invalid surrogate char found in the file name at location {i + 1}";
+                }
+                else if (!char.IsHighSurrogate(prevChar) && char.IsLowSurrogate(name[i]))
+                {
+                    codePoint = name[i];
+                    message = $"Found a low surrogate character without a preceding high surrogate character at location {i + 1}";
+                }
+                else if (char.IsHighSurrogate(prevChar) && !char.IsLowSurrogate(name[i]))
+                {
+                    --codepointIndex;
+                    codePoint = name[i];
+                    message = $"Invalid hight surrogate char found in the file name at location {i + 1}";
+                }
+                else if (char.IsSurrogatePair(name, i))
                 {
                     codePoint = char.ConvertToUtf32(name, i);
                     i += 1;
@@ -117,14 +136,24 @@ private IValidationResult ValidateInternal (INamedObjectInfo node, bool isDirect
                     codePoint = name[i];
                 }
 
-                if (codePoint < 0 || 
-                    codePoint >= NumberOfCodePoints || 
-                    _codePointBlackList[codePoint])
+                prevChar = name[i];
+
+                if (codePoint < 0 ||
+                    codePoint >= NumberOfCodePoints ||
+                    _codePointBlackList[codePoint] ||
+                    !string.IsNullOrEmpty(message))
                 {
                     // adding +1 so that positions are 1-based
                     // to make them more human friendly
                     positions.Add(codepointIndex + 1);
                 }
+
+                codepointIndex = i;
+
+                if (!string.IsNullOrEmpty(message))
+                {
+                    Trace.TraceWarning(message);
+                }
             }
 
             if (positions.Count > 0)
diff --git a/src/StorageSync/StorageSync/debug.Netcore.ps1 b/src/StorageSync/StorageSync/debug.Netcore.ps1
@@ -39,14 +39,12 @@ function Build-CharacterTable
     param ($Configuration)
 
     $blacklistOfCodePoints = $Configuration.BlacklistOfCodePoints
-    $whitelistOfCodePointRanges = $Configuration.WhitelistOfCodePointRanges
 
     $arraySize = 0x10FFFF + 1
     $blacklistedCodePointsTable = new-object bool[] $arraySize
     for ($i = 0; $i -lt $arraySize; $i += 1)
     {
-        $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i) -or
-                    ($whitelistOfCodePointRanges.Where({ ($_.Start -le $i) -and ($_.End -ge $i) }).Count -eq 0);
+        $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i);
     }
     return $blacklistedCodePointsTable
 }

Original file line number	Diff line number	Diff line change
`@@ -39,14 +39,12 @@ function Build-CharacterTable`
`39`	`39`	`param ($Configuration)`
`40`	`40`
`41`	`41`	`$blacklistOfCodePoints = $Configuration.BlacklistOfCodePoints`
`42`		`- $whitelistOfCodePointRanges = $Configuration.WhitelistOfCodePointRanges`
`43`	`42`
`44`	`43`	`$arraySize = 0x10FFFF + 1`
`45`	`44`	`$blacklistedCodePointsTable = new-object bool[] $arraySize`
`46`	`45`	`for ($i = 0; $i -lt $arraySize; $i += 1)`
`47`	`46`	`{`
`48`		`- $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i) -or`
`49`		`- ($whitelistOfCodePointRanges.Where({ ($_.Start -le $i) -and ($_.End -ge $i) }).Count -eq 0);`
	`47`	`+ $blacklistedCodePointsTable[$i] = $blacklistOfCodePoints.Contains($i);`
`50`	`48`	`}`
`51`	`49`	`return $blacklistedCodePointsTable`
`52`	`50`	`}`