[MERGE #5658 @rhuanjl] Add check for non-ascii fix:#4268

Merge pull request #5658 from rhuanjl:ascii

This PR adds an additional test to CI to block non-ascii characters, the tests, tools and Build folders are excluded from the check.

This is based on the eol check test but with a different grep command.

Additionally I found some non-ascii at the start of JavascriptString.cpp so have removed it.

@dilijev please can you take a look

Author: dilijev

jenkins/check_ascii.sh

@@ -0,0 +1,26 @@
+#-------------------------------------------------------------------------------------------------------
+# Copyright (C) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+#-------------------------------------------------------------------------------------------------------
+
+# Need to make sure that the reference to origin/master is available.
+# We know that HEAD is checked out so that the tests on that source can be run.
+
+# configure the sh environment to run scripts from the bin dir in case that's missing
+ls &> /dev/null # checking for ls script on the path
+if [ $? -ne 0 ]; then
+    PATH=/bin:/usr/bin:$PATH
+fi
+
+ERRFILE=check_ascii.sh.err
+rm -f $ERRFILE
+git diff --name-only `git merge-base origin/master HEAD` HEAD | grep -v -E "test/.*|tools/.*|Build/.*|/*.md|/*.txt|/*.vcxproj" | xargs -I % ./jenkins/check_file_ascii.sh %
+
+if [ -e $ERRFILE ]; then # if error file exists then there were errors
+    >&2 echo "--------------" # leading >&2 means echo to stderr
+    >&2 echo "--- ERRORS ---"
+    cat $ERRFILE 1>&2 # send output to stderr so it can be redirected as error if desired
+    exit 1 # tell the caller there was an error (so Jenkins will fail the CI task)
+else
+    echo "--- NO PROBLEMS DETECTED ---"
+fi

jenkins/check_file_ascii.sh

@@ -0,0 +1,36 @@
+#-------------------------------------------------------------------------------------------------------
+# Copyright (C) Microsoft. All rights reserved.
+# Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+#-------------------------------------------------------------------------------------------------------
+
+ERRFILE=check_ascii.sh.err
+ERRFILETEMP=$ERRFILE.0
+
+# display a helpful message for someone reading the log
+echo "Check ascii > Checking $1"
+
+if [ ! -e $1 ]; then # the file wasn't present; not necessarily an error
+    echo "WARNING: file not found: $1"
+    exit 0 # don't report an error but don't run the rest of this file
+fi
+
+# grep for non-ascii - also exclude unprintable control characters at the end of the range
+# specifically include x09 (tab) as it is used in pal sources which are not excluded
+# from this check
+LC_CTYPE=C grep -nP '[^\x09-\x7E]' $1 > $ERRFILETEMP
+if [ $? -eq 0 ]; then # grep found matches ($?==0), so we found non-ascii in the file
+    echo "ERROR: non-ascii characters were introduced in $1" >> $ERRFILE
+
+    # Display a hexdump sample of the lines with non-ascii characters in them
+    # Don't pollute the log with every single matching line, first 10 lines should be enough.
+    echo "Displaying first 10 lines of text where non-ascii characters were found:" >> $ERRFILE
+    LC_CTYPE=C grep -nP '[^\x09-\x7E]' $1 | xxd -g 1 > $ERRFILETEMP
+    head -n 10 $ERRFILETEMP >> $ERRFILE
+
+    # To help the user, display how many lines of text actually contained non-ascii characters.
+    LINECOUNT=`python -c "file=open('$ERRFILETEMP', 'r'); print len(file.readlines())"`
+    echo "Total lines containing non-ascii: $LINECOUNT" >> $ERRFILE
+    echo "--------------" >> $ERRFILE # same length as '--- ERRORS ---'
+fi
+
+rm -f $ERRFILETEMP

lib/Runtime/Library/JavascriptString.cpp

@@ -1,4 +1,4 @@
-//-------------------------------------------------------------------------------------------------------
+//-------------------------------------------------------------------------------------------------------
 // Copyright (C) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
 //-------------------------------------------------------------------------------------------------------

netci.groovy

@@ -339,6 +339,7 @@ if (!branch.endsWith('-ci')) {
 CreateStyleCheckTasks('./jenkins/check_copyright.sh', 'ubuntu_check_copyright', 'Copyright Check')
 CreateStyleCheckTasks('./jenkins/check_eol.sh', 'ubuntu_check_eol', 'EOL Check')
 CreateStyleCheckTasks('./jenkins/check_tabs.sh', 'ubuntu_check_tabs', 'Tab Check')
+CreateStyleCheckTasks('./jenkins/check_ascii.sh', 'ubuntu_check_ascii', 'ASCII Check')
 
 // --------------
 // XPLAT BRANCHES