Merge #14128: lint: Make sure we read the command line inputs using utf-8 decoding in python

5d62dcf lint: Make sure we read the command line inputs using utf-8 decoding in python (Chun Kuan Lee)

Pull request description:

  Make sure we read the command line inputs using utf-8 decoding in python

  occurred from travis cron job:

  contrib/verify-commits/verify-commits.py should run with utf-8, otherwise it would raise UnicodeDecodeError
  `UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 744: ordinal not in range(128)`

Tree-SHA512: 90e4ad57fdbbbecb0a21fc2d2b03a04f5ef125e54124719ef36e5a85326930b732b47534757a7c3a8730096f3947b009ec898191928b5c2d38f9f4b3e37db48d

Author: laanwj

contrib/devtools/optimize-pngs.py

@@ -27,7 +27,7 @@ def content_hash(filename):
 pngcrush = 'pngcrush'
 git = 'git'
 folders = ["src/qt/res/movies", "src/qt/res/icons", "share/pixmaps"]
-basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True).rstrip('\n')
+basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n')
 totalSaveBytes = 0
 noHashChange = True
 
@@ -50,7 +50,7 @@ def content_hash(filename):
                 sys.exit(0)
 
             #verify
-            if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True):
+            if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True, encoding='utf8'):
                 print("PNG file "+file+" is corrupted after crushing, check out pngcursh version")
                 sys.exit(1)
 

contrib/gitian-build.py

@@ -209,7 +209,7 @@ def main():
         subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
         os.chdir('../gitian-builder/inputs/bitcoin')
         subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
-        args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip()
+        args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip()
         args.version = 'pull-' + args.version
     print(args.commit)
     subprocess.check_call(['git', 'fetch'])

contrib/verify-commits/verify-commits.py

@@ -91,7 +91,7 @@ def main():
     no_sha1 = True
     prev_commit = ""
     initial_commit = current_commit
-    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True).splitlines()[0]
+    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
 
     # Iterate through commits
     while True:
@@ -112,7 +112,7 @@ def main():
             if prev_commit != "":
                 print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
                 print("Parents are:", file=sys.stderr)
-                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True).splitlines()[0].split(' ')
+                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
                 for parent in parents:
                     subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
             else:
@@ -122,25 +122,25 @@ def main():
         # Check the Tree-SHA512
         if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
             tree_hash = tree_sha512sum(current_commit)
-            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True).splitlines():
+            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True, encoding='utf8').splitlines():
                 print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
                 sys.exit(1)
 
         # Merge commits should only have two parents
-        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True).splitlines()[0].split(' ')
+        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
         if len(parents) > 2:
             print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
             sys.exit(1)
 
         # Check that the merge commit is clean
-        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True).splitlines()[0])
+        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0])
         check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60  # Only check commits in clean_merge days
         allow_unclean = current_commit in unclean_merge_allowed
         if len(parents) == 2 and check_merge and not allow_unclean:
-            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True).splitlines()[0]
+            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
             subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
             subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL)
-            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True).splitlines()[0]
+            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True, encoding='utf8').splitlines()[0]
             if current_tree != recreated_tree:
                 print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
                 subprocess.call([GIT, 'diff', current_commit])

test/lint/check-doc.py

@@ -26,8 +26,8 @@
 
 
 def main():
-    used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True)
-    docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True)
+    used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8')
+    docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8')
 
     args_used = set(re.findall(re.compile(REGEX_ARG), used))
     args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)

test/lint/lint-python-utf8-encoding.sh

@@ -17,4 +17,12 @@ if [[ ${OUTPUT} != "" ]]; then
     echo "${OUTPUT}"
     EXIT_CODE=1
 fi
+OUTPUT=$(git grep "check_output(" -- "*.py" | grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).")
+if [[ ${OUTPUT} != "" ]]; then
+    echo "Python's check_output(...) seems to be used to get program outputs without explicitly"
+    echo "specifying encoding=\"utf8\":"
+    echo
+    echo "${OUTPUT}"
+    EXIT_CODE=1
+fi
 exit ${EXIT_CODE}