Merge #12987: tests/tools: Enable additional Python flake8 rules for automatic linting via Travis

643aad1 Enable additional flake8 rules (practicalswift)
f020aca Minor Python cleanups to make flake8 pass with the new rules enabled (practicalswift)

Pull request description:

  Enabled rules:

  ```
  * E242: tab after ','
  * E266: too many leading '#' for block comment
  * E401: multiple imports on one line
  * E402: module level import not at top of file
  * E701: multiple statements on one line (colon)
  * E901: SyntaxError: invalid syntax
  * E902: TokenError: EOF in multi-line string
  * F821: undefined name 'Foo'
  * W293: blank line contains whitespace
  * W606: 'async' and 'await' are reserved keywords starting with Python 3.7
  ```

  Note to reviewers:
  * In general we don't allow whitespace cleanups to existing code, but in order to allow for enabling Travis checking for these rules a few smaller whitespace cleanups had to made as part of this PR.
  * Use [this `?w=1` link](https://github.com/bitcoin/bitcoin/pull/12987/files?w=1) to show a diff without whitespace changes.

  Before this commit:

  ```
  $ flake8 -qq --statistics --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W504,W601,W602,W603,W604,W605,W606 .
  5     E266 too many leading '#' for block comment
  4     E401 multiple imports on one line
  6     E402 module level import not at top of file
  5     E701 multiple statements on one line (colon)
  1     F812 list comprehension redefines 'n' from line 159
  4     F821 undefined name 'ConnectionRefusedError'
  28    W293 blank line contains whitespace
  ```

  After this commit:

  ```
  $ flake8 -qq --statistics --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W504,W601,W602,W603,W604,W605,W606 .
  $
  ```

Tree-SHA512: fc7d5e752298a50d4248afc620ee2c173135b4ca008e48e02913ac968e5a24a5fd5396926047ec62f1d580d537434ccae01f249bb2f3338fa59dc630bf97ca7a

Author: MarcoFalke

contrib/devtools/copyright_header.py

@@ -286,7 +286,7 @@ def exec_report(base_directory, verbose):
 def report_cmd(argv):
     if len(argv) == 2:
         sys.exit(REPORT_USAGE)
-        
+
     base_directory = argv[2]
     if not os.path.exists(base_directory):
         sys.exit("*** bad <base_directory>: %s" % base_directory)
@@ -444,7 +444,7 @@ def print_file_action_message(filename, action):
 def update_cmd(argv):
     if len(argv) != 3:
         sys.exit(UPDATE_USAGE)
-    
+
     base_directory = argv[2]
     if not os.path.exists(base_directory):
         sys.exit("*** bad base_directory: %s" % base_directory)
@@ -570,13 +570,13 @@ def insert_cmd(argv):
     _, extension = os.path.splitext(filename)
     if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
         sys.exit("*** cannot insert for file extension %s" % extension)
-   
+
     if extension == '.py': 
         style = 'python'
     else:
         style = 'cpp'
     exec_insert_header(filename, style)
-         
+
 ################################################################################
 # UI
 ################################################################################

contrib/devtools/github-merge.py

@@ -21,7 +21,8 @@
 import hashlib
 import subprocess
 import sys
-import json,codecs
+import json
+import codecs
 try:
     from urllib.request import Request,urlopen
 except:

contrib/devtools/lint-python.sh

@@ -15,21 +15,28 @@
 # E133 closing bracket is missing indentation
 # E223 tab before operator
 # E224 tab after operator
+# E242 tab after ','
+# E266 too many leading '#' for block comment
 # E271 multiple spaces after keyword
 # E272 multiple spaces before keyword
 # E273 tab after keyword
 # E274 tab before keyword
 # E275 missing whitespace after keyword
 # E304 blank lines found after function decorator
 # E306 expected 1 blank line before a nested definition
+# E401 multiple imports on one line
+# E402 module level import not at top of file
 # E502 the backslash is redundant between brackets
+# E701 multiple statements on one line (colon)
 # E702 multiple statements on one line (semicolon)
 # E703 statement ends with a semicolon
 # E714 test for object identity should be "is not"
 # E721 do not compare types, use "isinstance()"
 # E741 do not use variables named "l", "O", or "I"
 # E742 do not define classes named "l", "O", or "I"
 # E743 do not define functions named "l", "O", or "I"
+# E901 SyntaxError: invalid syntax
+# E902 TokenError: EOF in multi-line string
 # F401 module imported but unused
 # F402 import module from line N shadowed by loop variable
 # F404 future import(s) name after other statements
@@ -49,16 +56,19 @@
 # F707 an except: block as not the last exception handler
 # F811 redefinition of unused name from line N
 # F812 list comprehension redefines 'foo' from line N
+# F821 undefined name 'Foo'
 # F822 undefined name name in __all__
 # F823 local variable name … referenced before assignment
 # F831 duplicate argument name in function definition
 # F841 local variable 'foo' is assigned to but never used
 # W292 no newline at end of file
+# W293 blank line contains whitespace
 # W504 line break after binary operator
 # W601 .has_key() is deprecated, use "in"
 # W602 deprecated form of raising exception
 # W603 "<>" is deprecated, use "!="
 # W604 backticks are deprecated, use "repr()"
 # W605 invalid escape sequence "x"
+# W606 'async' and 'await' are reserved keywords starting with Python 3.7
 
-flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E271,E272,E273,E274,E275,E304,E306,E502,E702,E703,E714,E721,E741,E742,E743,F401,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F822,F823,F831,F841,W292,W504,W601,W602,W603,W604,W605 .
+flake8 --ignore=B,C,E,F,I,N,W --select=E112,E113,E115,E116,E125,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,F401,E901,E902,F402,F404,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W292,W293,W504,W601,W602,W603,W604,W605,W606 .

contrib/devtools/optimize-pngs.py

@@ -41,19 +41,19 @@ def content_hash(filename):
             file_path = os.path.join(absFolder, file)
             fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)}
             fileMetaMap['contentHashPre'] = content_hash(file_path)
-        
+
             try:
                 subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path],
                                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             except:
                 print("pngcrush is not installed, aborting...")
                 sys.exit(0)
-        
+
             #verify
             if "Not a PNG file" in subprocess.check_output([pngcrush, "-n", "-v", file_path], stderr=subprocess.STDOUT, universal_newlines=True):
                 print("PNG file "+file+" is corrupted after crushing, check out pngcursh version")
                 sys.exit(1)
-            
+
             fileMetaMap['sha256New'] = file_hash(file_path)
             fileMetaMap['contentHashPost'] = content_hash(file_path)
 
@@ -72,5 +72,5 @@ def content_hash(filename):
     totalSaveBytes += fileDict['osize'] - fileDict['psize']
     noHashChange = noHashChange and (oldHash == newHash)
     print(fileDict['file']+"\n  size diff from: "+str(fileDict['osize'])+" to: "+str(fileDict['psize'])+"\n  old sha256: "+oldHash+"\n  new sha256: "+newHash+"\n")
-    
+
 print("completed. Checksum stable: "+str(noHashChange)+". Total reduction: "+str(totalSaveBytes)+" bytes")

contrib/linearize/linearize-data.py

@@ -21,7 +21,6 @@
 
 settings = {}
 
-##### Switch endian-ness #####
 def hex_switchEndian(s):
 	""" Switches the endianness of a hex string (in pairs of hex chars) """
 	pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]

contrib/linearize/linearize-hashes.py

@@ -21,7 +21,6 @@
 
 settings = {}
 
-##### Switch endian-ness #####
 def hex_switchEndian(s):
 	""" Switches the endianness of a hex string (in pairs of hex chars) """
 	pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]

contrib/seeds/generate-seeds.py

@@ -34,7 +34,8 @@
 
 from base64 import b32decode
 from binascii import a2b_hex
-import sys, os
+import sys
+import os
 import re
 
 # ipv4 in ipv6 prefix
@@ -46,7 +47,7 @@ def name_to_ipv6(addr):
     if len(addr)>6 and addr.endswith('.onion'):
         vchAddr = b32decode(addr[0:-6], True)
         if len(vchAddr) != 16-len(pchOnionCat):
-            raise ValueError('Invalid onion %s' % s)
+            raise ValueError('Invalid onion %s' % vchAddr)
         return pchOnionCat + vchAddr
     elif '.' in addr: # IPv4
         return pchIPv4 + bytearray((int(x) for x in addr.split('.')))
@@ -132,7 +133,7 @@ def main():
     with open(os.path.join(indir,'nodes_test.txt'),'r') as f:
         process_nodes(g, f, 'pnSeed6_test', 18333)
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
-            
+
 if __name__ == '__main__':
     main()
 

contrib/seeds/makeseeds.py

@@ -6,6 +6,11 @@
 # Generate seeds.txt from Pieter's DNS seeder
 #
 
+import re
+import sys
+import dns.resolver
+import collections
+
 NSEEDS=512
 
 MAX_SEEDS_PER_ASN=2
@@ -22,11 +27,6 @@
     "54.94.195.96", "54.94.200.247"
 }
 
-import re
-import sys
-import dns.resolver
-import collections
-
 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
 PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
 PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")

contrib/testgen/base58.py

@@ -43,8 +43,10 @@ def b58encode(v):
     # leading 0-bytes in the input become leading-1s
     nPad = 0
     for c in v:
-        if c == 0: nPad += 1
-        else: break
+        if c == 0:
+            nPad += 1
+        else:
+            break
 
     return (__b58chars[0]*nPad) + result
 
@@ -98,7 +100,8 @@ def b58decode_chk(v):
 def get_bcaddress_version(strAddress):
     """ Returns None if strAddress is invalid.  Otherwise returns integer version of address. """
     addr = b58decode_chk(strAddress)
-    if addr is None or len(addr)!=21: return None
+    if addr is None or len(addr)!=21:
+        return None
     version = addr[0]
     return ord(version)
 

contrib/testgen/gen_base58_test_vectors.py

@@ -74,12 +74,12 @@ def gen_invalid_vector(template, corrupt_prefix, randomize_payload_size, corrupt
         prefix = os.urandom(1)
     else:
         prefix = bytearray(template[0])
-    
+
     if randomize_payload_size:
         payload = os.urandom(max(int(random.expovariate(0.5)), 50))
     else:
         payload = os.urandom(template[1])
-    
+
     if corrupt_suffix:
         suffix = os.urandom(len(template[2]))
     else:
@@ -114,7 +114,8 @@ def gen_invalid_vectors():
                 yield val,
 
 if __name__ == '__main__':
-    import sys, json
+    import sys
+    import json
     iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors}
     try:
         uiter = iters[sys.argv[1]]
@@ -124,7 +125,7 @@ def gen_invalid_vectors():
         count = int(sys.argv[2])
     except IndexError:
         count = 0
-   
+
     data = list(islice(uiter(), count))
     json.dump(data, sys.stdout, sort_keys=True, indent=4)
     sys.stdout.write('\n')