Python typing support, part 2 (#18196)

sbc100 · web-flow · commit dd6774a84087 · 2022-11-14T15:05:13.000-08:00
Another incremental step in adding types to our python code. Rather
than just running mypy on 2 source files, run on all files by default
with an exclude like (currently this checks 118 source files).
diff --git a/.mypy.ini b/.mypy.ini
@@ -1,6 +1,14 @@
 [mypy]
-mypy_path = third_party/leb128
-files = tools/shared.py,tools/config.py
+mypy_path = third_party/,third_party/ply,third_party/websockify
+files = .
+exclude = (?x)(
+  third_party |
+  conf\.py |
+  emrun\.py |
+  tools/scons/site_scons/site_tools/emscripten/__init__\.py |
+  site/source/get_wiki\.py |
+  test/parse_benchmark_output\.py
+ )
 
-[mypy-*.toolchain_profiler,*.filelock]
+[mypy-tools.create_dom_pk_codes,tools.webidl_binder,tools.toolchain_profiler,tools.filelock,tools.find_bigvars,leb128,ply.*]
 ignore_errors = True
diff --git a/test/benchmark_sse.py b/test/benchmark_sse.py
@@ -18,7 +18,7 @@
 from tools.shared import WINDOWS, CLANG_CXX, EMCC, PIPE
 from tools.shared import run_process
 from tools.config import V8_ENGINE
-from test.common import EMRUN, test_file
+from common import EMRUN, test_file
 import clang_native
 
 temp_dir = tempfile.mkdtemp()
diff --git a/test/common.py b/test/common.py
@@ -7,6 +7,7 @@
 from functools import wraps
 from pathlib import Path
 from subprocess import PIPE, STDOUT
+from typing import Dict, Tuple
 from urllib.parse import unquote, unquote_plus
 from http.server import HTTPServer, SimpleHTTPRequestHandler
 import contextlib
@@ -29,8 +30,7 @@
 
 import clang_native
 import jsrun
-from tools.shared import TEMP_DIR, EMCC, EMXX, DEBUG, EMCONFIGURE, EMCMAKE
-from tools.shared import EMSCRIPTEN_TEMP_DIR
+from tools.shared import EMCC, EMXX, DEBUG, EMCONFIGURE, EMCMAKE
 from tools.shared import get_canonical_temp_dir, path_from_root
 from tools.utils import MACOS, WINDOWS, read_file, read_binary, write_file, write_binary, exit_with_error
 from tools import shared, line_endings, building, config, utils
@@ -411,8 +411,8 @@ def __new__(mcs, name, bases, attrs):
 class RunnerCore(unittest.TestCase, metaclass=RunnerMeta):
   # default temporary directory settings. set_temp_dir may be called later to
   # override these
-  temp_dir = TEMP_DIR
-  canonical_temp_dir = get_canonical_temp_dir(TEMP_DIR)
+  temp_dir = shared.TEMP_DIR
+  canonical_temp_dir = get_canonical_temp_dir(shared.TEMP_DIR)
 
   # This avoids cluttering the test runner output, which is stderr too, with compiler warnings etc.
   # Change this to None to get stderr reporting, for debugging purposes
@@ -532,7 +532,7 @@ def setUp(self):
 
     if not EMTEST_SAVE_DIR:
       self.has_prev_ll = False
-      for temp_file in os.listdir(TEMP_DIR):
+      for temp_file in os.listdir(shared.TEMP_DIR):
         if temp_file.endswith('.ll'):
           self.has_prev_ll = True
 
@@ -947,7 +947,7 @@ def assertBinaryEqual(self, file1, file2):
     self.assertEqual(read_binary(file1),
                      read_binary(file2))
 
-  library_cache = {}
+  library_cache: Dict[str, Tuple[str, object]] = {}
 
   def get_build_dir(self):
     ret = self.in_dir('building')
@@ -1005,8 +1005,8 @@ def get_library(self, name, generated_libs, configure=['sh', './configure'],  #
 
   def clear(self):
     utils.delete_contents(self.get_dir())
-    if EMSCRIPTEN_TEMP_DIR:
-      utils.delete_contents(EMSCRIPTEN_TEMP_DIR)
+    if shared.EMSCRIPTEN_TEMP_DIR:
+      utils.delete_contents(shared.EMSCRIPTEN_TEMP_DIR)
 
   def run_process(self, cmd, check=True, **args):
     # Wrapper around shared.run_process.  This is desirable so that the tests
diff --git a/test/sockets/socket_relay.py b/test/sockets/socket_relay.py
@@ -18,11 +18,14 @@
 import socket
 import time
 import threading
+from typing import Optional
 
 ports = [int(sys.argv[1]), int(sys.argv[2])]
 
 
 class Listener(threading.Thread):
+  other: Optional[Listener] = None  # noqa: F821
+
   def run(self):
     self.conn = None
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
diff --git a/test/test_benchmark.py b/test/test_benchmark.py
@@ -12,6 +12,7 @@
 import unittest
 import zlib
 from pathlib import Path
+from typing import List
 
 if __name__ == '__main__':
   raise Exception('do not run this file directly; do something like: test/runner.py benchmark')
@@ -351,7 +352,7 @@ def get_output_files(self):
 
 # Benchmarkers
 
-benchmarkers = []
+benchmarkers: List[Benchmarker] = []
 
 if not common.EMTEST_FORCE64:
   benchmarkers += [
diff --git a/test/test_sanity.py b/test/test_sanity.py
@@ -17,7 +17,6 @@
 from common import parameterized, EMBUILDER
 from tools.config import EM_CONFIG
 from tools.shared import EMCC
-from tools.shared import CANONICAL_TEMP_DIR
 from tools.shared import config
 from tools.shared import EXPECTED_LLVM_VERSION, Cache
 from tools.utils import delete_file, delete_dir
@@ -355,7 +354,7 @@ def test_emcc(self):
     # but with EMCC_DEBUG=1 we should check
     with env_modify({'EMCC_DEBUG': '1'}):
       output = self.check_working(EMCC)
-    delete_dir(CANONICAL_TEMP_DIR)
+    delete_dir(shared.CANONICAL_TEMP_DIR)
 
     self.assertContained(SANITY_MESSAGE, output)
     output = self.check_working(EMCC)
@@ -683,7 +682,7 @@ def test_with_fake(report, expected):
         self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], expected)
 
     test_with_fake('got js backend! JavaScript (asm.js, emscripten) backend', 'LLVM has not been built with the WebAssembly backend')
-    delete_dir(CANONICAL_TEMP_DIR)
+    delete_dir(shared.CANONICAL_TEMP_DIR)
 
   def test_required_config_settings(self):
     # with no binaryen root, an error is shown
diff --git a/test/test_sockets.py b/test/test_sockets.py
@@ -10,6 +10,7 @@
 import sys
 import time
 from subprocess import Popen
+from typing import List
 
 if __name__ == '__main__':
   raise Exception('do not run this file directly; do something like: test/runner sockets')
@@ -60,7 +61,7 @@ def __enter__(self):
       process = Popen([os.path.abspath('server')])
       self.processes.append(process)
 
-    import websockify
+    import websockify  # type: ignore
 
     # start the websocket proxy
     print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
@@ -153,7 +154,7 @@ def PythonTcpEchoServerProcess(port):
 
 
 class sockets(BrowserCore):
-  emcc_args = []
+  emcc_args: List[str] = []
 
   @classmethod
   def setUpClass(cls):
diff --git a/tools/building.py b/tools/building.py
@@ -405,7 +405,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False):
   return output_file
 
 
-acorn_optimizer.counter = 0
+acorn_optimizer.counter = 0  # type: ignore
 
 WASM_CALL_CTORS = '__wasm_call_ctors'
 
@@ -1315,7 +1315,7 @@ def save_intermediate(src, dst):
     shutil.copyfile(src, dst)
 
 
-save_intermediate.counter = 0
+save_intermediate.counter = 0  # type: ignore
 
 
 def js_legalization_pass_flags():
diff --git a/tools/config.py b/tools/config.py
@@ -6,7 +6,7 @@
 import os
 import sys
 import logging
-from typing import List
+from typing import List, Optional
 
 from . import utils
 from .utils import path_from_root, exit_with_error, __rootpath__, which
@@ -21,16 +21,16 @@
 NODE_JS = None
 BINARYEN_ROOT = None
 SPIDERMONKEY_ENGINE = None
-V8_ENGINE = None
+V8_ENGINE: Optional[List[str]] = None
 LLVM_ROOT = None
 LLVM_ADD_VERSION = None
 CLANG_ADD_VERSION = None
 CLOSURE_COMPILER = None
 JAVA = None
-JS_ENGINES = None
+JS_ENGINES: List[List[str]] = []
 WASMER = None
 WASMTIME = None
-WASM_ENGINES: List[str] = []
+WASM_ENGINES: List[List[str]] = []
 FROZEN_CACHE = None
 CACHE = None
 PORTS = None
diff --git a/tools/emcoverage.py b/tools/emcoverage.py
@@ -36,7 +36,7 @@
 import uuid
 from glob import glob
 
-import coverage.cmdline
+import coverage.cmdline # type: ignore
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
diff --git a/tools/extract_metadata.py b/tools/extract_metadata.py
@@ -4,12 +4,14 @@
 # found in the LICENSE file.
 
 import logging
+from typing import List, Dict
 
 from . import webassembly
 from .webassembly import OpCode, AtomicOpCode, MemoryOpCode
 from .shared import exit_with_error
 from .settings import settings
 
+
 logger = logging.getLogger('extract_metadata')
 
 
@@ -277,16 +279,16 @@ def get_string_at(module, address):
 
 
 class Metadata:
-  imports: []
-  export: []
-  asmConsts: []
-  jsDeps: []
-  emJsFuncs: {}
-  emJsFuncTypes: []
-  features: []
-  invokeFuncs: []
+  imports: List[str]
+  export: List[str]
+  asmConsts: List[str]
+  jsDeps: List[str]
+  emJsFuncs: Dict[str, str]
+  emJsFuncTypes: Dict[str, str]
+  features: List[str]
+  invokeFuncs: List[str]
   mainReadsParams: bool
-  namedGlobals: []
+  namedGlobals: List[str]
 
   def __init__(self):
     pass
diff --git a/tools/file_packager.py b/tools/file_packager.py
@@ -73,6 +73,7 @@
 import sys
 from subprocess import PIPE
 from textwrap import dedent
+from typing import List
 
 __scriptdir__ = os.path.dirname(os.path.abspath(__file__))
 __rootdir__ = os.path.dirname(__scriptdir__)
@@ -93,7 +94,7 @@
 # to work around silly av false positives
 AV_WORKAROUND = 0
 
-excluded_patterns = []
+excluded_patterns: List[str] = []
 new_data_files = []
 
 
diff --git a/tools/ports/sqlite3.py b/tools/ports/sqlite3.py
@@ -12,7 +12,6 @@
 VERSION_YEAR = 2022
 HASH = 'cbaf4adb3e404d9aa403b34f133c5beca5f641ae1e23f84dbb021da1fb9efdc7c56b5922eb533ae5cb6d26410ac60cb3f026085591bc83ebc1c225aed0cf37ca'
 
-deps = []
 variants = {'sqlite3-mt': {'USE_PTHREADS': 1}}
 
 
diff --git a/tools/shared.py b/tools/shared.py
@@ -771,6 +771,10 @@ def get_llvm_target():
 FILE_PACKAGER = bat_suffix(path_from_root('tools/file_packager'))
 WASM_SOURCEMAP = bat_suffix(path_from_root('tools/wasm-sourcemap'))
 
+# These get set by setup_temp_dirs
+TEMP_DIR = None
+EMSCRIPTEN_TEMP_DIR = None
+
 setup_temp_dirs()
 
 Cache = cache.Cache(config.CACHE)
diff --git a/tools/system_libs.py b/tools/system_libs.py
@@ -12,6 +12,7 @@
 import textwrap
 from enum import IntEnum, auto
 from glob import iglob
+from typing import List, Optional
 
 from . import shared, building, utils
 from . import deps_info
@@ -285,7 +286,7 @@ def get_default_variation(cls, **kwargs):
   # automatically get the correct version of the library.
   # This should only be overridden in a concrete library class, e.g. libc,
   # and left as None in an abstract library class, e.g. MTLibrary.
-  name = None
+  name: Optional[str] = None
 
   # Set to true to prevent EMCC_FORCE_STDLIBS from linking this library.
   never_force = False
@@ -304,18 +305,18 @@ def get_default_variation(cls, **kwargs):
   # directory into the include path, you would write:
   #    includes = [('system', 'lib', 'a'), ('system', 'lib', 'b')]
   # The include path of the parent class is automatically inherited.
-  includes = []
+  includes: List[str] = []
 
   # By default, `get_files` look for source files for this library under `src_dir`.
   # It will either use the files listed in `src_files`, or use the glob pattern in
   # `src_glob`. You may not specify both `src_files` and `src_glob`.
   # When using `src_glob`, you can specify a list of files in `src_glob_exclude`
   # to be excluded from the library.
   # Alternatively, you can override `get_files` to use your own logic.
-  src_dir = None
-  src_files = None
-  src_glob = None
-  src_glob_exclude = None
+  src_dir: Optional[str] = None
+  src_files: Optional[List[str]] = []
+  src_glob: Optional[str] = None
+  src_glob_exclude: Optional[List[str]] = None
 
   # Whether to always generate WASM object files, even when LTO is set
   force_object_files = False
diff --git a/tools/webidl_binder.py b/tools/webidl_binder.py
@@ -10,6 +10,7 @@
 
 import os
 import sys
+from typing import List
 
 __scriptdir__ = os.path.dirname(os.path.abspath(__file__))
 __rootdir__ = os.path.dirname(__scriptdir__)
@@ -643,7 +644,7 @@ def render_function(class_name, func_name, sigs, return_type, non_pointer,
   mid_js += ['\n// ' + name + '\n']
   mid_c += ['\n// ' + name + '\n']
 
-  js_impl_methods = []
+  js_impl_methods: List[str] = []
 
   cons = interface.getExtendedAttribute('Constructor')
   if type(cons) == list:
