Improve Windows support (#248)

elopez · web-flow · commit 66fe16dab4e2 · 2022-04-01T13:55:23.000+02:00
* utils: npm: fix path detection on Windows Windows will throw ValueError and not OSError if the path is too long pathobj = WindowsPath('{"compilation_units": {"/home/monty/tob/tools/slither/tests/ast-parsing/function-0.6.0.sol": {"compiler":...unction abstractFunc2() public virtual override(C6) {}/n function abstractFunc3() public virtual override {}/n}"}}') args = () @functools.wraps(strfunc) def wrapped(pathobj, *args): > return strfunc(str(pathobj), *args) E ValueError: stat: path too long for Windows c:\hostedtoolcache\windows\python\3.6.8\x64\lib\pathlib.py:387: ValueError * platform: archive: fix path checking on Windows Windows may throw an exception if the path is too long, which is the case when the string is not really a path. Catch it and try decoding the JSON in that case. * platform: *: fix command execution on Windows Some of the command executions could fail on Windows with errors such as the following: FileNotFoundError: [WinError 2] The system cannot find the file specified This is because the program name used in the command (e.g. "npm") is unqualified, and PATH lookup is not performed on Windows when `shell=False` is used. We can avoid this problem by converting the executable name to a fully-qualified path by using `shutil.which` and passing it as `executable` on the `Popen` call. * utils: naming: use POSIX-style paths for relative paths When looking up full paths, Windows-style paths may show up, but standardize on using POSIX-style paths for everything else. * platform: solc: remove extra prints These were introduced in bb594de ("upgrade pylint and black for super-linter v4") and currently break Slither's tests, which expect a certain output. * workflows: ci: disable embark It is currently broken, and tries to pull a repository via git:// which GH Actions now blocks.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,6 +16,9 @@ jobs:
     strategy:
       matrix:
         type: ["solc", "truffle", "embark", "etherlime", "brownie", "waffle", "buidler", "hardhat"]
+        exclude:
+          # Currently broken, tries to pull git:// which is blocked by GH
+          - type: embark
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python 3.6
diff --git a/crytic_compile/platform/archive.py b/crytic_compile/platform/archive.py
@@ -77,11 +77,16 @@ def compile(self, crytic_compile: "CryticCompile", **_kwargs: str) -> None:
         # pylint: disable=import-outside-toplevel
         from crytic_compile.crytic_compile import get_platforms
 
-        if isinstance(self._target, str) and os.path.isfile(self._target):
-            with open(self._target, encoding="utf8") as f_target:
-                loaded_json = json.load(f_target)
-        else:
+        try:
+            if isinstance(self._target, str) and os.path.isfile(self._target):
+                with open(self._target, encoding="utf8") as f_target:
+                    loaded_json = json.load(f_target)
+            else:
+                loaded_json = json.loads(self._target)
+        except (OSError, ValueError):
+            # Can happen if the target is a very large string, isfile will throw an exception
             loaded_json = json.loads(self._target)
+
         (underlying_type, unit_tests) = standard.load_from_compile(crytic_compile, loaded_json)
         underlying_type = TypePlatform(underlying_type)
         platforms: List[Type[AbstractPlatform]] = get_platforms()
diff --git a/crytic_compile/platform/brownie.py b/crytic_compile/platform/brownie.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, List
@@ -54,7 +55,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
             cmd = base_cmd + ["compile"]
             try:
                 with subprocess.Popen(
-                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self._target
+                    cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    cwd=self._target,
+                    executable=shutil.which(cmd[0]),
                 ) as process:
                     stdout_bytes, stderr_bytes = process.communicate()
                     stdout, stderr = (
diff --git a/crytic_compile/platform/buidler.py b/crytic_compile/platform/buidler.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Tuple
@@ -70,7 +71,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
             )
 
             with subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self._target
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=self._target,
+                executable=shutil.which(cmd[0]),
             ) as process:
 
                 stdout_bytes, stderr_bytes = process.communicate()
diff --git a/crytic_compile/platform/dapp.py b/crytic_compile/platform/dapp.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -176,7 +177,11 @@ def _run_dapp(target: str) -> None:
 
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=target
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=target,
+            executable=shutil.which(cmd[0]),
         ) as process:
             _, _ = process.communicate()
     except OSError as error:
diff --git a/crytic_compile/platform/embark.py b/crytic_compile/platform/embark.py
@@ -5,6 +5,7 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, List
@@ -64,7 +65,9 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
             if write_embark_json:
                 try:
                     with subprocess.Popen(
-                        ["npm", "install", plugin_name], cwd=self._target
+                        ["npm", "install", plugin_name],
+                        cwd=self._target,
+                        executable=shutil.which("npm"),
                     ) as process:
                         _, stderr = process.communicate()
                         with open(
@@ -91,7 +94,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
                     cmd = ["npx"] + cmd
                 # pylint: disable=consider-using-with
                 process = subprocess.Popen(
-                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self._target
+                    cmd,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    cwd=self._target,
+                    executable=shutil.which(cmd[0]),
                 )
             except OSError as error:
                 # pylint: disable=raise-missing-from
diff --git a/crytic_compile/platform/etherlime.py b/crytic_compile/platform/etherlime.py
@@ -7,6 +7,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional, Any
@@ -48,7 +49,11 @@ def _run_etherlime(target: str, npx_disable: bool, compile_arguments: Optional[s
 
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=target
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=target,
+            executable=shutil.which(cmd[0]),
         ) as process:
             stdout_bytes, stderr_bytes = process.communicate()
             stdout, stderr = (
diff --git a/crytic_compile/platform/hardhat.py b/crytic_compile/platform/hardhat.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, List
@@ -70,7 +71,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
             )
 
             with subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self._target
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=self._target,
+                executable=shutil.which(cmd[0]),
             ) as process:
 
                 stdout_bytes, stderr_bytes = process.communicate()
diff --git a/crytic_compile/platform/solc.py b/crytic_compile/platform/solc.py
@@ -5,6 +5,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, List, Optional, Union, Any
@@ -367,7 +368,11 @@ def get_version(solc: str, env: Optional[Dict[str, str]]) -> str:
     cmd = [solc, "--version"]
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            executable=shutil.which(cmd[0]),
         ) as process:
             stdout_bytes, _ = process.communicate()
             stdout = stdout_bytes.decode()  # convert bytestrings to unicode strings
@@ -376,7 +381,6 @@ def get_version(solc: str, env: Optional[Dict[str, str]]) -> str:
                 raise InvalidCompilation(f"Solidity version not found: {stdout}")
             return version[0]
     except OSError as error:
-        print("get versions")
         # pylint: disable=raise-missing-from
         raise InvalidCompilation(error)
 
@@ -509,13 +513,21 @@ def _run_solc(
         # pylint: disable=consider-using-with
         if env:
             process = subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **additional_kwargs
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                executable=shutil.which(cmd[0]),
+                env=env,
+                **additional_kwargs,
             )
         else:
             process = subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **additional_kwargs
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                executable=shutil.which(cmd[0]),
+                **additional_kwargs,
             )
-        print(cmd)
     except OSError as error:
         # pylint: disable=raise-missing-from
         raise InvalidCompilation(error)
diff --git a/crytic_compile/platform/solc_standard_json.py b/crytic_compile/platform/solc_standard_json.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from typing import TYPE_CHECKING, Dict, List, Optional, Union, Any
 
@@ -146,6 +147,7 @@ def run_solc_standard_json(
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             env=env,
+            executable=shutil.which(cmd[0]),
             **additional_kwargs,
         ) as process:
 
diff --git a/crytic_compile/platform/truffle.py b/crytic_compile/platform/truffle.py
@@ -167,7 +167,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
                 _write_config(Path(self._target), config_used, overwritten_version)
 
             with subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self._target
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=self._target,
+                executable=shutil.which(cmd[0]),
             ) as process:
 
                 stdout_bytes, stderr_bytes = process.communicate()
@@ -368,7 +372,11 @@ def _get_version(truffle_call: List[str], cwd: str) -> Tuple[str, str]:
     cmd = truffle_call + ["version"]
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=cwd,
+            executable=shutil.which(cmd[0]),
         ) as process:
             sstdout, _ = process.communicate()
             ssstdout = sstdout.decode()  # convert bytestrings to unicode strings
diff --git a/crytic_compile/platform/vyper.py b/crytic_compile/platform/vyper.py
@@ -3,6 +3,7 @@
 """
 import json
 import os
+import shutil
 import subprocess
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, List, Optional
@@ -147,7 +148,12 @@ def _run_vyper(
     additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **additional_kwargs
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            executable=shutil.which(cmd[0]),
+            **additional_kwargs,
         ) as process:
             stdout, stderr = process.communicate()
             res = stdout.split(b"\n")
@@ -186,7 +192,12 @@ def _get_vyper_ast(
     additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
     try:
         with subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, **additional_kwargs
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=env,
+            executable=shutil.which(cmd[0]),
+            **additional_kwargs,
         ) as process:
             stdout, stderr = process.communicate()
             res = stdout.split(b"\n")
diff --git a/crytic_compile/platform/waffle.py b/crytic_compile/platform/waffle.py
@@ -6,6 +6,7 @@
 import logging
 import os
 import re
+import shutil
 import subprocess
 import tempfile
 from pathlib import Path
@@ -151,7 +152,11 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
 
                 try:
                     with subprocess.Popen(
-                        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=target
+                        cmd,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        cwd=target,
+                        executable=shutil.which(cmd[0]),
                     ) as process:
                         stdout, stderr = process.communicate()
                         if stdout:
@@ -323,7 +328,11 @@ def _get_version(compiler: str, cwd: str, config: Optional[Dict] = None) -> str:
         cmd = ["solc", "--version"]
         try:
             with subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=cwd,
+                executable=shutil.which(cmd[0]),
             ) as process:
                 stdout_bytes, _ = process.communicate()
                 stdout_txt = stdout_bytes.decode()  # convert bytestrings to unicode strings
@@ -339,7 +348,11 @@ def _get_version(compiler: str, cwd: str, config: Optional[Dict] = None) -> str:
         cmd = ["solcjs", "--version"]
         try:
             with subprocess.Popen(
-                cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=cwd,
+                executable=shutil.which(cmd[0]),
             ) as process:
                 stdout_bytes, _ = process.communicate()
                 stdout_txt = stdout_bytes.decode()  # convert bytestrings to unicode strings
diff --git a/crytic_compile/utils/naming.py b/crytic_compile/utils/naming.py
@@ -135,5 +135,8 @@ def convert_filename(
     short = relative_to_short(short)
 
     return Filename(
-        absolute=str(absolute), relative=str(relative), short=str(short), used=used_filename
+        absolute=str(absolute),
+        relative=relative.as_posix(),
+        short=short.as_posix(),
+        used=used_filename,
     )
diff --git a/crytic_compile/utils/npm.py b/crytic_compile/utils/npm.py
@@ -38,6 +38,6 @@ def get_package_name(target_txt: Union[str, "SolcStandardJson"]) -> Optional[str
                         return None
         return None
 
-    except OSError:
+    except (OSError, ValueError):
         # Can happen if the target is a very large string, is_dir will throw an exception
         return None
