feat: compile vyper 0.3.7 via standard json input

Author: 0xalpharush

crytic_compile/crytic_compile.py

@@ -15,7 +15,9 @@
 from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type, Union
 
 from crytic_compile.compilation_unit import CompilationUnit
-from crytic_compile.platform import all_platforms, solc_standard_json
+from crytic_compile.platform import all_platforms
+from crytic_compile.platform.solc_standard_json import SolcStandardJson
+from crytic_compile.platform.vyper import VyperStandardJson
 from crytic_compile.platform.abstract_platform import AbstractPlatform
 from crytic_compile.platform.all_export import PLATFORMS_EXPORT
 from crytic_compile.platform.solc import Solc
@@ -628,11 +630,7 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]:
     """
     use_solc_standard_json = kwargs.get("solc_standard_json", False)
 
-    # Attempt to perform glob expansion of target/filename
-    globbed_targets = glob.glob(target, recursive=True)
-
     # Check if the target refers to a valid target already.
-    # If it does not, we assume it's a glob pattern.
     compilations: List[CryticCompile] = []
     if os.path.isfile(target) or is_supported(target):
         if target.endswith(".zip"):
@@ -644,28 +642,33 @@ def compile_all(target: str, **kwargs: str) -> List[CryticCompile]:
                     compilations = load_from_zip(tmp.name)
         else:
             compilations.append(CryticCompile(target, **kwargs))
-    elif os.path.isdir(target) or len(globbed_targets) > 0:
-        # We create a new glob to find solidity files at this path (in case this is a directory)
-        filenames = glob.glob(os.path.join(target, "*.sol"))
-        if not filenames:
-            filenames = glob.glob(os.path.join(target, "*.vy"))
-            if not filenames:
-                filenames = globbed_targets
-
+    elif os.path.isdir(target):
+        solidity_filenames = glob.glob(os.path.join(target, "*.sol"))
+        vyper_filenames = glob.glob(os.path.join(target, "*.vy"))
         # Determine if we're using --standard-solc option to
         # aggregate many files into a single compilation.
         if use_solc_standard_json:
             # If we're using standard solc, then we generated our
             # input to create a single compilation with all files
-            standard_json = solc_standard_json.SolcStandardJson()
-            for filename in filenames:
-                standard_json.add_source_file(filename)
-            compilations.append(CryticCompile(standard_json, **kwargs))
+            solc_standard_json = SolcStandardJson()
+            solc_standard_json.add_source_files(solidity_filenames)
+            compilations.append(CryticCompile(solc_standard_json, **kwargs))
         else:
             # We compile each file and add it to our compilations.
-            for filename in filenames:
+            for filename in solidity_filenames:
                 compilations.append(CryticCompile(filename, **kwargs))
+
+        if vyper_filenames:
+            vyper_standard_json = VyperStandardJson()
+            vyper_standard_json.add_source_files(vyper_filenames)
+            compilations.append(CryticCompile(vyper_standard_json, **kwargs))
     else:
-        raise ValueError(f"{str(target)} is not a file or directory.")
+        raise NotImplementedError()
+        # TODO split glob into language
+        # # Attempt to perform glob expansion of target/filename
+        # globbed_targets = glob.glob(target, recursive=True)
+        # print(globbed_targets)
+
+        # raise ValueError(f"{str(target)} is not a file or directory.")
 
     return compilations

crytic_compile/platform/all_platforms.py

@@ -14,6 +14,6 @@
 from .solc_standard_json import SolcStandardJson
 from .standard import Standard
 from .truffle import Truffle
-from .vyper import Vyper
+from .vyper import VyperStandardJson
 from .waffle import Waffle
 from .foundry import Foundry

crytic_compile/platform/vyper.py

@@ -15,6 +15,7 @@
 from crytic_compile.platform.exceptions import InvalidCompilation
 from crytic_compile.platform.types import Type
 from crytic_compile.utils.naming import convert_filename
+from crytic_compile.utils.subprocess import run
 
 # Handle cycle
 from crytic_compile.utils.natspec import Natspec
@@ -25,14 +26,36 @@
 LOGGER = logging.getLogger("CryticCompile")
 
 
-class Vyper(AbstractPlatform):
+class VyperStandardJson(AbstractPlatform):
     """
     Vyper platform
     """
 
     NAME = "vyper"
     PROJECT_URL = "https://github.com/vyperlang/vyper"
     TYPE = Type.VYPER
+    standard_json_input: Dict = {
+        "language": "Vyper",
+        "sources": {},
+        "settings": {
+            "outputSelection": {
+                "*": {
+                    "*": [
+                        "abi",
+                        "devdoc",
+                        "userdoc",
+                        "evm.bytecode",
+                        "evm.deployedBytecode",
+                        "evm.deployedBytecode.sourceMap",
+                    ],
+                    "": ["ast"],
+                }
+            }
+        },
+    }
+
+    def __init__(self, target: Optional[Path] = None, **_kwargs: str):
+        super().__init__(target, **_kwargs)
 
     def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
         """Compile the target
@@ -44,46 +67,61 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
 
         """
         target = self._target
+        # If the target was a directory `add_source_file` should have been called
+        # by `compile_all`. Otherwise, we should have a single file target.
+        if self._target is not None and os.path.isfile(self._target):
+            self.add_source_files([target])
 
-        vyper = kwargs.get("vyper", "vyper")
+        vyper_bin = kwargs.get("vyper", "vyper")
+        output_file = Path("crytic-export/standard_input.json")
+        output_file.parent.mkdir(exist_ok=True, parents=True)
+        with open(output_file, "w") as f:
+            f.write(json.dumps(self.standard_json_input))
 
-        targets_json = _run_vyper(target, vyper)
+        compilation_artifacts = _run_vyper_standard_json(output_file.as_posix(), vyper_bin)
 
-        assert "version" in targets_json
         compilation_unit = CompilationUnit(crytic_compile, str(target))
 
+        compiler_version = compilation_artifacts["compiler"].split("-")[1]
+        assert compiler_version == "0.3.7"
         compilation_unit.compiler_version = CompilerVersion(
-            compiler="vyper", version=targets_json["version"], optimized=False
+            compiler="vyper", version=compiler_version, optimized=False
         )
 
-        assert target in targets_json
-
-        info = targets_json[target]
-        filename = convert_filename(target, _relative_to_short, crytic_compile)
-
-        contract_name = Path(target).parts[-1]
-
-        source_unit = compilation_unit.create_source_unit(filename)
-
-        source_unit.add_contract_name(contract_name)
-        compilation_unit.filename_to_contracts[filename].add(contract_name)
-        source_unit.abis[contract_name] = info["abi"]
-        source_unit.bytecodes_init[contract_name] = info["bytecode"].replace("0x", "")
-        source_unit.bytecodes_runtime[contract_name] = info["bytecode_runtime"].replace("0x", "")
-        # Vyper does not provide the source mapping for the init bytecode
-        source_unit.srcmaps_init[contract_name] = []
-        # info["source_map"]["pc_pos_map"] contains the source mapping in a simpler format
-        # However pc_pos_map_compressed" seems to follow solc's format, so for convenience
-        # We store the same
-        # TODO: create SourceMapping class, so that srcmaps_runtime would store an class
-        # That will give more flexebility to different compilers
-        source_unit.srcmaps_runtime[contract_name] = info["source_map"]["pc_pos_map_compressed"]
-
-        # Natspec not yet handled for vyper
-        source_unit.natspec[contract_name] = Natspec({}, {})
-
-        ast = _get_vyper_ast(target, vyper)
-        source_unit.ast = ast
+        for source_file, contract_info in compilation_artifacts["contracts"].items():
+            filename = convert_filename(source_file, _relative_to_short, crytic_compile)
+            source_unit = compilation_unit.create_source_unit(filename)
+            for contract_name, contract_metadata in contract_info.items():
+                source_unit.add_contract_name(contract_name)
+                compilation_unit.filename_to_contracts[filename].add(contract_name)
+
+                source_unit.abis[contract_name] = contract_metadata["abi"]
+                source_unit.bytecodes_init[contract_name] = contract_metadata["evm"]["bytecode"][
+                    "object"
+                ].replace("0x", "")
+                # Vyper does not provide the source mapping for the init bytecode
+                source_unit.srcmaps_init[contract_name] = []
+                source_unit.srcmaps_runtime[contract_name] = contract_metadata["evm"][
+                    "deployedBytecode"
+                ]["sourceMap"]
+                source_unit.bytecodes_runtime[contract_name] = contract_metadata["evm"][
+                    "deployedBytecode"
+                ]["object"].replace("0x", "")
+                source_unit.natspec[contract_name] = Natspec(
+                    contract_metadata["userdoc"], contract_metadata["devdoc"]
+                )
+
+        for source_file, ast in compilation_artifacts["sources"].items():
+            filename = convert_filename(source_file, _relative_to_short, crytic_compile)
+            source_unit = compilation_unit.create_source_unit(filename)
+            source_unit.ast = ast
+
+    def add_source_files(self, file_paths: List[str]) -> None:
+        for file_path in file_paths:
+            with open(file_path, "r") as f:
+                self.standard_json_input["sources"][file_path] = {
+                    "content": f.read(),
+                }
 
     def clean(self, **_kwargs: str) -> None:
         """Clean compilation artifacts
@@ -129,13 +167,16 @@ def _guessed_tests(self) -> List[str]:
         return []
 
 
-def _run_vyper(
-    filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None
+def _run_vyper_standard_json(
+    standard_input_path: str,
+    vyper: str,
+    env: Optional[Dict] = None,
+    working_dir: Optional[str] = None,
 ) -> Dict:
-    """Run vyper
+    """Run vyper and write compilation output to a file
 
     Args:
-        filename (str): vyper file
+        standard_input_path (str): path to the standard input json file
         vyper (str): vyper binary
         env (Optional[Dict], optional): Environment variables. Defaults to None.
         working_dir (Optional[str], optional): Working directory. Defaults to None.
@@ -146,81 +187,12 @@ def _run_vyper(
     Returns:
         Dict: Vyper json compilation artifact
     """
-    if not os.path.isfile(filename):
-        raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)")
-
-    cmd = [vyper, filename, "-f", "combined_json"]
-
-    additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
-    stderr = ""
-    LOGGER.info(
-        "'%s' running",
-        " ".join(cmd),
-    )
-    try:
-        with subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            env=env,
-            executable=shutil.which(cmd[0]),
-            **additional_kwargs,
-        ) as process:
-            stdout, stderr = process.communicate()
-            res = stdout.split(b"\n")
-            res = res[-2]
-            return json.loads(res)
-    except OSError as error:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(error)
-    except json.decoder.JSONDecodeError:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}")
-
-
-def _get_vyper_ast(
-    filename: str, vyper: str, env: Optional[Dict] = None, working_dir: Optional[str] = None
-) -> Dict:
-    """Get ast from vyper
-
-    Args:
-        filename (str): vyper file
-        vyper (str): vyper binary
-        env (Dict, optional): Environment variables. Defaults to None.
-        working_dir (str, optional): Working directory. Defaults to None.
-
-    Raises:
-        InvalidCompilation: If vyper failed to run
-
-    Returns:
-        Dict: [description]
-    """
-    if not os.path.isfile(filename):
-        raise InvalidCompilation(f"{filename} does not exist (are you in the correct directory?)")
-
-    cmd = [vyper, filename, "-f", "ast"]
-
-    additional_kwargs: Dict = {"cwd": working_dir} if working_dir else {}
-    stderr = ""
-    try:
-        with subprocess.Popen(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            env=env,
-            executable=shutil.which(cmd[0]),
-            **additional_kwargs,
-        ) as process:
-            stdout, stderr = process.communicate()
-            res = stdout.split(b"\n")
-            res = res[-2]
-            return json.loads(res)
-    except json.decoder.JSONDecodeError:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(f"Invalid vyper compilation\n{stderr}")
-    except Exception as exception:
-        # pylint: disable=raise-missing-from
-        raise InvalidCompilation(exception)
+    cmd = [vyper, standard_input_path, "--standard-json", "-o", "crytic-export/artifacts.json"]
+    success = run(cmd, cwd=working_dir, extra_env=env)
+    if success is None:
+        raise InvalidCompilation("Vyper compilation failed")
+    with open("crytic-export/artifacts.json", "r") as f:
+        return json.load(f)
 
 
 def _relative_to_short(relative: Path) -> Path: