Merge branch 'dev' into foundry-multiple-compilation-units-mypy

Author: montyly

.github/dependabot.yml

@@ -1,3 +1,4 @@
+---
 version: 2
 updates:
     - package-ecosystem: "github-actions"

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "windows-2022"]
-        type: ["brownie", "buidler", "dapp", "embark", "etherlime", "hardhat", "solc", "truffle", "waffle", "foundry"]
+        type: ["brownie", "buidler", "dapp", "embark", "etherlime", "hardhat", "solc", "truffle", "waffle", "foundry", "standard"]
         exclude:
           # Currently broken, tries to pull git:// which is blocked by GH
           - type: embark
@@ -39,13 +39,13 @@ jobs:
       id: node
       shell: bash
       run: |
-        if [ ${{ matrix.type }} = etherlime ]; then
-          echo '::set-output name=version::10.17.0'
+        if [ "${{ matrix.type }}" = "etherlime" ]; then
+          echo 'version=10.17.0' >> "$GITHUB_OUTPUT"
         else
-          echo '::set-output name=version::lts/*'
+          echo 'version=lts/*' >> "$GITHUB_OUTPUT"
         fi
     - name: Set up Node
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v3
       with:
         node-version: ${{ steps.node.outputs.version }}
     - name: Set up Python 3.8
@@ -69,6 +69,7 @@ jobs:
       env:
         TEST_TYPE: ${{ matrix.type }}
         GITHUB_ETHERSCAN: ${{ secrets.GITHUB_ETHERSCAN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       shell: bash
       run: |
         bash "scripts/ci_test_${TEST_TYPE}.sh"

.github/workflows/pytest.yml

@@ -1,3 +1,4 @@
+---
 name: Pytest
 
 defaults:
@@ -14,16 +15,16 @@ on:
     branches: [main, dev]
   schedule:
     # run CI every day even if no PRs/merges occur
-    - cron:  '0 12 * * *'
+    - cron: '0 12 * * *'
 
 jobs:
   tests:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
 
@@ -35,4 +36,4 @@ jobs:
         pip install solc-select
     - name: Run Tests
       run: |
-        pytest tests/test_metadata.py
+        pytest tests/test_metadata.py

crytic_compile/__main__.py

@@ -13,6 +13,7 @@
 from crytic_compile.crytic_compile import compile_all, get_platforms
 from crytic_compile.cryticparser import DEFAULTS_FLAG_IN_CONFIG, cryticparser
 from crytic_compile.platform import InvalidCompilation
+from crytic_compile.platform.all_export import PLATFORMS_EXPORT
 from crytic_compile.utils.zip import ZIP_TYPES_ACCEPTED, save_to_zip
 
 if TYPE_CHECKING:
@@ -50,8 +51,8 @@ def parse_args() -> argparse.Namespace:
 
     parser.add_argument(
         "--export-format",
-        help="""Export json with non crytic-compile format
-(default None. Accepted: standard, solc, truffle)""",
+        help=f"""Export json with non crytic-compile format
+            (default None. Accepted: ({", ".join(list(PLATFORMS_EXPORT))})""",
         action="store",
         dest="export_format",
         default=None,

crytic_compile/compilation_unit.py

@@ -3,8 +3,7 @@
 """
 import uuid
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, Set
-
+from typing import TYPE_CHECKING, Dict, Set, Optional
 
 from crytic_compile.compiler.compiler import CompilerVersion
 from crytic_compile.source_unit import SourceUnit
@@ -35,6 +34,9 @@ def __init__(self, crytic_compile: "CryticCompile", unique_id: str):
         # set containing all the filenames of this compilation unit
         self._filenames: Set[Filename] = set()
 
+        # mapping from absolute/relative/used to filename
+        self._filenames_lookup: Optional[Dict[str, Filename]] = None
+
         # compiler.compiler
         self._compiler_version: CompilerVersion = CompilerVersion(
             compiler="N/A", version="N/A", optimized=False
@@ -118,7 +120,6 @@ def create_source_unit(self, filename: Filename) -> SourceUnit:
         if not filename in self._source_units:
             source_unit = SourceUnit(self, filename)  # type: ignore
             self.filenames.add(filename)
-            self.crytic_compile.filenames.add(filename)
             self._source_units[filename] = source_unit
         return self._source_units[filename]
 
@@ -194,6 +195,36 @@ def relative_filename_from_absolute_filename(self, absolute_filename: str) -> st
             raise ValueError("f{absolute_filename} does not exist in {d}")
         return d_file[absolute_filename]
 
+    def filename_lookup(self, filename: str) -> Filename:
+        """Return a crytic_compile.naming.Filename from a any filename
+
+        Args:
+            filename (str): filename (used/absolute/relative)
+
+        Raises:
+            ValueError: If the filename is not in the project
+
+        Returns:
+            Filename: Associated Filename object
+        """
+        # pylint: disable=import-outside-toplevel
+        from crytic_compile.platform.truffle import Truffle
+
+        if isinstance(self.crytic_compile.platform, Truffle) and filename.startswith("project:/"):
+            filename = filename[len("project:/") :]
+
+        if self._filenames_lookup is None:
+            self._filenames_lookup = {}
+            for file in self._filenames:
+                self._filenames_lookup[file.absolute] = file
+                self._filenames_lookup[file.relative] = file
+                self._filenames_lookup[file.used] = file
+        if filename not in self._filenames_lookup:
+            raise ValueError(
+                f"{filename} does not exist in {[f.absolute for f in self._filenames_lookup.values()]}"
+            )
+        return self._filenames_lookup[filename]
+
     # endregion
     ###################################################################################
     ###################################################################################

crytic_compile/crytic_compile.py

@@ -19,7 +19,6 @@
 from crytic_compile.platform.all_export import PLATFORMS_EXPORT
 from crytic_compile.platform.solc import Solc
 from crytic_compile.platform.standard import export_to_standard
-from crytic_compile.platform.truffle import Truffle
 from crytic_compile.utils.naming import Filename
 from crytic_compile.utils.npm import get_package_name
 from crytic_compile.utils.zip import load_from_zip
@@ -78,12 +77,6 @@ def __init__(self, target: Union[str, AbstractPlatform], **kwargs: str) -> None:
         # dependencies is needed for platform conversion
         self._dependencies: Set = set()
 
-        # set containing all the filenames
-        self._filenames: Set[Filename] = set()
-
-        # mapping from absolute/relative/used to filename
-        self._filenames_lookup: Optional[Dict[str, Filename]] = None
-
         self._src_content: Dict = {}
 
         # Mapping each file to
@@ -152,27 +145,21 @@ def is_in_multiple_compilation_unit(self, contract: str) -> bool:
 
     ###################################################################################
     ###################################################################################
-    # region Filenames
+    # region Utils
     ###################################################################################
     ###################################################################################
-
     @property
     def filenames(self) -> Set[Filename]:
-        """All the project filenames
-
-        Returns:
-            Set[Filename]: Project's filenames
         """
-        return self._filenames
+        Return the set of all the filenames used
 
-    @filenames.setter
-    def filenames(self, all_filenames: Set[Filename]) -> None:
-        """Set the filenames
-
-        Args:
-            all_filenames (Set[Filename]): New filenames
+        Returns:
+             Set[Filename]: list of filenames
         """
-        self._filenames = all_filenames
+        filenames: Set[Filename] = set()
+        for compile_unit in self._compilation_units.values():
+            filenames = filenames.union(compile_unit.filenames)
+        return filenames
 
     def filename_lookup(self, filename: str) -> Filename:
         """Return a crytic_compile.naming.Filename from a any filename
@@ -186,21 +173,13 @@ def filename_lookup(self, filename: str) -> Filename:
         Returns:
             Filename: Associated Filename object
         """
+        for compile_unit in self.compilation_units.values():
+            try:
+                return compile_unit.filename_lookup(filename)
+            except ValueError:
+                pass
 
-        if isinstance(self.platform, Truffle) and filename.startswith("project:/"):
-            filename = filename[len("project:/") :]
-
-        if self._filenames_lookup is None:
-            self._filenames_lookup = {}
-            for file in self._filenames:
-                self._filenames_lookup[file.absolute] = file
-                self._filenames_lookup[file.relative] = file
-                self._filenames_lookup[file.used] = file
-        if filename not in self._filenames_lookup:
-            raise ValueError(
-                f"{filename} does not exist in {[f.absolute for f in self._filenames_lookup.values()]}"
-            )
-        return self._filenames_lookup[filename]
+        raise ValueError(f"{filename} does not exist")
 
     @property
     def dependencies(self) -> Set[str]:

crytic_compile/cryticparser/defaults.py

@@ -40,8 +40,8 @@
     "buidler_cache_directory": "cache",
     "buidler_skip_directory_name_fix": False,
     "hardhat_ignore_compile": False,
-    "hardhat_cache_directory": "cache",
-    "hardhat_artifacts_directory": "artifacts",
+    "hardhat_cache_directory": None,
+    "hardhat_artifacts_directory": None,
     "foundry_ignore_compile": False,
     "foundry_out_directory": "out",
     "export_dir": "crytic-export",

crytic_compile/platform/etherscan.py

@@ -152,7 +152,9 @@ def _handle_multiple_files(
     filtered_paths: List[str] = []
     for filename, source_code in source_codes.items():
         path_filename = PurePosixPath(filename)
-        if "contracts" in path_filename.parts and not filename.startswith("@"):
+        # https://etherscan.io/address/0x19bb64b80cbf61e61965b0e5c2560cc7364c6546#code has an import of erc721a/contracts/ERC721A.sol
+        # if the full path is lost then won't compile
+        if "contracts" == path_filename.parts[0] and not filename.startswith("@"):
             path_filename = PurePosixPath(
                 *path_filename.parts[path_filename.parts.index("contracts") :]
             )

crytic_compile/platform/hardhat.py

@@ -7,7 +7,7 @@
 import shutil
 import subprocess
 from pathlib import Path
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 from crytic_compile.compiler.compiler import CompilerVersion
 from crytic_compile.platform.exceptions import InvalidCompilation
@@ -49,7 +49,7 @@ def hardhat_like_parsing(
     )
     files = [str(f) for f in files if str(f).endswith(".json")]
     if not files:
-        txt = f"`hardhat compile` failed. Can you run it?\n{build_directory} is empty"
+        txt = f"`compile` failed. Can you run it?\n{build_directory} is empty"
         raise InvalidCompilation(txt)
 
     for file in files:
@@ -157,16 +157,20 @@ def compile(self, crytic_compile: "CryticCompile", **kwargs: str) -> None:
             "ignore_compile", False
         )
 
-        build_directory = Path(
-            self._target, kwargs.get("hardhat_artifacts_directory", "artifacts"), "build-info"
-        )
-
-        hardhat_working_dir: str = kwargs.get("hardhat_working_dir", self._target)
-
         base_cmd = ["hardhat"]
         if not kwargs.get("npx_disable", False):
             base_cmd = ["npx"] + base_cmd
 
+        detected_paths = self._get_hardhat_paths(base_cmd, kwargs)
+
+        build_directory = Path(
+            self._target,
+            detected_paths["artifacts"],
+            "build-info",
+        )
+
+        hardhat_working_dir = str(Path(self._target, detected_paths["root"]))
+
         if not hardhat_ignore_compile:
             cmd = base_cmd + ["compile", "--force"]
 
@@ -235,3 +239,76 @@ def _guessed_tests(self) -> List[str]:
             List[str]: The guessed unit tests commands
         """
         return ["hardhat test"]
+
+    def _get_hardhat_paths(
+        self, base_cmd: List[str], args: Dict[str, str]
+    ) -> Dict[str, Union[Path, str]]:
+        """Obtain hardhat configuration paths, defaulting to the
+        standard config if needed.
+
+        Args:
+            base_cmd ([str]): hardhat command
+            args (Dict[str, str]): crytic-compile options that may affect paths
+
+        Returns:
+            Dict[str, str]: hardhat paths configuration
+        """
+        target_path = Path(self._target)
+        default_paths = {
+            "root": target_path,
+            "configFile": target_path.joinpath("hardhat.config.js"),
+            "sources": target_path.joinpath("contracts"),
+            "cache": target_path.joinpath("cache"),
+            "artifacts": target_path.joinpath("artifacts"),
+            "tests": target_path.joinpath("test"),
+        }
+        override_paths = {}
+
+        if args.get("hardhat_cache_directory", None):
+            override_paths["cache"] = Path(target_path, args["hardhat_cache_directory"])
+
+        if args.get("hardhat_artifacts_directory", None):
+            override_paths["artifacts"] = Path(target_path, args["hardhat_artifacts_directory"])
+
+        if args.get("hardhat_working_dir", None):
+            override_paths["root"] = Path(target_path, args["hardhat_working_dir"])
+
+        print_paths = "console.log(JSON.stringify(config.paths))"
+        config_str = self._run_hardhat_console(base_cmd, print_paths)
+
+        try:
+            paths = json.loads(config_str or "{}")
+            return {**default_paths, **paths, **override_paths}
+        except ValueError as e:
+            LOGGER.info("Problem deserializing hardhat configuration: %s", e)
+            return {**default_paths, **override_paths}
+
+    def _run_hardhat_console(self, base_cmd: List[str], command: str) -> Optional[str]:
+        """Run a JS command in the hardhat console
+
+        Args:
+            base_cmd ([str]): hardhat command
+            command (str): console command to run
+
+        Returns:
+            Optional[str]: command output if execution succeeds
+        """
+        with subprocess.Popen(
+            base_cmd + ["console", "--no-compile"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=self._target,
+            executable=shutil.which(base_cmd[0]),
+        ) as process:
+            stdout_bytes, stderr_bytes = process.communicate(command.encode("utf-8"))
+            stdout, stderr = (
+                stdout_bytes.decode(),
+                stderr_bytes.decode(),
+            )
+
+            if stderr:
+                LOGGER.info("Problem executing hardhat: %s", stderr)
+                return None
+
+            return stdout