Re-enable running tests in external tests via IR in cases where they don't pass due to Hardhat heuristics

Author: cameel

test/externalTests/elementfi.sh

@@ -42,13 +42,12 @@ function elementfi_test
     local config_file="hardhat.config.ts"
     local config_var=config
 
-    local compile_only_presets=(
-        ir-optimize-evm+yul       # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
-    )
+    local compile_only_presets=()
     local settings_presets=(
         "${compile_only_presets[@]}"
         #ir-no-optimize           # Compilation fails with "YulException: Variable var_amount_9311 is 10 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # Compilation fails with "YulException: Variable var_amount_9311 is 10 slot(s) too deep inside the stack."
+        ir-optimize-evm+yul
         legacy-no-optimize
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
@@ -89,6 +88,13 @@ function elementfi_test
     # TODO: Remove when https://github.com/element-fi/elf-contracts/issues/243 is fixed.
     sed -i 's|^\s*require(_expiration - block\.timestamp < _unitSeconds);\s*$||g' contracts/ConvergentCurvePool.sol
 
+    # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables
+    # them for other presets but that's fine - we want same code run for benchmarks to be comparable.
+    # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115).
+    sed -i 's|it(\("fails to withdraw more shares than in balance"\)|it.skip(\1|g' test/compoundAssetProxyTest.ts
+    sed -i 's|it(\("should prevent withdrawal of Principal Tokens and Interest Tokens before the tranche expires "\)|it.skip(\1|g' test/trancheTest.ts
+    sed -i 's|it(\("should prevent withdrawal of more Principal Tokens and Interest Tokens than the user has"\)|it.skip(\1|g' test/trancheTest.ts
+
     # This test file is very flaky. There's one particular cases that fails randomly (see
     # https://github.com/element-fi/elf-contracts/issues/240) but some others also depends on an external
     # service which makes tests time out when that service is down.

test/externalTests/gnosis.sh

@@ -42,13 +42,12 @@ function gnosis_safe_test
     local config_file="hardhat.config.ts"
     local config_var=userConfig
 
-    local compile_only_presets=(
-        ir-optimize-evm+yul        # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
-    )
+    local compile_only_presets=()
     local settings_presets=(
         "${compile_only_presets[@]}"
         #ir-no-optimize            # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
         #ir-optimize-evm-only      # Compilation fails with "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
+        ir-optimize-evm+yul
         legacy-no-optimize
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
@@ -70,6 +69,11 @@ function gnosis_safe_test
     # TODO: Remove this when Hardhat implements them (https://github.com/nomiclabs/hardhat/issues/2051).
     sed -i "s|\(it\)\(('should revert if called directly', async () => {\)|\1.skip\2|g" test/handlers/CompatibilityFallbackHandler.spec.ts
 
+    # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables
+    # them for other presets but that's fine - we want same code run for benchmarks to be comparable.
+    # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115).
+    sed -i "s|\(it\)\(('should not allow to call setup on singleton'\)|\1.skip\2|g" test/core/GnosisSafe.Setup.spec.ts
+
     neutralize_package_lock
     neutralize_package_json_hooks
     force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"

test/externalTests/prb-math.sh

@@ -43,13 +43,12 @@ function prb_math_test
     local config_file="hardhat.config.ts"
     local config_var="config"
 
-    local compile_only_presets=(
-        ir-optimize-evm+yul       # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
-    )
+    local compile_only_presets=()
     local settings_presets=(
         "${compile_only_presets[@]}"
         #ir-no-optimize           # Compilation fails with "YulException: Variable var_y_1960 is 8 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # Compilation fails with "YulException: Variable var_y_1960 is 8 slot(s) too deep inside the stack."
+        ir-optimize-evm+yul
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
         legacy-no-optimize
@@ -68,6 +67,26 @@ function prb_math_test
     # yarn.lock. Remove the config to restore Yarn 1.x.
     rm .yarnrc.yml
 
+    # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables
+    # them for other presets but that's fine - we want same code run for benchmarks to be comparable.
+    # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115).
+    pushd test/contracts/prbMathUd60x18/pure/
+    sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts
+    sed -i 's|context(\("when the sum does not overflow"\)|context.skip(\1|g' add.test.ts
+    sed -i 's|context(\("when both operands are zero"\)|context.skip(\1|g' avg.test.ts
+    sed -i 's|context(\("when one operand is zero and the other is not zero"\)|context.skip(\1|g' avg.test.ts
+    sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts
+    sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts
+    popd
+    pushd test/contracts/prbMathSd59x18/pure/
+    sed -i 's|context(\("when the sum overflows"\)|context.skip(\1|g' add.test.ts
+    sed -i 's|context(\("when the sum underflows"\)|context.skip(\1|g' add.test.ts
+    sed -i 's|context(\("when the denominator is zero"\)|context.skip(\1|g' div.test.ts
+    sed -i 's|context(\("when x is zero"\)|context.skip(\1|g' inv.test.ts
+    sed -i 's|context(\("when the difference underflows"\)|context.skip(\1|g' sub.test.ts
+    sed -i 's|context(\("when the difference overflows"\)|context.skip(\1|g' sub.test.ts
+    popd
+
     neutralize_package_lock
     neutralize_package_json_hooks
     force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"

test/externalTests/zeppelin.sh

@@ -41,13 +41,12 @@ function zeppelin_test
     local ref="master"
     local config_file="hardhat.config.js"
 
-    local compile_only_presets=(
-        ir-optimize-evm+yul       # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
-    )
+    local compile_only_presets=()
     local settings_presets=(
         "${compile_only_presets[@]}"
         #ir-no-optimize           # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # Compilation fails with "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack."
+        ir-optimize-evm+yul
         legacy-no-optimize
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
@@ -59,6 +58,19 @@ function zeppelin_test
     setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
     download_project "$repo" "$ref_type" "$ref" "$DIR"
 
+    # Disable tests that won't pass on the ir presets due to Hardhat heuristics. Note that this also disables
+    # them for other presets but that's fine - we want same code run for benchmarks to be comparable.
+    # TODO: Remove this when Hardhat adjusts heuristics for IR (https://github.com/nomiclabs/hardhat/issues/2115).
+    pushd test/utils/
+    sed -i "s|it(\('reverts \)|it.skip(\1|g" math/SafeMath.test.js
+    sed -i "s|it(\('reverts \)|it.skip(\1|g" math/SignedSafeMath.test.js
+    sed -i "s|it(\('reverts \)|it.skip(\1|g" structs/EnumerableSet.behavior.js
+    popd
+
+    # In some cases Hardhat does not detect revert reasons properly via IR.
+    # TODO: Remove this when https://github.com/NomicFoundation/hardhat/issues/2453 gets fixed.
+    sed -i "s|it(\('reverts if the current value is 0'\)|it.skip(\1|g" test/utils/Counters.test.js
+
     neutralize_package_json_hooks
     force_hardhat_compiler_binary "$config_file" "$BINARY_TYPE" "$BINARY_PATH"
     force_hardhat_compiler_settings "$config_file" "$(first_word "$SELECTED_PRESETS")"