Merge pull request #12397 from ethereum/enable-ext-test-presets-that-compile-but-fail-tests

Enable external test presets that compile but fail tests

Author: chriseth

test/externalTests/colony.sh

@@ -36,12 +36,16 @@ function colony_test
     local repo="https://github.com/solidity-external-tests/colonyNetwork.git"
     local branch=develop_080
     local config_file="truffle.js"
+
+    local compile_only_presets=(
+        ir-no-optimize            # Compiles but tests run out of gas
+        ir-optimize-evm-only      # Compiles but tests run out of gas
+        legacy-no-optimize        # Compiles but tests run out of gas
+        legacy-optimize-evm-only  # Compiles but tests run out of gas
+    )
     local settings_presets=(
-        #ir-no-optimize            # Compiles but tests run out of gas
-        #ir-optimize-evm-only      # Compiles but tests run out of gas
+        "${compile_only_presets[@]}"
         ir-optimize-evm+yul
-        #legacy-no-optimize        # Compiles but tests run out of gas
-        #legacy-optimize-evm-only  # Compiles but tests run out of gas
         legacy-optimize-evm+yul
     )
 
@@ -67,7 +71,7 @@ function colony_test
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for preset in $selected_optimizer_presets; do
-        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
     done
 }
 

test/externalTests/common.sh

@@ -328,12 +328,16 @@ function compile_and_run_test
     local compile_fn="$1"
     local test_fn="$2"
     local verify_fn="$3"
+    local preset="$4"
+    local compile_only_presets="$5"
+
+    [[ $preset != *" "* ]] || assertFail "Preset names must not contain spaces."
 
     printLog "Running compile function..."
     time $compile_fn
     $verify_fn "$SOLCVERSION_SHORT" "$SOLCVERSION"
 
-    if [[ "$COMPILE_ONLY" == 1 ]]; then
+    if [[ "$COMPILE_ONLY" == 1 || " $compile_only_presets " == *" $preset "* ]]; then
         printLog "Skipping test function..."
     else
         printLog "Running test function..."
@@ -347,24 +351,26 @@ function truffle_run_test
     local binary_type="$2"
     local solc_path="$3"
     local preset="$4"
-    local compile_fn="$5"
-    local test_fn="$6"
+    local compile_only_presets="$5"
+    local compile_fn="$6"
+    local test_fn="$7"
 
     truffle_clean
     force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$preset"
-    compile_and_run_test compile_fn test_fn truffle_verify_compiler_version
+    compile_and_run_test compile_fn test_fn truffle_verify_compiler_version "$preset" "$compile_only_presets"
 }
 
 function hardhat_run_test
 {
     local config_file="$1"
     local preset="$2"
-    local compile_fn="$3"
-    local test_fn="$4"
+    local compile_only_presets="$3"
+    local compile_fn="$4"
+    local test_fn="$5"
 
     hardhat_clean
     force_hardhat_compiler_settings "$config_file" "$preset"
-    compile_and_run_test compile_fn test_fn hardhat_verify_compiler_version
+    compile_and_run_test compile_fn test_fn hardhat_verify_compiler_version "$preset" "$compile_only_presets"
 }
 
 function external_test

test/externalTests/ens.sh

@@ -36,7 +36,10 @@ function ens_test
     local repo="https://github.com/ensdomains/ens.git"
     local branch=master
     local config_file="truffle.js"
+
+    local compile_only_presets=()
     local settings_presets=(
+        "${compile_only_presets[@]}"
         #ir-no-optimize           # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack."
         ir-optimize-evm+yul
@@ -65,7 +68,7 @@ function ens_test
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for preset in $selected_optimizer_presets; do
-        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
     done
 }
 

test/externalTests/gnosis-v2.sh

@@ -36,11 +36,15 @@ function gnosis_safe_test
     local repo="https://github.com/solidity-external-tests/safe-contracts.git"
     local branch=v2_080
     local config_file="truffle-config.js"
+
+    local compile_only_presets=(
+        legacy-no-optimize        # "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit"
+    )
     local settings_presets=(
+        "${compile_only_presets[@]}"
         #ir-no-optimize           # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
         ir-optimize-evm+yul
-        #legacy-no-optimize       # "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit"
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
     )
@@ -65,7 +69,7 @@ function gnosis_safe_test
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for preset in $selected_optimizer_presets; do
-        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
     done
 }
 

test/externalTests/gnosis.sh

@@ -36,7 +36,10 @@ function gnosis_safe_test
     local repo="https://github.com/solidity-external-tests/safe-contracts.git"
     local branch=development_080
     local config_file="truffle-config.js"
+
+    local compile_only_presets=()
     local settings_presets=(
+        "${compile_only_presets[@]}"
         #ir-no-optimize            # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
         #ir-optimize-evm-only      # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
         ir-optimize-evm+yul
@@ -64,7 +67,7 @@ function gnosis_safe_test
     [[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
 
     for preset in $selected_optimizer_presets; do
-        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
+        truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
     done
 }
 

test/externalTests/zeppelin.sh

@@ -36,10 +36,14 @@ function zeppelin_test
     local repo="https://github.com/OpenZeppelin/openzeppelin-contracts.git"
     local branch=master
     local config_file="hardhat.config.js"
+
+    local compile_only_presets=(
+        ir-optimize-evm+yul       # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
+    )
     local settings_presets=(
+        "${compile_only_presets[@]}"
         #ir-no-optimize           # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack."
         #ir-optimize-evm-only     # "YulException: Variable var_account_852 is 4 slot(s) too deep inside the stack."
-        #ir-optimize-evm+yul      # Compiles but tests fail. See https://github.com/nomiclabs/hardhat/issues/2115
         legacy-no-optimize
         legacy-optimize-evm-only
         legacy-optimize-evm+yul
@@ -60,7 +64,7 @@ function zeppelin_test
     replace_version_pragmas
 
     for preset in $selected_optimizer_presets; do
-        hardhat_run_test "$config_file" "$preset" compile_fn test_fn
+        hardhat_run_test "$config_file" "$preset" "${compile_only_presets[*]}" compile_fn test_fn
     done
 }