Skip to content

Commit 8da9239

Browse files
chrisethchriseth
authored
Merge pull request #12396 from ethereum/ext-tests-via-ir
External tests via IR
2 parents 08abab1 + 3ec05d0 commit 8da9239

File tree

8 files changed

+126
-81
lines changed

8 files changed

+126
-81
lines changed

‎.circleci/config.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1054,8 +1054,9 @@ jobs:
10541054
docker:
10551055
- image: circleci/node:<<parameters.nodejs_version>>
10561056
resource_class: <<parameters.resource_class>>
1057-
# NOTE: Each external test does 3 separate compile&test runs
1058-
parallelism: 3
1057+
# NOTE: Each external test runs up to 6 independent settings presets. If parallelism is higher than
1058+
# actual preset count, some runs will exit immediately. If it's lower, some runs will get more than one preset.
1059+
parallelism: 6
10591060
environment:
10601061
TERM: xterm
10611062
COMPILE_ONLY: <<parameters.compile_only>>

‎scripts/common.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ function safe_kill
211211

212212
function circleci_select_steps
213213
{
214+
# We expect multiple lines in $all_steps, one step per line
214215
local all_steps="$1"
215216
(( $# == 1 )) || assertFail
216217

@@ -222,11 +223,26 @@ function circleci_select_steps
222223
fi
223224
}
224225

226+
function circleci_select_steps_multiarg
227+
{
228+
# We expect multiple arguments, one step per argument.
229+
circleci_select_steps "$(printf '%s\n' "$@")"
230+
}
231+
225232
function circleci_step_selected
226233
{
227234
local selected_steps="$1"
228235
local step="$2"
236+
(( $# == 2 )) || assertFail
229237
[[ $step != *" "* ]] || assertFail "Step names must not contain spaces."
230238

231239
[[ " $selected_steps " == *" $step "* ]] || return 1
232240
}
241+
242+
function first_word
243+
{
244+
local words="$1"
245+
(( $# == 1 )) || assertFail
246+
247+
echo "$words" | cut -d " " -f 1
248+
}

‎test/externalTests/colony.sh

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,25 @@ function colony_test
3636
local repo="https://github.com/solidity-external-tests/colonyNetwork.git"
3737
local branch=develop_080
3838
local config_file="truffle.js"
39-
# On levels 1 and 2 it compiles but tests run out of gas
40-
local min_optimizer_level=3
41-
local max_optimizer_level=3
39+
local settings_presets=(
40+
#ir-no-optimize # Compiles but tests run out of gas
41+
#ir-optimize-evm-only # Compiles but tests run out of gas
42+
ir-optimize-evm+yul
43+
#legacy-no-optimize # Compiles but tests run out of gas
44+
#legacy-optimize-evm-only # Compiles but tests run out of gas
45+
legacy-optimize-evm+yul
46+
)
4247

43-
local selected_optimizer_levels
44-
selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
45-
print_optimizer_levels_or_exit "$selected_optimizer_levels"
48+
local selected_optimizer_presets
49+
selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}")
50+
print_optimizer_presets_or_exit "$selected_optimizer_presets"
4651

4752
setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
4853
download_project "$repo" "$branch" "$DIR"
4954
[[ $BINARY_TYPE == native ]] && replace_global_solc "$BINARY_PATH"
5055

5156
neutralize_package_json_hooks
52-
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
57+
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")"
5358
yarn install
5459
git submodule update --init
5560

@@ -61,8 +66,8 @@ function colony_test
6166
replace_version_pragmas
6267
[[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
6368

64-
for level in $selected_optimizer_levels; do
65-
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
69+
for preset in $selected_optimizer_presets; do
70+
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
6671
done
6772
}
6873

‎test/externalTests/common.sh

Lines changed: 37 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,13 @@ set -e
2424

2525
CURRENT_EVM_VERSION=london
2626

27-
function print_optimizer_levels_or_exit
27+
function print_optimizer_presets_or_exit
2828
{
29-
local selected_levels="$1"
29+
local selected_presets="$1"
3030

31-
[[ $selected_levels != "" ]] || { printWarning "No steps to run. Exiting."; exit 0; }
31+
[[ $selected_presets != "" ]] || { printWarning "No presets to run. Exiting."; exit 0; }
3232

33-
printLog "Selected optimizer levels: ${selected_levels}"
33+
printLog "Selected settings presets: ${selected_presets}"
3434
}
3535

3636
function verify_input
@@ -141,7 +141,7 @@ function force_truffle_compiler_settings
141141
local config_file="$1"
142142
local binary_type="$2"
143143
local solc_path="$3"
144-
local level="$4"
144+
local preset="$4"
145145
local evm_version="${5:-"$CURRENT_EVM_VERSION"}"
146146

147147
[[ $binary_type == native || $binary_type == solcjs ]] || assertFail
@@ -153,14 +153,16 @@ function force_truffle_compiler_settings
153153
echo "Config file: $config_file"
154154
echo "Binary type: $binary_type"
155155
echo "Compiler path: $solc_path"
156-
echo "Optimization level: $level"
157-
echo "Optimizer settings: $(optimizer_settings_for_level "$level")"
156+
echo "Settings preset: ${preset}"
157+
echo "Settings: $(settings_from_preset "$preset" "$evm_version")"
158158
echo "EVM version: $evm_version"
159+
echo "Compiler version: ${SOLCVERSION_SHORT}"
160+
echo "Compiler version (full): ${SOLCVERSION}"
159161
echo "-------------------------------------"
160162

161163
# Forcing the settings should always work by just overwriting the solc object. Forcing them by using a
162164
# dedicated settings objects should only be the fallback.
163-
echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$level" "$evm_version");" >> "$config_file"
165+
echo "module.exports['compilers'] = $(truffle_compiler_settings "$solc_path" "$preset" "$evm_version");" >> "$config_file"
164166
}
165167

166168
function force_hardhat_compiler_binary
@@ -180,22 +182,22 @@ function force_hardhat_compiler_binary
180182
function force_hardhat_compiler_settings
181183
{
182184
local config_file="$1"
183-
local level="$2"
185+
local preset="$2"
184186
local evm_version="${3:-"$CURRENT_EVM_VERSION"}"
185187

186188
printLog "Configuring Hardhat..."
187189
echo "-------------------------------------"
188190
echo "Config file: ${config_file}"
189-
echo "Optimization level: ${level}"
190-
echo "Optimizer settings: $(optimizer_settings_for_level "$level")"
191+
echo "Settings preset: ${preset}"
192+
echo "Settings: $(settings_from_preset "$preset" "$evm_version")"
191193
echo "EVM version: ${evm_version}"
192194
echo "Compiler version: ${SOLCVERSION_SHORT}"
193195
echo "Compiler version (full): ${SOLCVERSION}"
194196
echo "-------------------------------------"
195197

196198
{
197199
echo -n 'module.exports["solidity"] = '
198-
hardhat_compiler_settings "$SOLCVERSION_SHORT" "$level" "$evm_version"
200+
hardhat_compiler_settings "$SOLCVERSION_SHORT" "$preset" "$evm_version"
199201
} >> "$config_file"
200202
}
201203

@@ -236,22 +238,27 @@ function run_test
236238
replace_version_pragmas
237239

238240
printLog "Running compile function..."
239-
$compile_fn
241+
time $compile_fn
240242

241243
printLog "Running test function..."
242244
$test_fn
243245
}
244246

245-
function optimizer_settings_for_level
247+
function settings_from_preset
246248
{
247-
local level="$1"
248-
249-
case "$level" in
250-
1) echo "{enabled: false}" ;;
251-
2) echo "{enabled: true, details: {yul: false}}" ;;
252-
3) echo "{enabled: true, details: {yul: true}}" ;;
249+
local preset="$1"
250+
local evm_version="$2"
251+
252+
case "$preset" in
253+
# NOTE: Remember to update `parallelism` of `t_ems_ext` job in CI config if you add/remove presets
254+
legacy-no-optimize) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: false}}" ;;
255+
ir-no-optimize) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: false}}" ;;
256+
legacy-optimize-evm-only) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: true, details: {yul: false}}}" ;;
257+
ir-optimize-evm-only) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: true, details: {yul: false}}}" ;;
258+
legacy-optimize-evm+yul) echo "{evmVersion: '${evm_version}', viaIR: false, optimizer: {enabled: true, details: {yul: true}}}" ;;
259+
ir-optimize-evm+yul) echo "{evmVersion: '${evm_version}', viaIR: true, optimizer: {enabled: true, details: {yul: true}}}" ;;
253260
*)
254-
fail "Optimizer level not found. Please define OPTIMIZER_LEVEL=[1, 2, 3]"
261+
fail "Unknown settings preset: '${preset}'."
255262
;;
256263
esac
257264
}
@@ -269,16 +276,13 @@ function replace_global_solc
269276
function truffle_compiler_settings
270277
{
271278
local solc_path="$1"
272-
local level="$2"
279+
local preset="$2"
273280
local evm_version="$3"
274281

275282
echo "{"
276283
echo " solc: {"
277284
echo " version: \"${solc_path}\","
278-
echo " settings: {"
279-
echo " optimizer: $(optimizer_settings_for_level "$level"),"
280-
echo " evmVersion: \"${evm_version}\""
281-
echo " }"
285+
echo " settings: $(settings_from_preset "$preset" "$evm_version")"
282286
echo " }"
283287
echo "}"
284288
}
@@ -310,15 +314,12 @@ function hardhat_solc_build_subtask {
310314

311315
function hardhat_compiler_settings {
312316
local solc_version="$1"
313-
local level="$2"
317+
local preset="$2"
314318
local evm_version="$3"
315319

316320
echo "{"
317321
echo " version: '${solc_version}',"
318-
echo " settings: {"
319-
echo " optimizer: $(optimizer_settings_for_level "$level"),"
320-
echo " evmVersion: '${evm_version}'"
321-
echo " }"
322+
echo " settings: $(settings_from_preset "$preset" "$evm_version")"
322323
echo "}"
323324
}
324325

@@ -329,7 +330,7 @@ function compile_and_run_test
329330
local verify_fn="$3"
330331

331332
printLog "Running compile function..."
332-
$compile_fn
333+
time $compile_fn
333334
$verify_fn "$SOLCVERSION_SHORT" "$SOLCVERSION"
334335

335336
if [[ "$COMPILE_ONLY" == 1 ]]; then
@@ -345,24 +346,24 @@ function truffle_run_test
345346
local config_file="$1"
346347
local binary_type="$2"
347348
local solc_path="$3"
348-
local optimizer_level="$4"
349+
local preset="$4"
349350
local compile_fn="$5"
350351
local test_fn="$6"
351352

352353
truffle_clean
353-
force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$optimizer_level"
354+
force_truffle_compiler_settings "$config_file" "$binary_type" "$solc_path" "$preset"
354355
compile_and_run_test compile_fn test_fn truffle_verify_compiler_version
355356
}
356357

357358
function hardhat_run_test
358359
{
359360
local config_file="$1"
360-
local optimizer_level="$2"
361+
local preset="$2"
361362
local compile_fn="$3"
362363
local test_fn="$4"
363364

364365
hardhat_clean
365-
force_hardhat_compiler_settings "$config_file" "$optimizer_level"
366+
force_hardhat_compiler_settings "$config_file" "$preset"
366367
compile_and_run_test compile_fn test_fn hardhat_verify_compiler_version
367368
}
368369

‎test/externalTests/ens.sh

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,18 @@ function ens_test
3636
local repo="https://github.com/ensdomains/ens.git"
3737
local branch=master
3838
local config_file="truffle.js"
39-
local min_optimizer_level=1
40-
local max_optimizer_level=3
39+
local settings_presets=(
40+
#ir-no-optimize # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack."
41+
#ir-optimize-evm-only # "YulException: Variable var_ttl_236 is 1 slot(s) too deep inside the stack."
42+
ir-optimize-evm+yul
43+
legacy-no-optimize
44+
legacy-optimize-evm-only
45+
legacy-optimize-evm+yul
46+
)
4147

42-
local selected_optimizer_levels
43-
selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
44-
print_optimizer_levels_or_exit "$selected_optimizer_levels"
48+
local selected_optimizer_presets
49+
selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}")
50+
print_optimizer_presets_or_exit "$selected_optimizer_presets"
4551

4652
setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
4753
download_project "$repo" "$branch" "$DIR"
@@ -52,14 +58,14 @@ function ens_test
5258

5359
neutralize_package_lock
5460
neutralize_package_json_hooks
55-
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
61+
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")"
5662
npm install
5763

5864
replace_version_pragmas
5965
[[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
6066

61-
for level in $selected_optimizer_levels; do
62-
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
67+
for preset in $selected_optimizer_presets; do
68+
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
6369
done
6470
}
6571

‎test/externalTests/gnosis-v2.sh

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,18 @@ function gnosis_safe_test
3636
local repo="https://github.com/solidity-external-tests/safe-contracts.git"
3737
local branch=v2_080
3838
local config_file="truffle-config.js"
39-
# level 1: "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit"
40-
local min_optimizer_level=2
41-
local max_optimizer_level=3
39+
local settings_presets=(
40+
#ir-no-optimize # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
41+
#ir-optimize-evm-only # "YulException: Variable var_call_430_mpos is 1 slot(s) too deep inside the stack."
42+
ir-optimize-evm+yul
43+
#legacy-no-optimize # "Error: while migrating GnosisSafe: Returned error: base fee exceeds gas limit"
44+
legacy-optimize-evm-only
45+
legacy-optimize-evm+yul
46+
)
4247

43-
local selected_optimizer_levels
44-
selected_optimizer_levels=$(circleci_select_steps "$(seq "$min_optimizer_level" "$max_optimizer_level")")
45-
print_optimizer_levels_or_exit "$selected_optimizer_levels"
48+
local selected_optimizer_presets
49+
selected_optimizer_presets=$(circleci_select_steps_multiarg "${settings_presets[@]}")
50+
print_optimizer_presets_or_exit "$selected_optimizer_presets"
4651

4752
setup_solc "$DIR" "$BINARY_TYPE" "$BINARY_PATH"
4853
download_project "$repo" "$branch" "$DIR"
@@ -53,14 +58,14 @@ function gnosis_safe_test
5358

5459
neutralize_package_lock
5560
neutralize_package_json_hooks
56-
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$min_optimizer_level"
61+
force_truffle_compiler_settings "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$(first_word "$selected_optimizer_presets")"
5762
npm install --package-lock
5863

5964
replace_version_pragmas
6065
[[ $BINARY_TYPE == solcjs ]] && force_solc_modules "${DIR}/solc"
6166

62-
for level in $selected_optimizer_levels; do
63-
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$level" compile_fn test_fn
67+
for preset in $selected_optimizer_presets; do
68+
truffle_run_test "$config_file" "$BINARY_TYPE" "${DIR}/solc" "$preset" compile_fn test_fn
6469
done
6570
}
6671

0 commit comments

Comments
 (0)