Merge pull request #12441 from ethereum/benchmarking-ext-tests

Benchmarking external tests

Author: Leo

.circleci/config.yml

@@ -583,7 +583,7 @@ defaults:
       project: uniswap
       binary_type: native
       nodejs_version: '16'
-  - job_native_test_prb_math: &job_native_test_prb_math
+  - job_native_test_ext_prb_math: &job_native_test_ext_prb_math
       <<: *workflow_ubuntu2004_static
       name: t_native_test_ext_prb_math
       project: prb-math
@@ -1206,8 +1206,34 @@ jobs:
                 name: External <<parameters.project>> tests (native)
                 command: |
                   test/externalTests/<<parameters.project>>.sh native /tmp/workspace/solc/solc
+      - store_artifacts:
+          path: reports/externalTests/
+      # persist_to_workspace fails if the directory does not exist and the test script will create
+      # it only if it actually has benchmark results.
+      - run: mkdir -p reports/externalTests/
+      - persist_to_workspace:
+          root: .
+          paths:
+            - reports/externalTests/
       - gitter_notify_failure_unless_pr
 
+  c_ext_benchmarks:
+    <<: *base_node_small
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run:
+          name: Combine benchmark reports
+          command: cat reports/externalTests/benchmark-*.json | scripts/externalTests/merge_benchmarks.sh > reports/externalTests/all-benchmarks.json
+      - run:
+          name: Summarize reports
+          command: cat reports/externalTests/all-benchmarks.json | scripts/externalTests/summarize_benchmarks.sh > reports/externalTests/summarized-benchmarks.json
+      - store_artifacts:
+          path: reports/externalTests/all-benchmarks.json
+      - store_artifacts:
+          path: reports/externalTests/summarized-benchmarks.json
+
   b_win: &b_win
     <<: *base_win_powershell_large
     steps:
@@ -1455,9 +1481,27 @@ workflows:
       - t_ems_ext: *job_native_test_ext_pool_together
       - t_ems_ext: *job_native_test_ext_perpetual_pools
       - t_ems_ext: *job_native_test_ext_uniswap
-      - t_ems_ext: *job_native_test_prb_math
+      - t_ems_ext: *job_native_test_ext_prb_math
       - t_ems_ext: *job_native_test_ext_elementfi
 
+      - c_ext_benchmarks:
+          <<: *workflow_trigger_on_tags
+          requires:
+            - t_ems_compile_ext_colony
+            - t_native_compile_ext_gnosis
+            - t_native_test_ext_gnosis_v2
+            - t_native_test_ext_zeppelin
+            - t_native_test_ext_ens
+            - t_native_test_ext_trident
+            - t_native_test_ext_euler
+            - t_native_test_ext_yield_liquidator
+            - t_native_test_ext_bleeps
+            - t_native_test_ext_pool_together
+            - t_native_test_ext_perpetual_pools
+            - t_native_test_ext_uniswap
+            - t_native_test_ext_prb_math
+            - t_native_test_ext_elementfi
+
       # Windows build and tests
       - b_win: *workflow_trigger_on_tags
       - b_win_release: *workflow_trigger_on_tags

scripts/externalTests/merge_benchmarks.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# ------------------------------------------------------------------------------
+# Reads multiple individual benchmark reports produced by scripts from
+# test/externalTests/ from standard input and creates a combined report.
+#
+# Usage:
+#    <script name>.sh < <CONCATENATED_REPORTS>
+#
+# CONCATENATED_REPORTS: JSON report files concatenated into a single stream (e.g. using cat).
+#
+# Example:
+#    cat reports/externalTests/benchmark-*.json | <script name>.sh
+# ------------------------------------------------------------------------------
+# This file is part of solidity.
+#
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2021 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -euo pipefail
+
+# We expect a series of dicts of the form {"<project>": {"<preset>": {...}}}.
+# Unfortunately jq's built-in `add` filter can't handle nested dicts and
+# would just overwrite values sharing a project name instead of merging them.
+
+# This is done by first grouping the dicts into an array of the form
+# [
+#     [{"key": "<project1>", "value": {"<preset1>": {...}}}, {"key": "<project1>", "value": {"<preset2>": {...}}, ...],
+#     [{"key": "<project2>", "value": {"<preset1>": {...}}}, {"key": "<project2>", "value": {"<preset2>": {...}}, ...],
+#     ...
+# ]
+# and then using reduce() on each group sharing the same project name to convert it into a
+# dict having preset names as keys.
+jq --slurp  --indent 4 --sort-keys '
+    map(to_entries[]) |
+    group_by(.key) |
+    map({
+        (.[0].key): (
+            reduce (.[].value | to_entries[]) as {$key, $value} (
+                {}; . + {
+                    ($key): $value
+                }
+            )
+        )
+    }) |
+    add
+'