diff --git a/.github/workflows/kute_multi_self.yml b/.github/workflows/kute_multi_self.yml index a4546f469..0aae2968d 100644 --- a/.github/workflows/kute_multi_self.yml +++ b/.github/workflows/kute_multi_self.yml @@ -90,7 +90,7 @@ jobs: python3 report_html.py --resultsPath results --clients "${{ github.event.inputs.client }}" --testsPath ${{ github.event.inputs.test }} --runs $runs - name: Zip the results folder - run: zip -r results.zip reports + run: zip -r results.zip extracted-results/results/reports - name: Upload results as artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/multi-parallel.yml b/.github/workflows/multi-parallel.yml index 4a18da877..2f222aa6a 100644 --- a/.github/workflows/multi-parallel.yml +++ b/.github/workflows/multi-parallel.yml @@ -34,10 +34,10 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.x @@ -199,7 +199,7 @@ jobs: - name: Zip the results folder run: | - zip -r reports.zip reports + zip -r reports.zip extracted-results/results/reports - name: Upload artifact uses: actions/upload-artifact@v3 diff --git a/README.md b/README.md index 2c0b7aef0..bc45bc680 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ bash run.sh -t "testPath" -w "warmupFilePath" -c "client1,client2" -r runNumber Example run: ```shell -run.sh -t "tests/" -w "warmup/warmup-1000bl-16wi-24tx.txt" -c "nethermind,geth,reth" -r 8 +bash run.sh -t "tests/" -w "warmup/warmup-1000bl-16wi-24tx.txt" -c "nethermind,geth,reth" -r 8 ``` Flags: @@ -61,6 +61,7 @@ Flags: - `--c` it's used to define the clients that you want to run the benchmarks. Separate the clients with a comma. - `--r` it's used to define the number of iterations that you want to run the benchmarks. It's a numeric value. - `--i` it's used to define the images that you want to use to run the benchmarks. Separate the images with a comma, and match the clients. Use `default` if you want to ignore the values. +- `--o` it’s used to define the output directory where the results will be saved. Now you're ready to run the benchmarks locally! \ No newline at end of file diff --git a/report_html.py b/report_html.py index e67b051b7..af15569d1 100644 --- a/report_html.py +++ b/report_html.py @@ -149,13 +149,11 @@ def get_html_report(client_results, clients, results_paths, test_cases, methods, soup = BeautifulSoup(results_to_print, 'lxml') formatted_html = soup.prettify() print(formatted_html) - if not os.path.exists('reports'): - os.mkdir('reports') - with open(f'reports/index.html', 'w') as file: + with open(f'{results_paths}/reports/index.html', 'w') as file: file.write(formatted_html) for client, gas_table in csv_table.items(): - with open(f'reports/output_{client}.csv', 'w', newline='') as csvfile: + with open(f'{results_paths}/reports/output_{client}.csv', 'w', newline='') as csvfile: # Create a CSV writer object csvwriter = csv.writer(csvfile) csvwriter.writerow( diff --git a/report_tables.py b/report_tables.py index 0f8192bdf..edd053956 100644 --- a/report_tables.py +++ b/report_tables.py @@ -37,9 +37,7 @@ def get_table_report(client_results, clients, results_paths, test_cases, methods results_to_print += '\n' print(results_to_print) - if not os.path.exists('reports'): - os.mkdir('reports') - with open(f'reports/tables_norm.txt', 'w') as file: + with open(f'{results_paths}/reports/tables_norm.txt', 'w') as file: file.write(results_to_print) diff --git a/run.sh b/run.sh index bed2d401b..47504933e 100755 --- a/run.sh +++ b/run.sh @@ -6,16 +6,18 @@ WARMUP_FILE="warmup/warmup-1000bl-16wi-24tx.txt" CLIENTS="nethermind,geth,reth" RUNS=8 IMAGES="default" +OUTPUT_DIR="results" # Parse command line arguments -while getopts "t:w:c:r:i:" opt; do +while getopts "t:w:c:r:i:o:" opt; do case $opt in t) TEST_PATH="$OPTARG" ;; w) WARMUP_FILE="$OPTARG" ;; c) CLIENTS="$OPTARG" ;; r) RUNS="$OPTARG" ;; i) IMAGES="$OPTARG" ;; - *) echo "Usage: $0 [-t test_path] [-w warmup_file] [-c clients] [-r runs] [-i images]" >&2 + o) OUTPUT_DIR="$OPTARG" ;; + *) echo "Usage: $0 [-t test_path] [-w warmup_file] [-c clients] [-r runs] [-i images] [-o output_dir]" >&2 exit 1 ;; esac done @@ -24,7 +26,7 @@ IFS=',' read -ra CLIENT_ARRAY <<< "$CLIENTS" IFS=',' read -ra IMAGE_ARRAY <<< "$IMAGES" # Set up environment -mkdir -p results +mkdir -p "$OUTPUT_DIR" # Install dependencies pip install -r requirements.txt @@ -46,10 +48,10 @@ for run in $(seq 1 $RUNS); do if [ -z "$WARMUP_FILE" ]; then echo "Running script without warm up." - python3 run_kute.py --output results --testsPath "$TEST_PATH" --jwtPath /tmp/jwtsecret --client $client --run $run + python3 run_kute.py --output "$OUTPUT_DIR" --testsPath "$TEST_PATH" --jwtPath /tmp/jwtsecret --client $client --run $run else echo "Using provided warm up file: $WARMUP_FILE" - python3 run_kute.py --output results --testsPath "$TEST_PATH" --jwtPath /tmp/jwtsecret --warmupPath "$WARMUP_FILE" --client $client --run $run + python3 run_kute.py --output "$OUTPUT_DIR" --testsPath "$TEST_PATH" --jwtPath /tmp/jwtsecret --warmupPath "$WARMUP_FILE" --client $client --run $run fi cd "scripts/$client" @@ -60,9 +62,9 @@ for run in $(seq 1 $RUNS); do done # Get metrics from results -python3 report_tables.py --resultsPath results --clients "$CLIENTS" --testsPath "$TEST_PATH" --runs $RUNS -python3 report_html.py --resultsPath results --clients "$CLIENTS" --testsPath "$TEST_PATH" --runs $RUNS +python3 report_tables.py --resultsPath "$OUTPUT_DIR" --clients "$CLIENTS" --testsPath "$TEST_PATH" --runs $RUNS +python3 report_html.py --resultsPath "$OUTPUT_DIR" --clients "$CLIENTS" --testsPath "$TEST_PATH" --runs $RUNS # Zip the results folder -zip -r results.zip reports \ No newline at end of file +zip -r "${OUTPUT_DIR}.zip" "$OUTPUT_DIR" \ No newline at end of file diff --git a/setup_node.py b/setup_node.py index e9425f248..47d3d954f 100644 --- a/setup_node.py +++ b/setup_node.py @@ -41,8 +41,7 @@ def main(): parser.add_argument('--client', type=str, help='Client that we want to spin up.', default="nethermind") parser.add_argument('--image', type=str, help='Docker image of the client we are going to use.') parser.add_argument('--imageBulk', type=str, help='Docker image of the client we are going to use.', - default='{"nethermind": "default", "besu": "default", "geth": "default", "reth": "default"}, ' - '"erigon": "default"}') + default='{"nethermind": "default", "besu": "default", "geth": "default", "reth": "default", "erigon": "default"}') # Parse command-line arguments args = parser.parse_args() diff --git a/utils.py b/utils.py index ad7df71a9..26a78aac7 100644 --- a/utils.py +++ b/utils.py @@ -59,7 +59,7 @@ def extract_response_and_result(results_path, client, test_case_name, gas_used, if len(text) == 0: return False, 0 # Get latest line - for line in text.split('\n'): + for line in reversed(text.split('\n')): if len(line) < 1: continue if not check_sync_status(line): @@ -262,6 +262,10 @@ def get_payload_status(self): def print_computer_specs(): info = "Computer Specs:\n" cpu = cpuinfo.get_cpu_info() + try: + cpu_freq = psutil.cpu_freq().current + except AttributeError: + cpu_freq = "N/A" system_info = { 'Processor': platform.processor(), 'System': platform.system(), @@ -272,7 +276,7 @@ def print_computer_specs(): 'RAM': f'{psutil.virtual_memory().total / (1024 ** 3):.2f} GB', 'CPU': cpu['brand_raw'], 'Numbers of CPU': cpu['count'], - 'CPU GHz': cpu['hz_actual_friendly'] + 'CPU GHz': f'{cpu_freq} MHz' if cpu_freq != "N/A" else "N/A" } # Print the specifications