feat: Enhance Integration Tests with Load Testing for the indexer-service (#723)

* feat: add new load test to measure receipt handler in service

* refactor: remove duplicated start script

* chore: move constants to its own file

* feat: Implement CLI for test execution using Clap

Introduces a command-line interface powered by Clap to select and
configure integration tests. This allows for more flexible test runs,
including the ability to pass specific parameters like 'num_receipts'
to different test suites.

Additionally, constants used across tests have been centralized into
a dedicated module for improved organization and maintainability.

* chore: remove line to wait for db

* style: fix typo in comments

Author: neithanmo

Cargo.lock

@@ -9,10 +9,10 @@ services:
       - ./local-network/.env:/opt/.env:ro
       - ./profiling:/opt/profiling:rw
       - ./indexer-service/config.toml:/opt/config/config.toml
-      - ./indexer-service/start-perf.sh:/usr/local/bin/start-perf.sh
+      - ./indexer-service/start.sh:/usr/local/bin/start.sh
       - ../migrations:/opt/migrations:ro
       - ../target/release/indexer-service-rs:/usr/local/bin/indexer-service-rs
-    entrypoint: ["/usr/local/bin/start-perf.sh"]
+    entrypoint: ["/usr/local/bin/start.sh"]
     environment:
       - RUST_BACKTRACE=1
       - RUST_LOG=debug

contrib/docker-compose.prof.yml

@@ -23,9 +23,7 @@ COPY --from=build /root/target/release/indexer-service-rs /usr/local/bin/indexer
 # Copy our start script into the image
 COPY contrib/indexer-service/start.sh /usr/local/bin/start.sh
 COPY contrib/indexer-service/config.toml /opt/config/config.toml
-COPY contrib/indexer-service/start-perf.sh /usr/local/bin/start-perf.sh
 
 RUN chmod +x /usr/local/bin/start.sh
-RUN chmod +x /usr/local/bin/start-perf.sh
 
-ENTRYPOINT [ "/usr/local/bin/start-perf.sh" ]
+ENTRYPOINT [ "/usr/local/bin/start.sh" ]

contrib/indexer-service/Dockerfile

@@ -13,12 +13,6 @@ VERIFIER_ADDRESS=$(jq -r '."1337".TAPVerifier.address' /opt/contracts.json)
 # Override with test values taken from test-assets/src/lib.rs
 ALLOCATION_ID="0xfa44c72b753a66591f241c7dc04e8178c30e13af" # ALLOCATION_ID_0
 
-# Wait for postgres to be ready
-until pg_isready -h postgres -U postgres -d indexer_components_1; do
-    stdbuf -oL echo "Waiting for postgres..."
-    sleep 2
-done
-
 # Get network subgraph deployment ID
 NETWORK_DEPLOYMENT=$(curl -s "http://graph-node:8000/subgraphs/name/graph-network" \
     -H 'content-type: application/json' \
@@ -57,8 +51,76 @@ stdbuf -oL echo "Testing graph-node endpoints..."
 curl -s "http://graph-node:8000" >/dev/null && stdbuf -oL echo "Query endpoint OK" || stdbuf -oL echo "Query endpoint FAILED"
 curl -s "http://graph-node:8030/graphql" >/dev/null && stdbuf -oL echo "Status endpoint OK" || stdbuf -oL echo "Status endpoint FAILED"
 
-# Run service with verbose logging
+# Set profiling tool based on environment variable
+# Default is no profiling
+PROFILER="${PROFILER:-none}"
+stdbuf -oL echo "🔍 DEBUG: Profiling with: $PROFILER"
+
+# Set environment variables for the service
 export RUST_BACKTRACE=full
-# export RUST_LOG=debug
-export RUST_LOG=trace
-exec /usr/local/bin/indexer-service-rs --config /opt/config.toml
+export RUST_LOG="${RUST_LOG:-trace}"
+
+# Create output directory if it doesn't exist
+mkdir -p /opt/profiling/indexer-service
+chmod 777 /opt/profiling
+chmod 777 /opt/profiling/indexer-service
+
+stdbuf -oL echo "📁 DEBUG: Profiling output directory: $(ls -la /opt/profiling)"
+
+case "$PROFILER" in
+flamegraph)
+    stdbuf -oL echo "🔥 Starting with profiler..."
+
+    # Start the service in the background with output redirection
+    stdbuf -oL echo "🚀 Starting service..."
+    exec /usr/local/bin/indexer-service-rs --config /opt/config.toml
+    ;;
+strace)
+    stdbuf -oL echo "🔍 Starting with strace..."
+    # -f: follow child processes
+    # -tt: print timestamps with microsecond precision
+    # -T: show time spent in each syscall
+    # -e trace=all: trace all system calls
+    # -s 256: show up to 256 characters per string
+    # -o: output file
+    exec strace -f -tt -T -e trace=all -s 256 -o /opt/profiling/indexer-service/strace.log /usr/local/bin/indexer-service-rs --config /opt/config.toml
+    ;;
+valgrind)
+    stdbuf -oL echo "🔍 Starting with Valgrind profiling..."
+
+    # Start with Massif memory profiler
+    stdbuf -oL echo "🔄 Starting Valgrind Massif memory profiling..."
+    exec valgrind --tool=massif \
+        --massif-out-file=/opt/profiling/indexer-service/massif.out \
+        --time-unit=B \
+        --detailed-freq=10 \
+        --max-snapshots=100 \
+        --threshold=0.5 \
+        /usr/local/bin/indexer-service-rs --config /opt/config.toml
+    ;;
+# Use callgrind_annotate indexer-service.callgrind.out
+# for human-friendly report of callgrind output
+# Ideally you should set:
+# [profile.release.package."*"]
+# debug = true
+# force-frame-pointers = true
+# in the Cargo.toml
+callgrind)
+    stdbuf -oL echo "🔍 Starting with Callgrind CPU profiling..."
+    exec valgrind --tool=callgrind \
+        --callgrind-out-file=/opt/profiling/indexer-service/callgrind.out \
+        --cache-sim=yes \
+        --branch-sim=yes \
+        --collect-jumps=yes \
+        --collect-systime=yes \
+        --collect-bus=yes \
+        --dump-instr=yes \
+        --dump-line=yes \
+        --compress-strings=no \
+        /usr/local/bin/indexer-service-rs --config /opt/config.toml
+    ;;
+none)
+    stdbuf -oL echo "🔍 Starting without profiling..."
+    exec /usr/local/bin/indexer-service-rs --config /opt/config.toml
+    ;;
+esac

contrib/indexer-service/start-perf.sh

@@ -23,3 +23,5 @@ bip39 = { workspace = true }
 rand.workspace = true
 
 indexer-receipt = { path = "../crates/indexer-receipt" }
+num_cpus = "1.16.0"
+clap = { version = "4.0", features = ["derive"] }

contrib/indexer-service/start.sh

@@ -0,0 +1,31 @@
+// Copyright 2023-, Edge & Node, GraphOps, and Semiotic Labs.
+// SPDX-License-Identifier: Apache-2.0
+//
+//
+//! Constants used in the integration tests for the TAP RAV generation
+//! their value is taken from local-network .env variable
+
+pub const INDEXER_URL: &str = "http://localhost:7601";
+
+pub const GATEWAY_API_KEY: &str = "deadbeefdeadbeefdeadbeefdeadbeef";
+pub const GATEWAY_URL: &str = "http://localhost:7700";
+pub const TAP_AGENT_METRICS_URL: &str = "http://localhost:7300/metrics";
+
+// The deployed gateway and indexer
+// use this verifier contract
+// which must be part of the eip712 domain
+// and the signing key account0_secret
+// they must match otherwise receipts would be rejected
+pub const TAP_VERIFIER_CONTRACT: &str = "0x8198f5d8F8CfFE8f9C413d98a0A55aEB8ab9FbB7";
+pub const ACCOUNT0_SECRET: &str =
+    "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80";
+pub const CHAIN_ID: u64 = 1337;
+
+pub const SUBGRAPH_ID: &str = "QmV4R5g7Go94bVFmKTVFG7vaMTb1ztUUWb45mNrsc7Yyqs";
+
+pub const GRAPH_URL: &str = "http://localhost:8000/subgraphs/name/graph-network";
+
+pub const GRT_DECIMALS: u8 = 18;
+pub const GRT_BASE: u128 = 10u128.pow(GRT_DECIMALS as u32);
+
+pub const MAX_RECEIPT_VALUE: u128 = GRT_BASE / 10_000;