Setup local testnet for integration testing

.github/workflows/license_headers_check.yml

@@ -2,7 +2,7 @@ name: License headers check
 
 on:
   push:
-    branches: [ main, dev ]
+    branches: [main, dev]
   pull_request:
   workflow_dispatch:
 
@@ -31,4 +31,11 @@ jobs:
             -ignore 'migrations/*.sql' \
             -ignore 'crates/dips/src/proto/*' \
             -ignore 'crates/dips/proto/*' \
+            -ignore 'contrib/**/*' \
+            -ignore '*.sh' \
+            -ignore 'Dockerfile' \
+            -ignore '*.toml' \
+            -ignore '*.yml' \
+            -ignore '*.yaml' \
+            -ignore 'integration-tests/*.sh' \
             .

.github/workflows/tap_integration_test.yml.disable

@@ -0,0 +1,31 @@
+name: Test GraphTally Protocol
+on:
+  push:
+    # branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up Docker cache
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Run make setup
+        run: make setup
+
+      - name: Run tests
+        run: make rav_tests
+
+      - name: Tear down containers
+        if: always()
+        run: make down

.gitignore

@@ -16,3 +16,7 @@ indexer.toml
 crates/dips/node_modules/
 crates/dips/generated/
 crates/dips/npm-debug.log*
+contrib/local-network
+node_modules
+target/
+*.code-workspace

Cargo.toml

@@ -11,6 +11,7 @@ members = [
     "crates/tap-agent",
     "crates/test-assets",
     "crates/watcher",
+    "integration-tests",
 ]
 resolver = "2"
 

contrib/README.md

@@ -0,0 +1,96 @@
+# Developer Setup Guide
+
+This guide provides the step to set up a development workflow for the indexer-service project testing.
+Including how to build and deploy the required containers, and how to use hot-reload for faster dev iterations.
+
+### Note
+
+The current implementation leverages local-network for the common services and uses its testing configuration.
+
+## Available Commands
+
+We provide the following Make/Just commands to streamline your development workflow:
+
+- `just setup` - Full setup of all services, dependencies and binary compilation
+- `just reload` - Rebuild Rust binaries and restart services after code changes
+- `just logs` - Watch log output from all services
+- `just down` - Stop all services
+- `just test-local` - Run integration tests against local services
+
+## Initial Setup
+
+To get started with development:
+
+1. Clone the repository
+2. Run the full setup to initialize all services:
+
+```bash
+just setup
+```
+
+This will:
+
+- Clone the local-network repo if needed
+- Start core infrastructure services
+- Deploy contracts and required services
+- Set up and start the indexer and tap-agent services
+
+## Development Workflow
+
+After the initial setup, you can use the fast development workflow:
+
+1. Make changes to the Rust code in `crates/`
+2. Run the reload command to rebuild and restart indexer and tap-agent services:
+
+```bash
+just reload
+```
+
+This workflow is much faster because:
+
+- It only recompiles the Rust code
+- It doesn't rebuild Docker containers
+- It restarts only the necessary services
+
+The `reload` command will automatically show the logs after restarting, so you can see if your changes are working properly.
+
+## How It Works
+
+The development workflow uses volume mounts to avoid rebuilding containers:
+
+1. A base Docker image with all dependencies is created once
+2. Your code is compiled locally on your machine
+3. The compiled binaries are mounted into the containers
+4. Services are restarted with the new binaries
+
+This approach avoids the time-consuming container rebuild process while still maintaining a consistent containerized environment.
+
+## Viewing Logs
+
+To monitor the services during development:
+
+```bash
+just logs
+```
+
+This will show a live stream of logs from all services, which is useful for debugging.
+
+## Running Tests
+
+To run integration tests against your local environment:
+
+```bash
+just local-test
+```
+
+This is currently a work in progress, and additional testing strategies are still being defined and implemented.
+
+## Stopping Services
+
+When you're done working:
+
+```bash
+just down
+```
+
+This will stop and remove all the containers defined in the docker-compose file.

contrib/base/Dockerfile

@@ -0,0 +1,6 @@
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openssl ca-certificates protobuf-compiler postgresql-client curl \
+    jq \
+    && rm -rf /var/lib/apt/lists/*

contrib/docker-compose.dev.yml

@@ -0,0 +1,57 @@
+services:
+  indexer-service:
+    image: indexer-base:latest
+    container_name: indexer-service
+    volumes:
+      - ../target/release/indexer-service-rs:/usr/local/bin/indexer-service-rs
+      - ./indexer-service/start.sh:/usr/local/bin/start.sh
+      - ./indexer-service/config.toml:/opt/config/config.toml
+      - ../local-network/contracts.json:/opt/contracts.json:ro
+      - ../local-network/.env:/opt/.env:ro
+      - ../migrations:/opt/migrations:ro
+    entrypoint: ["/usr/local/bin/start.sh"]
+    environment:
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    ports:
+      - "7601:7601"
+    networks:
+      - local-network
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7601/"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+  tap-agent:
+    image: indexer-base:latest # Pre-built base image with dependencies
+    container_name: tap-agent
+    depends_on:
+      indexer-service:
+        condition: service_healthy
+    volumes:
+      - ../target/release/indexer-tap-agent:/usr/local/bin/indexer-tap-agent
+      - ./tap-agent/start.sh:/usr/local/bin/start.sh
+      - ./tap-agent:/opt/config:ro
+      - ./local-network/.env:/opt/.env:ro
+      - ./local-network/contracts.json:/opt/contracts.json:ro
+      - ../migrations:/opt/migrations:ro
+    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
+    environment:
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    ports:
+      # to expose the metrics port
+      - "7300:7300"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7300/metrics"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+networks:
+  local-network:
+    external: true
+    name: local-network_default

contrib/docker-compose.yml

@@ -0,0 +1,57 @@
+services:
+  indexer-service:
+    build:
+      context: .. # Go up one level to the root directory
+      dockerfile: ./contrib/indexer-service/Dockerfile
+    container_name: indexer-service
+    volumes:
+      - ./indexer-service:/opt/config:ro # From contrib dir to indexer-service dir
+      - ./local-network/contracts.json:/opt/contracts.json:ro
+      - ./local-network/.env:/opt/.env:ro
+      - ../migrations:/opt/migrations:ro
+    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
+    environment:
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    ports:
+      - "7601:7601"
+    networks:
+      - local-network
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7601/"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+  tap-agent:
+    build:
+      context: ..
+      dockerfile: ./contrib/tap-agent/Dockerfile
+    container_name: tap-agent
+    depends_on:
+      indexer-service:
+        condition: service_healthy
+    volumes:
+      - ./tap-agent:/opt/config:ro # From contrib dir to tap-agent dir
+      - ./local-network/.env:/opt/.env:ro
+      - ./local-network/contracts.json:/opt/contracts.json:ro
+      - ../migrations:/opt/migrations:ro
+    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
+    environment:
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    ports:
+      # to expose the metrics port
+      - "7300:7300"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:7300/metrics"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 10s
+
+networks:
+  local-network:
+    external: true
+    name: local-network_default

contrib/indexer-service/Dockerfile

@@ -0,0 +1,32 @@
+FROM rust:1.81-bookworm as build
+WORKDIR /root
+# Copy from the root project directory (two levels up)
+COPY ../../ .
+# Force SQLx to use the offline mode to statically check the database queries against
+# the prepared files in the `.sqlx` directory.
+#
+ENV SQLX_OFFLINE=true
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    protobuf-compiler && rm -rf /var/lib/apt/lists/*
+
+RUN cargo build --release --bin indexer-service-rs
+
+########################################################################################
+
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openssl ca-certificates protobuf-compiler postgresql-client curl \
+    jq \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /root/target/release/indexer-service-rs /usr/local/bin/indexer-service-rs
+
+# Copy our start script into the image
+COPY contrib/indexer-service/start.sh /usr/local/bin/start.sh
+COPY contrib/indexer-service/config.toml /opt/config/config.toml
+
+RUN chmod +x /usr/local/bin/start.sh
+
+ENTRYPOINT [ "/usr/local/bin/start.sh" ]

contrib/indexer-service/config.toml

@@ -0,0 +1,46 @@
+[indexer]
+indexer_address = "INDEXER_ADDRESS_PLACEHOLDER"
+operator_mnemonic = "INDEXER_MNEMONIC_PLACEHOLDER"
+
+[database]
+postgres_url = "postgres://postgres@postgres:POSTGRES_PORT_PLACEHOLDER/indexer_components_1"
+
+[graph_node]
+query_url = "http://graph-node:8000"
+status_url = "http://graph-node:8030/graphql"
+
+[subgraphs.network]
+query_url = "http://graph-node:8000/subgraphs/name/graph-network"
+deployment_id = "NETWORK_DEPLOYMENT_PLACEHOLDER"
+
+[subgraphs.escrow]
+query_url = "http://graph-node:8000/subgraphs/name/semiotic/tap"
+deployment_id = "ESCROW_DEPLOYMENT_PLACEHOLDER"
+
+[blockchain]
+chain_id = 1337
+receipts_verifier_address = "VERIFIER_ADDRESS_PLACEHOLDER"
+
+[service]
+host_and_port = "0.0.0.0:7601"
+free_query_auth_token = "freestuff"
+
+[service.tap]
+max_receipt_value_grt = "0.001"
+
+[tap]
+max_amount_willing_to_lose_grt = 1000
+
+[tap.rav_request]
+# Set a lower timestamp buffer threshold
+timestamp_buffer_secs = 30
+# The trigger value divisor is used to calculate the trigger value for the RAV request.
+# using the formula:
+# trigger_value = max_amount_willing_to_lose_grt / trigger_value_divisor
+# where the default value for max_amount_willing_to_lose_grt is 1000
+# the idea to set this for trigger_value to be 0.002
+# requiring the sender to send at least 20 receipts of 0.0001 grt
+trigger_value_divisor = 500_000
+
+[tap.sender_aggregator_endpoints]
+"ACCOUNT0_ADDRESS_PLACEHOLDER" = "http://tap-aggregator:7610"