feat: Horizon upgrade support and improved development workflow (#811)

* fix(docker): resolve dev container startup and standardize compose configs

  - Fix dev containers exiting immediately by adding proper entrypoints
  - Consolidate profiling config from docker-compose.prof.yml into dev.yml
  - Standardize volume mounts and add missing horizon.json mappings
  - Update configs for Horizon upgrade (GraphTallyCollector vs TAPVerifier)
  - Add profiling support with proper security options and permissions

  Resolves issues preventing indexer-service and tap-agent from running in dev mode

* refactor(indexer-service): update for Horizon upgrade and improve config generation

  - Migrate from TAPVerifier to GraphTallyCollector contract addresses
  - Replace sed-based template config with inline TOML generation
  - Add horizon.json support and Horizon migration detection
  - Streamline profiling configuration and reduce script verbosity

* refactor(tap-agent): update for Horizon upgrade and add profiling capabilities

  - Migrate from TAPVerifier to GraphTallyCollector contract addresses
  - Replace sed-based config templates with inline generation
  - Bundle horizon.json and .env files in Docker image
  - Simplify startup script structure and improve logging
  - Remove indexer-service dependency for independent startup

* chore(dev-workflow): improve container management

  - Separate prod/dev workflows and add services-status command
  - Improve container cleanup and colorized output for better UX

* chore(test): remove profiling script no longer needed

* fix(integration-tests): update fund_escrow.sh for correct file paths and formatting

  - Fix contract file paths to use ../contrib/local-network/ directory
  - Update error message to reflect correct execution directory
  - Remove set -e to allow graceful error handling
  - Clean up formatting and spacing consistency

* feat(setup): add timing reports and improve disk usage tracking

  - Add script execution time tracking with minutes/seconds display
  - Improve Docker size calculation with robust fallback methods
  - Remove debug output (ls, pwd) and unused environment variables
  - Update gateway container to use horizon.json instead of tap-contracts.json
  - Disable set -e to allow graceful error handling
  - Add structured completion report with timing and disk usage metrics

* fix(test): clone local-network submodule necessary for testing containers

* fix(test): remove unnecessary sections for compose file

Author: neithanmo

.github/workflows/tap_integration_test.yml

@@ -45,6 +45,17 @@ jobs:
       #     registry: ghcr.io
       #     username: ${{ github.actor }}
       #     password: ${{ secrets.GITHUB_TOKEN }}
+      # TODO: Improve this later
+      # evaluate if it makes sense to use a git submodule instead
+      - name: Clone local-network dependency for Docker builds
+        run: |
+          cd contrib/
+          if [ ! -d "local-network" ]; then
+            git clone https://github.com/semiotic-ai/local-network.git
+            cd local-network
+            git checkout suchapalaver/test/horizon
+          fi
+
       - name: Build indexer-service image with Docker Compose
         run: docker compose -f contrib/docker-compose.yml build indexer-service
 

contrib/docker-compose.dev.yml

@@ -1,18 +1,23 @@
 services:
   indexer-service:
     image: indexer-base:latest
+    build:
+      context: ..
     container_name: indexer-service
     volumes:
-      - ../target/release/indexer-service-rs:/usr/local/bin/indexer-service-rs
-      - ./indexer-service/start.sh:/usr/local/bin/start.sh
-      - ./indexer-service/config.toml:/opt/config/config.toml
       - ./local-network/tap-contracts.json:/opt/contracts.json:ro
+      - ./local-network/horizon.json:/opt/horizon.json:ro
       - ./local-network/.env:/opt/.env:ro
+      - ./profiling:/opt/profiling:rw
+      # - ./indexer-service/config.toml:/opt/config/config.toml
+      - ./indexer-service/start.sh:/usr/local/bin/start.sh
       - ../migrations:/opt/migrations:ro
+      - ../target/release/indexer-service-rs:/usr/local/bin/indexer-service-rs
     entrypoint: ["/usr/local/bin/start.sh"]
     environment:
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
+      - PROFILER=${PROFILER:-none}
     ports:
       - "7601:7601"
     networks:
@@ -23,23 +28,29 @@ services:
       timeout: 3s
       retries: 10
       start_period: 10s
+    cap_add:
+      - SYS_ADMIN
+    privileged: true
+    security_opt:
+      - seccomp:unconfined
 
   tap-agent:
     image: indexer-base:latest # Pre-built base image with dependencies
     container_name: tap-agent
-    depends_on:
-      indexer-service:
-        condition: service_healthy
     volumes:
       - ../target/release/indexer-tap-agent:/usr/local/bin/indexer-tap-agent
-      - ./tap-agent:/opt/config:ro
+      - ./tap-agent/start.sh:/usr/local/bin/start.sh
+      # - ./tap-agent:/opt/config:ro
+      - ./profiling:/opt/profiling:rw
       - ./local-network/.env:/opt/.env:ro
       - ./local-network/tap-contracts.json:/opt/contracts.json:ro
+      - ./local-network/horizon.json:/opt/horizon.json:ro
       - ../migrations:/opt/migrations:ro
-    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
+    entrypoint: ["/usr/local/bin/start.sh"]
     environment:
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
+      - PROFILER=${PROFILER:-none}
     ports:
       # to expose the metrics port
       - "7300:7300"
@@ -51,6 +62,11 @@ services:
       timeout: 3s
       retries: 10
       start_period: 10s
+    cap_add:
+      - SYS_ADMIN
+    privileged: true
+    security_opt:
+      - seccomp:unconfined
 
 networks:
   local-network:

contrib/docker-compose.prof.yml

@@ -5,11 +5,10 @@ services:
       dockerfile: ./contrib/indexer-service/Dockerfile
     container_name: indexer-service
     volumes:
-      - ./indexer-service:/opt/config:ro # From contrib dir to indexer-service dir
       - ./local-network/tap-contracts.json:/opt/contracts.json:ro
+      - ./local-network/horizon.json:/opt/horizon.json:ro
       - ./local-network/.env:/opt/.env:ro
       - ../migrations:/opt/migrations:ro
-    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
     environment:
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
@@ -33,16 +32,15 @@ services:
       indexer-service:
         condition: service_healthy
     volumes:
-      - ./tap-agent:/opt/config:ro # From contrib dir to tap-agent dir
       - ./local-network/.env:/opt/.env:ro
       - ./local-network/tap-contracts.json:/opt/contracts.json:ro
+      - ./local-network/horizon.json:/opt/horizon.json:ro
       - ../migrations:/opt/migrations:ro
-    entrypoint: ["/bin/bash", "-c", "/opt/config/start.sh"]
     environment:
       - RUST_BACKTRACE=1
       - RUST_LOG=debug
+      - PROFILER=${PROFILER:-none}
     ports:
-      # to expose the metrics port
       - "7300:7300"
     networks:
       - local-network

contrib/docker-compose.yml

@@ -21,8 +21,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=build /root/target/release/indexer-service-rs /usr/local/bin/indexer-service-rs
 
 # Copy our start script into the image
+COPY contrib/local-network/.env /opt/.env
 COPY contrib/indexer-service/start.sh /usr/local/bin/start.sh
 COPY contrib/indexer-service/config.toml /opt/config/config.toml
+COPY contrib/local-network/horizon.json /opt/horizon.json
 
 RUN chmod +x /usr/local/bin/start.sh