fix: Update docker compose for readme

Author: Sital999

README.md

@@ -20,7 +20,62 @@ and align with the decentralized governance objectives of Cardano's Voltaire era
 
 ### Using Docker
 
-Setup the required dependencies by running the command locally.
+Before running whole service locally using docker you need to add few of the environment variables
+on file `docker-compose.dev.yml`
+
+##### api
+- AGENT_MNEMONIC 
+  - Add seed phrase to generate wallet
+
+##### agent_manager
+- KUBER_API_KEY
+  - Visit [KuberIde](https://kuberide.com/kuber/settings/api-keys) and generate api-key
+- MANAGER_WALLET_ADDRESS (OPTIONAL)
+- MANAGER_WALLET_SIGNING_KEY (OPTIONAL)
+  - Add a wallet address having sufficient ADA so that it can be used to transfer ADA to agent when requested
+- FAUCET_API_KEY (OPTIONAL)
+  - Add faucet api key to load ADA which will be used to transfer ADA to agents as per request. And it will only be used if the provided `MANAGER_WALLET_ADDRESS` doesnot have sufficient ADA.
+- AGENT_MNEMONIC
+    - Add seed phrase to generate wallet that should be same as added in `api`
+- BLOCKFROST_API_KEY (Required if ENABLE_BLOCKFROST_SUBMIT_API is 'True' or enabled)
+  - Visit [Blockfrost](https://blockfrost.io/) and sign up and generate api key
+
+Note: environment variable `ENABLE_BLOCKFROST_SUBMIT_API` is preferred as if it is not enabled then `Kuber` will be used to submit the transaction which might take couple of minutes.
+
+##### dbsync
+- DATABASE_URL
+  - Add database url of dbsync
+
+Furthermore all env are setup to run in `Sanchonet` so if you want to run in `Preprod` or `Preview`
+Network then following environment variables are to be updated:
+
+##### frontend
+- NEXT_PUBLIC_NETWORK_NAME
+  -  preview or preprod
+
+##### api and manager
+- DB_SYNC_BASE_URL
+  - https://preprod-dbync.agents.cardanoapi.io/api for `preprod`
+  - https://preview-dbync.agents.cardanoapi.io/api for `preview`
+
+##### manager only
+- KUBER_BASE_URL
+  - https://preview.kuber.cardanoapi.io for `preview`
+  - https://preprod.kuber.cardanoapi.io for `preprod`
+
+- CARDANO_NETWORK_MAGIC
+  - 3 for `preview`
+  - 2 for `preprod`
+
+- BLOCKFROST_API_KEY
+  - Visit [Blockfrost](https://blockfrost.io/) and sign up and generate api key based on desired network type 
+
+- NETWORK_NAME
+  - preprod or preview
+
+##### dbsync
+- DATABASE_URL
+  - Update the dbsync database url and database name accordingly
 
 ```shell
 docker compose -f docker-compose.dev.yml up -d

api/.env.example

@@ -15,7 +15,7 @@ KAFKA_PREFIX=
 
 JWT_SECRET_KEY=""
 
-METADATA_API=''
-DB_SYNC_API=''
+METADATA_BASE_URL=''
+DB_SYNC_BASE_URL=''
 
 ELASTIC_APM_SERVER_URL=https://apm.sireto.io

api/backend/app/controllers/health.py

@@ -73,7 +73,7 @@ async def readiness_check():
 
 async def dbsync_health_check():
     async with aiohttp.ClientSession() as session:
-        async with session.get(api_settings.DB_SYNC_API + "/health") as response:
+        async with session.get(api_settings.DB_SYNC_BASE_URL + "/health") as response:
             if response.status == 200:
                 return True
             return False

api/backend/app/controllers/internal/metadata_router.py

@@ -9,13 +9,13 @@ class MetadataRouter(Routable):
 
     def __init__(self):
         super().__init__()
-        self.metadata_api = APISettings().METADATA_API
+        self.metadata_base_url = APISettings().METADATA_BASE_URL
 
     @get("/metadata")
     async def fetch_metadata(self, metadata_url: str):
         async with aiohttp.ClientSession() as session:
             async with session.get(
-                f"{self.metadata_api}/metadata?url={metadata_url}&hash=1111111111111111111111111111111111111111111111111111111111111112"
+                f"{self.metadata_base_url}/metadata?url={metadata_url}&hash=1111111111111111111111111111111111111111111111111111111111111112"
             ) as resp:
                 response = await resp.json()
                 if resp.ok:

api/backend/app/services/agent_service.py

@@ -205,7 +205,7 @@ async def fetch_data(self, url, session: ClientSession):
                 raise HTTPException(status_code=400, content="Error fetching agent Drep details")
 
     async def fetch_balance(self, stake_address: str, session: ClientSession):
-        async with session.get(f"{api_settings.DB_SYNC_API}/address/balance?address={stake_address}") as response:
+        async with session.get(f"{api_settings.DB_SYNC_BASE_URL}/address/balance?address={stake_address}") as response:
             try:
                 return await response.json()
             except:
@@ -215,7 +215,7 @@ async def fetch_balance(self, stake_address: str, session: ClientSession):
                 )
 
     async def fetch_drep_details(self, drep_id: str, session: ClientSession) -> Dict[str, float | bool]:
-        async with session.get(f"{api_settings.DB_SYNC_API}/drep/{drep_id}") as response:
+        async with session.get(f"{api_settings.DB_SYNC_BASE_URL}/drep/{drep_id}") as response:
             try:
                 res = await response.json()
                 voting_power = res.get("votingPower") / (10**6) if res.get("votingPower") else 0
@@ -228,7 +228,7 @@ async def fetch_drep_details(self, drep_id: str, session: ClientSession) -> Dict
                 )
 
     async def fetch_stake_address_details(self, stake_address: str, session: ClientSession):
-        async with session.get(f"{api_settings.DB_SYNC_API}/stake-address?address={stake_address}") as response:
+        async with session.get(f"{api_settings.DB_SYNC_BASE_URL}/stake-address?address={stake_address}") as response:
             try:
                 is_stake_registered = False
                 res = await response.json()
@@ -250,7 +250,7 @@ async def fetch_stake_address_details(self, stake_address: str, session: ClientS
                 )
 
     async def fetch_delegation_details(self, stake_address: str, session: ClientSession):
-        async with session.get(f"{api_settings.DB_SYNC_API}/delegation?address={stake_address}") as response:
+        async with session.get(f"{api_settings.DB_SYNC_BASE_URL}/delegation?address={stake_address}") as response:
             try:
                 res = await response.json()
                 drep_id = res.get("drep", {}).get("drep_id") if res.get("drep") else None

api/backend/app/services/drep_service.py

@@ -58,7 +58,7 @@ async def fetch_internal_dreps(self, page: int, page_size: int, search: str | No
     async def fetch_metadata(self, agent: Agent, index: int, agents: list[Any], session: ClientSession):
         drep_dict = {}
         drep_id = convert_base64_to_hex(agent.wallet_details[0].stake_key_hash)
-        async with session.get(f"{api_settings.DB_SYNC_API}/drep?search={drep_id}") as response:
+        async with session.get(f"{api_settings.DB_SYNC_BASE_URL}/drep?search={drep_id}") as response:
             response_json = await response.json()
             if response_json["items"]:
                 if drep_id == response_json["items"][0]["drepId"]:
@@ -68,7 +68,7 @@ async def fetch_metadata(self, agent: Agent, index: int, agents: list[Any], sess
                 metadata_hash = drep_dict.get("metadataHash")
                 try:
                     async with session.get(
-                        f"{api_settings.METADATA_API}/metadata?url={url}&hash={metadata_hash}"
+                        f"{api_settings.METADATA_BASE_URL}/metadata?url={url}&hash={metadata_hash}"
                     ) as metadata_resp:
                         metadata_resp_json = await metadata_resp.json()
                         if "hash" in metadata_resp_json:
@@ -84,9 +84,9 @@ async def fetch_metadata(self, agent: Agent, index: int, agents: list[Any], sess
 
     async def fetch_external_dreps(self, page: int, page_size: int, search: str | None):
         if search:
-            fetchUrl = f"{api_settings.DB_SYNC_API}/drep?search={search}"
+            fetchUrl = f"{api_settings.DB_SYNC_BASE_URL}/drep?search={search}"
         else:
-            fetchUrl = f"{api_settings.DB_SYNC_API}/drep?page={page}&size={page_size}"
+            fetchUrl = f"{api_settings.DB_SYNC_BASE_URL}/drep?page={page}&size={page_size}"
 
         async with aiohttp.ClientSession() as session:
             async with session.get(fetchUrl) as response:
@@ -129,7 +129,7 @@ async def fetch_metadata_for_drep(self, metadata_hash: str, url: str, drep: Any)
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.get(
-                    f"{api_settings.METADATA_API}/metadata?url={url}&hash={metadata_hash}"
+                    f"{api_settings.METADATA_BASE_URL}/metadata?url={url}&hash={metadata_hash}"
                 ) as metadata_resp:
                     metadata_resp_json = await metadata_resp.json()
                     if "hash" in metadata_resp_json:

api/backend/app/services/proposal_service.py

@@ -66,7 +66,7 @@ async def get_internal_proposals(self, page: int = 1, pageSize: int = 10, search
     async def add_metadata_and_agent_detail_in_internal_proposal(
         self, proposal: TriggerHistoryDto, index: int, results: list[Any]
     ):
-        url = f"{api_settings.DB_SYNC_API}/proposal?proposal={proposal.txHash}"
+        url = f"{api_settings.DB_SYNC_BASE_URL}/proposal?proposal={proposal.txHash}"
         proposal_data = await self._fetch_proposal_data(url)
         if not proposal_data:
             results[index] = ""
@@ -83,9 +83,9 @@ async def add_metadata_and_agent_detail_in_internal_proposal(
             results[index] = proposal_dict
 
     async def get_external_proposals(self, page: int, pageSize: int, sort: str, search: str | None = None):
-        search_url = f"{api_settings.DB_SYNC_API}/proposal?page={page}&size={pageSize}&sort={sort}"
+        search_url = f"{api_settings.DB_SYNC_BASE_URL}/proposal?page={page}&size={pageSize}&sort={sort}"
         if search:
-            search_url = f"{api_settings.DB_SYNC_API}/proposal?proposal={search}"
+            search_url = f"{api_settings.DB_SYNC_BASE_URL}/proposal?proposal={search}"
 
         async with aiohttp.ClientSession() as session:
             async with session.get(search_url) as response:
@@ -126,7 +126,7 @@ async def _fetch_metadata(self, metadata_hash: str, url: str, proposal_dict: Any
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.get(
-                    f"{api_settings.METADATA_API}/metadata?url={url}&hash={metadata_hash}"
+                    f"{api_settings.METADATA_BASE_URL}/metadata?url={url}&hash={metadata_hash}"
                 ) as metadata_resp:
                     metadata_resp_json = await metadata_resp.json()
                     if "hash" in metadata_resp_json:

api/backend/config/api_settings.py

@@ -5,9 +5,9 @@ class APISettings(BaseSettings):
     APP_ENV: str = "production"
     SECURE: bool = None
     JWT_SECRET_KEY: str = ""
-    DB_SYNC_API: str = "https://dbsyncapi.agents.cardanoapi.io/api"
+    DB_SYNC_BASE_URL: str = "https://dbsyncapi.agents.cardanoapi.io/api"
     SAME_SITE = "None"
-    METADATA_API: str = "https://metadata.drep.id/api"
+    METADATA_BASE_URL: str = "https://metadata.drep.id/api"
     GOV_ACTION_API: str = "https://govtool.cardanoapi.io/api"
     KAFKA_TOPIC_PREFIX: str = ""
     KAFKA_PREFIX: str = ""

docker-compose.dev.yml

@@ -25,8 +25,8 @@ services:
       - DOCS_URL=/api/docs
       - AGENT_MNEMONIC='group shadow belt culture garage sport demand target twist tribe milk dumb divide mango bench fantasy okay unfair error twice struggle naive squirrel romance'
       - KAFKA_ENABLED='True'
-      - METADATA_API='https://metadata.drep.id/api'
-      - DB_SYNC_API='https://dbsyncapi.agents.cardanoapi.io/api'
+      - METADATA_BASE_URL='https://metadata.drep.id/api'
+      - DB_SYNC_BASE_URL='http://dbsync:9000/api'
     depends_on:
       - postgres
       - kafka
@@ -49,10 +49,10 @@ services:
       - CLIENT_ID=my-app
       - CARDANO_NODE_URL=172.31.0.4:3004
       - KUBER_BASE_URL='https://sanchonet.kuber.cardanoapi.io'
-      - KUBER_API_KEY='l3HSdpCx86BOZFEvHspmYZ8KiVOnnvJDLKkC7JMVKUu9pZlV0Ld0qbn3pTQ3ER'
-      - MANAGER_WALLET_ADDRESS=addr_test1qrd3hs7rlxwwdzthe6hj026dmyt3y0heuulctscyydh2kguh4xfmpjqkd25vfq69hcvj27jqyk4hvnyxu7vma2c4kvps8eh2m3
-      - MANAGER_WALLET_SIGNING_KEY=addr_sk14wrrctnv9cyr05vjnwrjcs7mujzuxf4zj5nm7fna5nx7fxrnrupqwhwycx
-      - FAUCET_API_KEY=ayEO6dlVF18oTslr9eIMUNgE35GqVfFz
+      - KUBER_API_KEY=l3HSdpCx86BOZFEvHspmYZ8KiVOnnvJDLKkC7JMVKUu9pZlV0Ld0qbn3pTQ3ER
+      - MANAGER_WALLET_ADDRESS=
+      - MANAGER_WALLET_SIGNING_KEY=
+      - FAUCET_API_KEY=
       - AGENT_MNEMONIC='group shadow belt culture garage sport demand target twist tribe milk dumb divide mango bench fantasy okay unfair error twice struggle naive squirrel romance'
       - METADATA_BASE_URL='https://metadata.drep.id'
       - DB_SYNC_BASE_URL='https://dbsyncapi.agents.cardanoapi.io/api'
@@ -62,7 +62,19 @@ services:
       - SERVER_PORT=3002
       - NETWORK_NAME='sanchonet'
 
-#  #Agent (build image)
+# DbSync
+  dbsync:
+    build:
+      context: ./dbsync-api
+      dockerfile: Dockerfile
+    ports:
+      - "9000:9000"
+    environment:
+      - PORT=9000
+      - CORS_ENABLE=true
+      - DATABASE_URL=postgres://dbsync:9PW%fj36ozwm^8u@mVwoh!uE&[email protected]:8433/dbsync_sanchonet
+
+  #  #Agent (build image)
 #  agent:
 #    build:
 #      context: ./agent-node

docker-compose.local.yml

@@ -18,66 +18,66 @@ services:
       - cardano-autonomous-backend
 
 #  #Pgadmin
-#  pgadmin:
-#    image: dpage/pgadmin4:8.2
-#    container_name: pg-admin
-#    environment:
-#      PGADMIN_DEFAULT_EMAIL: [email protected]
-#      PGADMIN_DEFAULT_PASSWORD: password
-#    ports:
-#      - "5050:80"
-#    networks:
-#      - cardano-autonomous-backend
-#
-##Zookeper
-#  zookeeper:
-#    image: confluentinc/cp-zookeeper:7.0.1
-#    container_name: zookeeper_agent_autonomous
-#    ports:
-#      - "2181:2181"
-#    environment:
-#      ZOOKEEPER_CLIENT_PORT: 2181
-#      ZOOKEEPER_TICK_TIME: 2000
-#      ZOOKEEPER_SYNC_LIMIT: 2
-#    networks:
-#      - cardano-autonomous-backend
-#
-# #Kafka
-#  kafka:
-#    image: confluentinc/cp-kafka:7.0.1
-#    hostname: kafka
-#    container_name: kafka_agent_autonomous
-#    depends_on:
-#      - zookeeper
-#    ports:
-#      - "9092:9092"
-#    environment:
-#      KAFKA_BROKER_ID: 1
-#      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-#      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER_NET:PLAINTEXT,HOST_NET:PLAINTEXT
-#      KAFKA_ADVERTISED_LISTENERS:  DOCKER_NET://kafka:9093,HOST_NET://localhost:9092
-#      KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER_NET
-#      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-#      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-#      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-#      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-#      KAFKA_JMX_PORT: 9999
-#      KAFKA_CREATE_TOPICS: "trigger_config_updates:1:1"
-#    networks:
-#      - cardano-autonomous-backend
-#
-##Kafka UI
-#  kafka-ui:
-#    image: provectuslabs/kafka-ui
-#    container_name: kafka-ui-agent-autonomous
-#    ports:
-#      - "8080:8080"
-#    restart: always
-#    environment:
-#      - KAFKA_CLUSTERS_0_NAME=local
-#      - KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS=kafka:9093
-#    networks:
-#      - cardano-autonomous-backend
+  pgadmin:
+    image: dpage/pgadmin4:8.2
+    container_name: pg-admin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: [email protected]
+      PGADMIN_DEFAULT_PASSWORD: password
+    ports:
+      - "5050:80"
+    networks:
+      - cardano-autonomous-backend
+
+#Zookeper
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.0.1
+    container_name: zookeeper_agent_autonomous
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+      ZOOKEEPER_SYNC_LIMIT: 2
+    networks:
+      - cardano-autonomous-backend
+
+ #Kafka
+  kafka:
+    image: confluentinc/cp-kafka:7.0.1
+    hostname: kafka
+    container_name: kafka_agent_autonomous
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER_NET:PLAINTEXT,HOST_NET:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS:  DOCKER_NET://kafka:9093,HOST_NET://localhost:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER_NET
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_JMX_PORT: 9999
+      KAFKA_CREATE_TOPICS: "trigger_config_updates:1:1"
+    networks:
+      - cardano-autonomous-backend
+
+#Kafka UI
+  kafka-ui:
+    image: provectuslabs/kafka-ui
+    container_name: kafka-ui-agent-autonomous
+    ports:
+      - "8080:8080"
+    restart: always
+    environment:
+      - KAFKA_CLUSTERS_0_NAME=local
+      - KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS=kafka:9093
+    networks:
+      - cardano-autonomous-backend
 
 networks:
   cardano-autonomous-backend: