add better .env.examples file, add better docs for dev setup, fix orchestrator + validator s3 parsing (#592)

JannikSt · web-flow · commit a17f07dcd858 · 2025-06-25T21:52:57.000+10:00
diff --git a/.env.example b/.env.example
@@ -1,50 +1,65 @@
-RPC_URL=
-NETWORK_ID=
-MIN_STAKE_AMOUNT=
-WORKER_COMPUTE_POOL_ID=
-WORKER_EXTERNAL_IP=
+# Prime Network Configuration
+# ===========================
 
-# Discovery service URLs (comma-separated for multiple services)
+# Network
+# -------
+RPC_URL=http://localhost:8545
+NETWORK_ID=31337
+MIN_STAKE_AMOUNT=10
+WORKER_COMPUTE_POOL_ID=0
+# Address that we get when running deploy_work_validation.sh in smart contracts
+# This automatically runs when starting the worker
+WORK_VALIDATION_CONTRACT=0x0B306BF915C4d645ff596e518fAf3F9669b97016
+
+# Discovery
+# ---------
 DISCOVERY_URLS=http://localhost:8089
 
-# Legacy support for single discovery URL (use DISCOVERY_URLS instead)
-# DISCOVERY_URL=http://localhost:8089
-
-# Private keys of privileged accounts
-# See deploy.sh files in smart-contracts
-PRIVATE_KEY_FEDERATOR=
-FEDERATOR_ADDRESS=
-PRIVATE_KEY_VALIDATOR=
-VALIDATOR_ADDRESS=
-
-# Provider with their node
-# This can simple be a provider that is funded as part of the anvil start cmd. 
-# You can also generate wallets using prime-worker generate-wallets
-PRIVATE_KEY_PROVIDER=
-PROVIDER_ADDRESS=
-PRIVATE_KEY_NODE=
-NODE_ADDRESS=
-
-# Pool Owner
-POOL_OWNER_PRIVATE_KEY=
-POOL_OWNER_ADDRESS=
-
-# Contracts - you get these addresses after deploying the contracts
-PRIME_NETWORK_ADDRESS=
-AI_TOKEN_ADDRESS=
-COMPUTE_REGISTRY_ADDRESS=
-DOMAIN_REGISTRY_ADDRESS=
-STAKE_MANAGER_ADDRESS=
-COMPUTE_POOL_ADDRESS=
-
-# Optional
-WORK_VALIDATION_CONTRACT=
-# Toploc configurations as JSON array string
-# Example: [{"server_url": "toploc url", "auth_token": "toploc auth token", "file_prefix_filter": "optional prefix filter"}]
-TOPLOC_CONFIGS=
+# Accounts (Anvil Test Accounts - DO NOT USE IN PRODUCTION)
+# ---------------------------------------------------------
+# Federator (Account #0)
+PRIVATE_KEY_FEDERATOR=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+FEDERATOR_ADDRESS=0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
+
+# Validator (Account #1)
+PRIVATE_KEY_VALIDATOR=0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+VALIDATOR_ADDRESS=0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+
+# Provider (Account #2)
+PRIVATE_KEY_PROVIDER=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a
+PROVIDER_ADDRESS=0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC
+
+# Node (Account #3)
+PRIVATE_KEY_NODE=0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6
+NODE_ADDRESS=0x90F79bf6EB2c4f870365E785982E1f101E93b906
+
+# Pool Owner (Account #5)
+POOL_OWNER_PRIVATE_KEY=0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba
+POOL_OWNER_ADDRESS=0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc
+
+
+# Storage (Optional)
+# ------------------
 S3_CREDENTIALS=
-BUCKET_NAME=
+HOURLY_S3_LIMIT=12
+BUCKET_NAME=sample-bucket
+
+# Location Service (Optional)
+# ---------------------------
+LOCATION_SERVICE_URL=https://ipapi.co
+LOCATION_SERVICE_API_KEY=
+
+# Orchestrator
+# ------------
+NODE_GROUP_CONFIGS='[{"name":"test-config","min_group_size":1,"max_group_size":1,"compute_requirements":null}]'
+
+# Validator
+# ---------
+TOPLOC_CONFIGS='[]'
+TOPLOC_SERVER_URL=
+TOPLOC_AUTH_TOKEN=
+VALIDATOR_API_KEY=prime
 
-# Webhook configurations as JSON array string
-# Example: [{"url": "webhook url", "bearer_token": "webhook bearer token"}]
-WEBHOOK_CONFIGS=
+# Webhooks (Optional)
+# -------------------
+WEBHOOK_CONFIGS='[]'
diff --git a/Makefile b/Makefile
@@ -90,11 +90,11 @@ watch-check:
 
 watch-validator:
 	set -a; source ${ENV_FILE}; set +a; \
-	cargo watch -w crates/validator/src -x "run --bin validator -- --validator-key $${PRIVATE_KEY_VALIDATOR} --rpc-url $${RPC_URL} --discovery-urls $${DISCOVERY_URLS:-$${DISCOVERY_URL:-http://localhost:8089}} --pool-id $${WORKER_COMPUTE_POOL_ID} --bucket-name $${BUCKET_NAME} -l $${LOG_LEVEL:-info} --toploc-grace-interval $${TOPLOC_GRACE_INTERVAL:-30} --incomplete-group-grace-period-minutes $${INCOMPLETE_GROUP_GRACE_PERIOD_MINUTES:-1} --use-grouping"
+	cargo watch -w crates/validator/src -x "run --bin validator -- --validator-key $${PRIVATE_KEY_VALIDATOR} --rpc-url $${RPC_URL} --discovery-urls $${DISCOVERY_URLS:-$${DISCOVERY_URL:-http://localhost:8089}} --pool-id $${WORKER_COMPUTE_POOL_ID} $${BUCKET_NAME:+--bucket-name $${BUCKET_NAME}} -l $${LOG_LEVEL:-info} --toploc-grace-interval $${TOPLOC_GRACE_INTERVAL:-30} --incomplete-group-grace-period-minutes $${INCOMPLETE_GROUP_GRACE_PERIOD_MINUTES:-1} --use-grouping"
 
 watch-orchestrator:
 	set -a; source ${ENV_FILE}; set +a; \
-	cargo watch -w crates/orchestrator/src -x "run --bin orchestrator -- -r $$RPC_URL -k $$POOL_OWNER_PRIVATE_KEY -d 0  -p 8090 -i 10 -u http://localhost:8090 --discovery-urls $${DISCOVERY_URLS:-$${DISCOVERY_URL:-http://localhost:8089}} --compute-pool-id $$WORKER_COMPUTE_POOL_ID --bucket-name $$BUCKET_NAME -l $${LOG_LEVEL:-info} --hourly-s3-upload-limit $${HOURLY_S3_LIMIT:-3} --max-healthy-nodes-with-same-endpoint $${MAX_HEALTHY_NODES_WITH_SAME_ENDPOINT:-2}"
+	cargo watch -w crates/orchestrator/src -x "run --bin orchestrator -- -r $$RPC_URL -k $$POOL_OWNER_PRIVATE_KEY -d 0  -p 8090 -i 10 -u http://localhost:8090 --discovery-urls $${DISCOVERY_URLS:-$${DISCOVERY_URL:-http://localhost:8089}} --compute-pool-id $$WORKER_COMPUTE_POOL_ID $${BUCKET_NAME:+--bucket-name $$BUCKET_NAME} -l $${LOG_LEVEL:-info} --hourly-s3-upload-limit $${HOURLY_S3_LIMIT:-3} --max-healthy-nodes-with-same-endpoint $${MAX_HEALTHY_NODES_WITH_SAME_ENDPOINT:-2}"
 
 build-worker:
 	cargo build --release --bin worker
diff --git a/crates/orchestrator/src/api/routes/storage.rs b/crates/orchestrator/src/api/routes/storage.rs
@@ -26,6 +26,13 @@ async fn request_upload(
     request_upload: web::Json<RequestUploadRequest>,
     app_state: Data<AppState>,
 ) -> HttpResponse {
+    let Some(storage_provider) = &app_state.storage_provider else {
+        return HttpResponse::InternalServerError().json(serde_json::json!({
+            "success": false,
+            "error": "Storage provider not found"
+        }));
+    };
+
     // Check file size limit first
     if request_upload.file_size > MAX_FILE_SIZE {
         return HttpResponse::PayloadTooLarge().json(serde_json::json!({
@@ -217,8 +224,7 @@ async fn request_upload(
 
     log::info!("Generating mapping file for sha256: {sha256} to file: {file_name}",);
 
-    if let Err(e) = app_state
-        .storage_provider
+    if let Err(e) = storage_provider
         .generate_mapping_file(sha256, &file_name)
         .await
     {
@@ -234,8 +240,7 @@ async fn request_upload(
     );
 
     // Generate signed upload URL
-    match app_state
-        .storage_provider
+    match storage_provider
         .generate_upload_signed_url(
             &file_name,
             Some(file_type.to_string()),
diff --git a/crates/orchestrator/src/api/server.rs b/crates/orchestrator/src/api/server.rs
@@ -117,7 +117,7 @@ async fn health_check(data: web::Data<AppState>) -> HttpResponse {
 
 pub struct AppState {
     pub store_context: Arc<StoreContext>,
-    pub storage_provider: Arc<dyn StorageProvider>,
+    pub storage_provider: Option<Arc<dyn StorageProvider>>,
     pub heartbeats: Arc<LoopHeartbeats>,
     pub redis_store: Arc<RedisStore>,
     pub hourly_upload_limit: i64,
@@ -135,7 +135,7 @@ pub async fn start_server(
     port: u16,
     store_context: Arc<StoreContext>,
     admin_api_key: String,
-    storage_provider: Arc<dyn StorageProvider>,
+    storage_provider: Option<Arc<dyn StorageProvider>>,
     heartbeats: Arc<LoopHeartbeats>,
     redis_store: Arc<RedisStore>,
     hourly_upload_limit: i64,
diff --git a/crates/orchestrator/src/api/tests/helper.rs b/crates/orchestrator/src/api/tests/helper.rs
@@ -57,7 +57,7 @@ pub async fn create_test_app_state() -> Data<AppState> {
         store_context: store_context.clone(),
         contracts: None,
         pool_id: 1,
-        storage_provider,
+        storage_provider: Some(storage_provider),
         heartbeats: Arc::new(LoopHeartbeats::new(&mode)),
         hourly_upload_limit: 12,
         redis_store: store.clone(),
@@ -127,7 +127,7 @@ pub async fn create_test_app_state_with_nodegroups() -> Data<AppState> {
         store_context: store_context.clone(),
         contracts: None,
         pool_id: 1,
-        storage_provider,
+        storage_provider: Some(storage_provider),
         heartbeats: Arc::new(LoopHeartbeats::new(&mode)),
         hourly_upload_limit: 12,
         redis_store: store.clone(),
@@ -193,7 +193,7 @@ pub async fn create_test_app_state_with_metrics() -> Data<AppState> {
         store_context: store_context.clone(),
         contracts: None,
         pool_id: 1,
-        storage_provider,
+        storage_provider: Some(storage_provider),
         heartbeats: Arc::new(LoopHeartbeats::new(&mode)),
         hourly_upload_limit: 12,
         redis_store: store.clone(),
diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs
@@ -400,13 +400,22 @@ async fn main() -> Result<()> {
 
     let port = args.port;
     let server_store_context = store_context.clone();
-
     let s3_credentials = std::env::var("S3_CREDENTIALS").ok();
-
-    let gcs_storage = GcsStorageProvider::new(&args.bucket_name.unwrap(), &s3_credentials.unwrap())
-        .await
-        .unwrap();
-    let storage_provider = Arc::new(gcs_storage);
+    let storage_provider: Option<Arc<dyn shared::utils::StorageProvider>> =
+        match (args.bucket_name.as_ref(), s3_credentials) {
+            (Some(bucket_name), Some(s3_credentials))
+                if !bucket_name.is_empty() && !s3_credentials.is_empty() =>
+            {
+                let gcs_storage = GcsStorageProvider::new(bucket_name, &s3_credentials)
+                    .await
+                    .unwrap_or_else(|_| panic!("Failed to create GCS storage provider"));
+                Some(Arc::new(gcs_storage) as Arc<dyn shared::utils::StorageProvider>)
+            }
+            _ => {
+                info!("Bucket name or S3 credentials not provided, storage provider disabled");
+                None
+            }
+        };
 
     // Always start server regardless of mode
     tokio::select! {
diff --git a/crates/validator/src/main.rs b/crates/validator/src/main.rs
@@ -341,34 +341,43 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                         std::process::exit(1);
                     }
                 };
-
                 let s3_credentials = std::env::var("S3_CREDENTIALS").ok();
-                let gcs_storage =
-                    GcsStorageProvider::new(&args.bucket_name.unwrap(), &s3_credentials.unwrap())
-                        .await
-                        .unwrap();
-                let storage_provider = Arc::new(gcs_storage);
-
-                Some(SyntheticDataValidator::new(
-                    pool_id,
-                    validator,
-                    contracts.prime_network.clone(),
-                    configs,
-                    penalty,
-                    storage_provider,
-                    redis_store,
-                    cancellation_token,
-                    args.toploc_work_validation_interval,
-                    args.toploc_work_validation_unknown_status_expiry_seconds,
-                    args.toploc_grace_interval,
-                    args.batch_trigger_size,
-                    args.use_grouping,
-                    args.disable_toploc_invalidation,
-                    args.incomplete_group_grace_period_minutes,
-                    args.toploc_invalidation_type,
-                    args.work_unit_invalidation_type,
-                    Some(metrics_ctx.clone()),
-                ))
+
+                match (args.bucket_name.as_ref(), s3_credentials) {
+                    (Some(bucket_name), Some(s3_credentials))
+                        if !bucket_name.is_empty() && !s3_credentials.is_empty() =>
+                    {
+                        let gcs_storage = GcsStorageProvider::new(bucket_name, &s3_credentials)
+                            .await
+                            .unwrap_or_else(|_| panic!("Failed to create GCS storage provider"));
+                        let storage_provider = Arc::new(gcs_storage);
+
+                        Some(SyntheticDataValidator::new(
+                            pool_id,
+                            validator,
+                            contracts.prime_network.clone(),
+                            configs,
+                            penalty,
+                            storage_provider,
+                            redis_store,
+                            cancellation_token,
+                            args.toploc_work_validation_interval,
+                            args.toploc_work_validation_unknown_status_expiry_seconds,
+                            args.toploc_grace_interval,
+                            args.batch_trigger_size,
+                            args.use_grouping,
+                            args.disable_toploc_invalidation,
+                            args.incomplete_group_grace_period_minutes,
+                            args.toploc_invalidation_type,
+                            args.work_unit_invalidation_type,
+                            Some(metrics_ctx.clone()),
+                        ))
+                    }
+                    _ => {
+                        info!("Bucket name or S3 credentials not provided, skipping synthetic data validation");
+                        None
+                    }
+                }
             }
             None => {
                 error!("Synthetic data validator not found");
diff --git a/docs/development-setup.md b/docs/development-setup.md
@@ -100,25 +100,6 @@ This will start:
 - Redis instance
 - Supporting infrastructure
 
-## Docker Compose Setup
-
-You can run all supporting services (chain, validator, discovery, orchestrator) in docker compose.
-
-1. Start docker compose:
-   ```bash
-   docker compose up
-   ```
-
-2. Run Setup:  
-   ```bash
-   make setup
-   ```
-
-3. Launch a worker:
-   - Adjust the .env var `WORKER_EXTERNAL_IP` to: `WORKER_EXTERNAL_IP=host.docker.internal` 
-   - Launch the worker using `make watch-worker`
-   - Whitelist the worker once you see the whitelist alert using: `make whitelist-provider`
-
 ## Running a Worker Node
 
 Once the core services are running, you can start a worker node in a new terminal:
@@ -127,9 +108,9 @@ make watch-worker
 ```
 
 The worker will automatically connect to the discovery service and begin processing tasks.
+Your worker will show an error that it is not whitelisted yet. You'll have to run `make whitelist-provider` here.
 It takes a couple of seconds until the worker is whitelisted. This is done using a simple loop on the second page of tmux.
-
-You can find more details on the APIs in the orchestrator and discovery service directory.
+You should see your worker eventually on the orchestrator. Checkout the orchestrator doc also: `http://localhost:8090/docs`
 
 ## Remote GPU Development
 > ⚠️ **IMPORTANT**: The video shows the whitelist process happening automatically. Currently, this must be done manually using the command `make whitelist-provider`.
@@ -155,4 +136,22 @@ Set up your remote GPU worker:
 To gracefully shutdown all services:
 ```bash
 make down
-```
+```
+
+## Docker Compose Setup
+You can run all supporting services (chain, validator, discovery, orchestrator) (if you only want to work on the worker sw) in docker compose.
+
+1. Start docker compose:
+   ```bash
+   docker compose up
+   ```
+
+2. Run Setup:  
+   ```bash
+   make setup
+   ```
+
+3. Launch a worker:
+   - Adjust the .env var `WORKER_EXTERNAL_IP` to: `WORKER_EXTERNAL_IP=host.docker.internal` 
+   - Launch the worker using `make watch-worker`
+   - Whitelist the worker once you see the whitelist alert using: `make whitelist-provider`
