feat: add experiment 1

Author: jvdtoorn

Makefile

@@ -1,4 +1,4 @@
-.PHONY: up down clean-volumes restart build logs help run test setup install-dev clean-env verify
+.PHONY: up down clean-volumes restart build logs help run test setup install-dev clean-env verify experiment experiment-down
 
 # Default target
 .DEFAULT_GOAL := help
@@ -77,6 +77,60 @@ up-debug: clean-volumes
 ## run: Alias for 'up' (accepts config parameter)
 run: up
 
+## experiment: Run an experiment (make experiment name=<experiment_name>)
+experiment: clean-volumes
+	@if [ -z "$(name)" ]; then \
+		echo "❌ Error: Please provide experiment name: make experiment name=my_experiment"; \
+		exit 1; \
+	fi
+	@if [ ! -f "config/experiments/$(name).yaml" ]; then \
+		echo "❌ Error: Experiment config not found: config/experiments/$(name).yaml"; \
+		echo ""; \
+		echo "Available experiments:"; \
+		ls -1 config/experiments/*.yaml 2>/dev/null | xargs -n 1 basename | sed 's/.yaml//' | sed 's/^/  - /' || echo "  (none)"; \
+		exit 1; \
+	fi
+	@echo "🧪 Starting experiment: $(name)"
+	@echo "📋 Using config: config/experiments/$(name).yaml"
+	@mkdir -p output/$(name)
+	EXPERIMENT_NAME=$(name) CONFIG_PATH=./config/experiments/$(name).yaml docker compose up -d
+	@echo "✅ Experiment started!"
+	@echo ""
+	@echo "Experiment: $(name)"
+	@echo "Output: output/$(name)/"
+	@echo ""
+	@echo "View logs: make logs-sim-worker"
+
+## experiment-debug: Run an experiment with debug mode enabled (make experiment-debug name=<experiment_name>)
+experiment-debug: clean-volumes
+	@if [ -z "$(name)" ]; then \
+		echo "❌ Error: Please provide experiment name: make experiment-debug name=my_experiment"; \
+		exit 1; \
+	fi
+	@if [ ! -f "config/experiments/$(name).yaml" ]; then \
+		echo "❌ Error: Experiment config not found: config/experiments/$(name).yaml"; \
+		echo ""; \
+		echo "Available experiments:"; \
+		ls -1 config/experiments/*.yaml 2>/dev/null | xargs -n 1 basename | sed 's/.yaml//' | sed 's/^/  - /' || echo "  (none)"; \
+		exit 1; \
+	fi
+	@echo "🧪🐛 Starting experiment with DEBUG mode: $(name)"
+	@echo "📋 Using config: config/experiments/$(name).yaml"
+	@mkdir -p output/$(name)
+	EXPERIMENT_NAME=$(name) DEBUG_MODE=true CONFIG_PATH=./config/experiments/$(name).yaml docker compose up -d
+	@echo "✅ Experiment started in debug mode!"
+	@echo ""
+	@echo "Experiment: $(name)"
+	@echo "Output: output/$(name)/"
+	@echo "Debug files: output/$(name)/run_*/"
+	@echo ""
+	@echo "View logs: make logs-sim-worker"
+
+## experiment-down: Stop experiment services
+experiment-down:
+	docker compose down
+	@echo "✅ Experiment stopped"
+
 ## down: Stop all containers
 down:
 	@echo "⏹️  Stopping OpenDT services..."

config/experiments/experiment_1.yaml

@@ -0,0 +1,42 @@
+workload: "SURF"
+
+simulation:
+  # 1.0 = Realtime, -1 = Max Speed
+  speed_factor: 300
+  window_size_minutes: 15
+  heartbeat_cadence_minutes: 1
+  # Experiment mode: write results to parquet instead of Kafka
+  experiment_mode: true
+
+features:
+  calibration_enabled: false
+
+kafka:
+  bootstrap_servers: "kafka:29092"
+  topics:
+    workload:
+      name: "dc.workload"
+      config:
+        # 24 hours
+        retention.ms: "86400000"
+    power:
+      name: "dc.power"
+      config:
+        # 1 hour
+        retention.ms: "3600000"
+    topology:
+      name: "dc.topology"
+      config:
+        # Compacted topic - keeps latest topology
+        cleanup.policy: "compact"
+        # Keep at least 1 hour
+        min.compaction.lag.ms: "3600000"
+    system:
+      name: "sys.config"
+      config:
+        cleanup.policy: "compact"
+    results:
+      name: "sim.results"
+      config:
+        # 7 days retention for simulation results
+        retention.ms: "604800000"

docker-compose.yml

@@ -132,9 +132,12 @@ services:
       KAFKA_BOOTSTRAP_SERVERS: kafka:29092
       CONFIG_FILE: /app/config/simulation.yaml
       PYTHONUNBUFFERED: 1
-      # Debug mode: set to "true" to write results to files instead of Kafka
+      # Debug mode: set to "true" to write debug files alongside main output
       DEBUG_MODE: ${DEBUG_MODE:-false}
       DEBUG_OUTPUT_DIR: /app/output
+      # Experiment mode: experiment name (used for output directory)
+      EXPERIMENT_NAME: ${EXPERIMENT_NAME:-default}
+      EXPERIMENT_OUTPUT_DIR: /app/output
     volumes:
       # Mount source code for hot reload
       - ./services/sim-worker:/app/services/sim-worker

libs/common/opendt_common/config.py

@@ -47,6 +47,10 @@ class SimConfig(BaseModel):
     heartbeat_cadence_minutes: int = Field(
         default=1, description="Cadence in simulation minutes for workload heartbeat messages", gt=0
     )
+    experiment_mode: bool = Field(
+        default=False,
+        description="Enable experiment mode (write results to parquet instead of Kafka)",
+    )
 
     @field_validator("speed_factor")
     @classmethod

pyproject.toml

@@ -8,18 +8,19 @@ dependencies = [
     "pandas>=2.0.0",
     "pyarrow>=13.0.0",
     "pyyaml>=6.0.0",
-    
+    "matplotlib>=3.7.0",
+
     # Kafka (used by most services)
     "kafka-python>=2.0.2",
-    
+
     # Validation
     "pydantic>=2.0.0",
     "pydantic-settings>=2.0.0",
-    
+
     # Web framework (for API service)
     "fastapi>=0.104.0",
     "uvicorn[standard]>=0.24.0",
-    
+
     # Database (for services that need it)
     "psycopg2-binary>=2.9.0",
     "sqlalchemy>=2.0.0",