Skip to content

Commit 27d36f0

Browse files
jvdtoornjvdtoorn
authored
feat: add calibration
* feat: simplify task accumulation in simulator * feat: add v1 of calibrator * feat: show power usage diagram on dashboard * feat: improve simulation timing
1 parent 95103d6 commit 27d36f0

File tree

161 files changed

+4328
-2286
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

161 files changed

+4328
-2286
lines changed

‎.devcontainer/devcontainer.json‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎.gitignore‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
1-
scratch/
1+
# OpenDT runtime directories
2+
.run_id
3+
data/*
4+
!data/.gitkeep
25

3-
### OpenDT runtime experiment files ###
4-
experiment.json
5-
.experiment.json.crc
6-
kafka/opendt_sim_inputs/
7-
opendt_sim_inputs/
8-
output/
9-
!output/README.md
10-
!output/.gitkeep
6+
scratch/
117

128
### VS Code ###
139
.vscode/
@@ -66,6 +62,8 @@ env/
6662
.uv/
6763
uv.lock
6864

65+
.ruff_cache/
66+
6967
# Python artifacts
7068
__pycache__/
7169
*.py[cod]

‎.python-version‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎Makefile‎

Lines changed: 78 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,19 @@ up: clean-volumes
2828
echo "❌ Error: Config file not found: $(config)"; \
2929
exit 1; \
3030
fi
31-
@if [ "$(build)" = "true" ]; then \
31+
@echo "🔧 Initializing run..."
32+
@$(PYTHON) scripts/opendt_cli.py init --config $(config)
33+
@RUN_ID=$$(cat .run_id) && \
34+
if [ ! -f "data/$$RUN_ID/.env" ]; then \
35+
echo "❌ Error: data/$$RUN_ID/.env not found after initialization"; \
36+
exit 1; \
37+
fi && \
38+
set -a && . ./data/$$RUN_ID/.env && set +a && \
39+
if [ "$(build)" = "true" ]; then \
3240
echo "🔨 Rebuilding Docker images (no cache)..."; \
33-
CONFIG_PATH=$(config) docker compose build --no-cache; \
34-
echo "✅ Images rebuilt!"; \
35-
fi
36-
CONFIG_PATH=$(config) docker compose up -d
41+
docker compose $$PROFILE_FLAG build --no-cache; \
42+
fi && \
43+
docker compose $$PROFILE_FLAG up -d
3744
@echo "✅ Services started!"
3845
@echo ""
3946
@echo "Available services:"
@@ -44,101 +51,29 @@ up: clean-volumes
4451
@echo ""
4552
@echo "View logs: make logs"
4653

47-
## up-debug: Start services in DEBUG mode (sim-worker writes results to ./output/ instead of Kafka)
48-
up-debug: clean-volumes
49-
@echo "🐛 Starting OpenDT services in DEBUG MODE..."
50-
@echo "📋 Using config: $(config)"
51-
@mkdir -p output
52-
@if [ ! -f "$(config)" ]; then \
53-
echo "❌ Error: Config file not found: $(config)"; \
54-
exit 1; \
55-
fi
56-
@if [ "$(build)" = "true" ]; then \
57-
echo "🔨 Rebuilding Docker images (no cache)..."; \
58-
CONFIG_PATH=$(config) DEBUG_MODE=true docker compose build --no-cache; \
59-
echo "✅ Images rebuilt!"; \
60-
fi
61-
CONFIG_PATH=$(config) DEBUG_MODE=true docker compose up -d
62-
@echo "✅ Services started in DEBUG mode!"
63-
@echo ""
64-
@echo "🐛 DEBUG MODE: sim-worker will write results to ./output/"
65-
@echo " Kafka publishing is DISABLED for sim-worker"
66-
@echo ""
67-
@echo "Available services:"
68-
@echo " - Dashboard: http://localhost:8000"
69-
@echo " - Postgres: localhost:5432"
70-
@echo " - Kafka: localhost:9092"
71-
@echo ""
72-
@echo "View logs: make logs-sim-worker"
73-
@echo "View results: ls -la output/"
74-
7554
## run: Alias for 'up' (accepts config parameter)
7655
run: up
7756

78-
## experiment: Run an experiment (make experiment name=<experiment_name>)
79-
experiment: clean-volumes
80-
@if [ -z "$(name)" ]; then \
81-
echo "❌ Error: Please provide experiment name: make experiment name=my_experiment"; \
82-
exit 1; \
83-
fi
84-
@if [ ! -f "config/experiments/$(name).yaml" ]; then \
85-
echo "❌ Error: Experiment config not found: config/experiments/$(name).yaml"; \
86-
echo ""; \
87-
echo "Available experiments:"; \
88-
ls -1 config/experiments/*.yaml 2>/dev/null | xargs -n 1 basename | sed 's/.yaml//' | sed 's/^/ - /' || echo " (none)"; \
89-
exit 1; \
90-
fi
91-
@echo "🧪 Starting experiment: $(name)"
92-
@echo "📋 Using config: config/experiments/$(name).yaml"
93-
@mkdir -p output/$(name)
94-
EXPERIMENT_NAME=$(name) CONFIG_PATH=./config/experiments/$(name).yaml docker compose up -d
95-
@echo "✅ Experiment started!"
96-
@echo ""
97-
@echo "Experiment: $(name)"
98-
@echo "Output: output/$(name)/"
99-
@echo ""
100-
@echo "View logs: make logs-sim-worker"
101-
102-
## experiment-debug: Run an experiment with debug mode enabled (make experiment-debug name=<experiment_name>)
103-
experiment-debug: clean-volumes
104-
@if [ -z "$(name)" ]; then \
105-
echo "❌ Error: Please provide experiment name: make experiment-debug name=my_experiment"; \
106-
exit 1; \
107-
fi
108-
@if [ ! -f "config/experiments/$(name).yaml" ]; then \
109-
echo "❌ Error: Experiment config not found: config/experiments/$(name).yaml"; \
110-
echo ""; \
111-
echo "Available experiments:"; \
112-
ls -1 config/experiments/*.yaml 2>/dev/null | xargs -n 1 basename | sed 's/.yaml//' | sed 's/^/ - /' || echo " (none)"; \
113-
exit 1; \
114-
fi
115-
@echo "🧪🐛 Starting experiment with DEBUG mode: $(name)"
116-
@echo "📋 Using config: config/experiments/$(name).yaml"
117-
@mkdir -p output/$(name)
118-
EXPERIMENT_NAME=$(name) DEBUG_MODE=true CONFIG_PATH=./config/experiments/$(name).yaml docker compose up -d
119-
@echo "✅ Experiment started in debug mode!"
120-
@echo ""
121-
@echo "Experiment: $(name)"
122-
@echo "Output: output/$(name)/"
123-
@echo "Debug files: output/$(name)/run_*/"
124-
@echo ""
125-
@echo "View logs: make logs-sim-worker"
126-
127-
## experiment-down: Stop experiment services
128-
experiment-down:
129-
docker compose down
130-
@echo "✅ Experiment stopped"
131-
13257
## down: Stop all containers
13358
down:
13459
@echo "⏹️ Stopping OpenDT services..."
135-
CONFIG_PATH=$(config) docker compose down
60+
@RUN_ID=$$(cat .run_id 2>/dev/null || true); \
61+
if [ -n "$$RUN_ID" ] && [ -f "data/$$RUN_ID/.env" ]; then \
62+
set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG down; \
63+
else \
64+
docker compose down; \
65+
fi
13666
@echo "✅ Services stopped!"
13767

13868
## clean-volumes: Stop containers and delete persistent volumes (Kafka & Postgres)
13969
clean-volumes:
14070
@echo "🧹 Stopping containers and cleaning persistent volumes..."
141-
CONFIG_PATH=$(config) docker compose down -v
71+
@RUN_ID=$$(cat .run_id 2>/dev/null || true); \
72+
if [ -n "$$RUN_ID" ] && [ -f "data/$$RUN_ID/.env" ]; then \
73+
set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG down -v; \
74+
else \
75+
docker compose down -v; \
76+
fi
14277
@echo "🗑️ Removing named volumes..."
14378
-docker volume rm opendt-postgres-data 2>/dev/null || true
14479
-docker volume rm opendt-kafka-data 2>/dev/null || true
@@ -147,13 +82,25 @@ clean-volumes:
14782
## restart: Restart all services (without cleaning volumes)
14883
restart:
14984
@echo "♻️ Restarting OpenDT services..."
150-
CONFIG_PATH=$(config) docker compose restart
85+
@RUN_ID=$$(cat .run_id 2>/dev/null || true); \
86+
if [ -n "$$RUN_ID" ] && [ -f "data/$$RUN_ID/.env" ]; then \
87+
set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG restart; \
88+
else \
89+
echo "⚠️ Run environment not found, restarting without profile"; \
90+
docker compose restart; \
91+
fi
15192
@echo "✅ Services restarted!"
15293

15394
## build: Rebuild all Docker images
15495
build:
15596
@echo "🔨 Building Docker images..."
156-
CONFIG_PATH=$(config) docker compose build --no-cache
97+
@RUN_ID=$$(cat .run_id 2>/dev/null || true); \
98+
if [ -n "$$RUN_ID" ] && [ -f "data/$$RUN_ID/.env" ]; then \
99+
set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG build --no-cache; \
100+
else \
101+
echo "⚠️ Run environment not found, building without profile"; \
102+
docker compose build --no-cache; \
103+
fi
157104
@echo "✅ Images built!"
158105

159106
## rebuild: Clean, rebuild (no cache), and start (alias for make up build=true)
@@ -212,39 +159,67 @@ clean-env:
212159

213160
## logs: Tail logs for all services
214161
logs:
215-
CONFIG_PATH=$(config) docker compose logs -f
162+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG logs -f
216163

217164
## logs-dashboard: Tail logs for dashboard service only
218165
logs-dashboard:
219-
CONFIG_PATH=$(config) docker compose logs -f dashboard
166+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose logs -f dashboard
220167

221168
## logs-kafka: Tail logs for Kafka service only
222169
logs-kafka:
223-
CONFIG_PATH=$(config) docker compose logs -f kafka
170+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose logs -f kafka
224171

225172
## logs-dc-mock: Tail logs for dc-mock service only
226173
logs-dc-mock:
227-
CONFIG_PATH=$(config) docker compose logs -f dc-mock
228-
229-
## logs-sim-worker: Tail logs for sim-worker service only
230-
logs-sim-worker:
231-
CONFIG_PATH=$(config) docker compose logs -f sim-worker
174+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose logs -f dc-mock
175+
176+
## logs-simulator: Tail logs for simulator service only
177+
logs-simulator:
178+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose logs -f simulator
179+
180+
## logs-calibrator: Tail logs for calibrator service only
181+
logs-calibrator:
182+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose --profile calibration logs -f calibrator
183+
184+
## up-with-calibration: Start services including calibrator
185+
# up-with-calibration: clean-volumes
186+
# @echo "🚀 Starting OpenDT services with calibration enabled..."
187+
# @echo "📋 Using config: $(config)"
188+
# @if [ ! -f "$(config)" ]; then \
189+
# echo "❌ Error: Config file not found: $(config)"; \
190+
# exit 1; \
191+
# fi
192+
# @if [ "$(build)" = "true" ]; then \
193+
# echo "🔨 Rebuilding Docker images (no cache)..."; \
194+
# CONFIG_PATH=$(config) docker compose --profile calibration build --no-cache; \
195+
# echo "✅ Images rebuilt!"; \
196+
# fi
197+
# CONFIG_PATH=$(config) docker compose --profile calibration up -d
198+
# @echo "✅ Services started with calibration!"
199+
# @echo ""
200+
# @echo "Available services:"
201+
# @echo " - Dashboard: http://localhost:8000"
202+
# @echo " - Calibrator: Automatic power model calibration"
203+
# @echo " - Postgres: localhost:5432"
204+
# @echo " - Kafka: localhost:9092"
205+
# @echo ""
206+
# @echo "View logs: make logs-calibrator"
232207

233208
## ps: Show running containers
234209
ps:
235-
CONFIG_PATH=$(config) docker compose ps
210+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose $$PROFILE_FLAG ps
236211

237212
## shell-dashboard: Open a shell in the dashboard container
238213
shell-dashboard:
239-
CONFIG_PATH=$(config) docker compose exec dashboard /bin/bash
214+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose exec dashboard /bin/bash
240215

241216
## shell-postgres: Open psql in the Postgres container
242217
shell-postgres:
243-
CONFIG_PATH=$(config) docker compose exec postgres psql -U opendt -d opendt
218+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose exec postgres psql -U opendt -d opendt
244219

245220
## kafka-topics: List Kafka topics
246221
kafka-topics:
247-
CONFIG_PATH=$(config) docker compose exec kafka kafka-topics --bootstrap-server localhost:9092 --list
222+
@RUN_ID=$$(cat .run_id) && set -a && . ./data/$$RUN_ID/.env && set +a && docker compose exec kafka kafka-topics --bootstrap-server localhost:9092 --list
248223

249224
## help: Show this help message
250225
help:

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ workload: "SURF" # Data directory name
154154
simulation:
155155
speed_factor: 300 # 300x real-time
156156
window_size_minutes: 5 # 5-minute windows
157-
heartbeat_cadence_minutes: 1
157+
heartbeat_frequency_minutes: 1
158158
experiment_mode: false
159159

160160
kafka:

‎config/default.yaml‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
workload: "SURF"
1+
# OpenDT Default Configuration
22

3-
simulation:
4-
# 1.0 = Realtime, -1 = Max Speed
5-
speed_factor: 300
6-
window_size_minutes: 5
7-
heartbeat_cadence_minutes: 1
8-
experiment_mode: false
9-
10-
features:
3+
global:
4+
speed_factor: 1 # 1.0 = Realtime, -1 = Max Speed
115
calibration_enabled: false
126

7+
services:
8+
dc-mock:
9+
workload: "SURF"
10+
heartbeat_frequency_minutes: 1
11+
12+
simulator:
13+
simulation_frequency_minutes: 5
14+
1315
kafka:
14-
bootstrap_servers: "kafka:29092"
1516
topics:
1617
workload:
1718
name: "dc.workload"

‎config/experiments/experiment_1.yaml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
1-
workload: "SURF"
1+
# OpenDT Experiment Configuration
22

3-
simulation:
4-
# 1.0 = Realtime, -1 = Max Speed
5-
speed_factor: 300
6-
window_size_minutes: 15
7-
heartbeat_cadence_minutes: 1
8-
# Experiment mode: write results to parquet
9-
experiment_mode: true
10-
11-
features:
3+
global:
4+
speed_factor: 300 # 1.0 = Realtime, -1 = Max Speed
125
calibration_enabled: false
136

7+
services:
8+
dc-mock:
9+
workload: "SURF"
10+
heartbeat_frequency_minutes: 1
11+
12+
simulator:
13+
simulation_frequency_minutes: 15
14+
1415
kafka:
15-
bootstrap_servers: "kafka:29092"
1616
topics:
1717
workload:
1818
name: "dc.workload"

0 commit comments

Comments
 (0)