Merge pull request #187 from DataCloud-project/carbontracker

Carbontracker
* graphql api endpoint
* carbontracker service

Author: goranbs

.github/workflows/carbontracker-image.yaml

@@ -0,0 +1,55 @@
+name: Create and publish the carbontracker docker image
+
+on:
+  push:
+    branches:
+      - main
+      - carbontracker
+    tags:
+      - "*"
+  pull_request:
+    paths:
+      - "carbontracker/**"
+
+env:
+  REGISTRY: ghcr.io
+  carbontracker_IMAGE_NAME: datacloud-project/sim-pipe-carbontracker
+
+jobs:
+  build-and-push-carbontracker-image:
+    name: Build and push carbontracker image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for carbontracker main image
+        id: meta-main
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.carbontracker_IMAGE_NAME }}
+
+      - name: Build and push Docker carbontracker image
+        uses: docker/build-push-action@v4
+        with:
+          context: ./carbontracker/
+          push: true
+          tags: ${{ steps.meta-main.outputs.tags }}
+          labels: ${{ steps.meta-main.outputs.labels }}
+          platforms: linux/arm64,linux/amd64

carbontracker/Dockerfile

@@ -0,0 +1,28 @@
+# File: Dockerfile
+FROM python:3.13-slim
+
+WORKDIR /app
+
+# Install system dependencies for reportlab and other packages
+RUN apt-get update && \
+    apt-get install -y \
+        git \
+        build-essential \
+        libfreetype6-dev \
+        libjpeg-dev \
+        libpng-dev \
+        zlib1g-dev \
+        libffi-dev && \
+    pip install uv && \
+    rm -rf /var/lib/apt/lists/*
+
+
+COPY pyproject.toml ./
+COPY src/ ./src/
+RUN uv sync
+
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
+
+EXPOSE 8000
+CMD ["uv", "run", "uvicorn", "fastapi_agent.main:app", "--host", "0.0.0.0", "--port", "8000"]

carbontracker/README.md

@@ -0,0 +1,42 @@
+# carbontracker fastapi agent
+
+Simple fastapi server hosting `process` endpoint that takes cpu json data as input
+
+Build image
+```bash
+docker build -t carbontracker-agent .
+```
+
+Run image as server
+```bash
+docker run -p 8000:8000 carbontracker-agent
+```
+
+Call the fastapi endpoint
+```bash
+jq -Rs '{data: .}' sample_simpipe_metrics.json | curl -X POST http://localhost:8000/process -H 'Content-Type: application/json' -d @-
+```
+
+Output
+```json
+{"stdout":"CarbonTracker: INFO - Detected CPU: -\nCarbonTracker: INFO - Matched CPU - to 6th Gen A10-8700P APU with TDP 15.0W\nCarbonTracker: INFO - Using TDP of 7.50W for -\nCarbonTracker: WARNING - ElectricityMaps API key not set. Will default to average carbon intensity.\nCarbonTracker: WARNING - ElectricityMaps API key not set. Will default to average carbon intensity.\nCarbonTracker: WARNING - Failed to retrieve carbon intensity: Defaulting to average carbon intensity 30.080084 gCO2/kWh.\nCarbonTracker: WARNING - Failed to retrieve carbon intensity: Defaulting to average carbon intensity 30.080084 gCO2/kWh.\n\n=== CarbonTracker Simulation Results ===\nSource: data.json\n========================================\n\nSimulation Details:\n+ Duration: 0:00:11.00\n+ CPU Model: -\n+ CPU Power Data: Available\n+ Average Power Usage: 7.50 W\n+ Energy Usage: 18.37 J\n+ Location: Trondheim, Trøndelag, NO\n+ Carbon Intensity: 30.08 gCO2/kWh\n\nConsumption Summary:\n+ Energy: 0.000005 kWh\n+ CO2eq: 0.000 g\n\nEnvironmental Impact Equivalents:\n+ 0.000001 km travelled by car\n\nSimulation log written to ./logs\n","stderr":"   Building fastapi-agent @ file:///app\n      Built fastapi-agent @ file:///app\nUninstalled 1 package in 0.24ms\nInstalled 1 package in 1ms\n","parsed":"{\n  \"Duration\": \"0:00:11.00\",\n  \"CPU Model\": \"-\",\n  \"CPU Power Data\": \"Available\",\n  \"Average Power Usage\": \"7.50 W\",\n  \"Energy Usage\": \"18.37 J\",\n  \"Location\": \"Trondheim, Trøndelag, NO\",\n  \"Carbon Intensity\": \"30.08 gCO2/kWh\",\n  \"Energy\": \"0.000005 kWh\",\n  \"CO2eq\": \"0.000 g\"\n}"}
+```
+
+Parse json output:
+```python
+p = json.loads(data["parsed"])
+print(p)
+```
+
+will print:
+```json
+{'Duration': '0:00:11.00',
+ 'CPU Model': '-',
+ 'CPU Power Data': 'Available',
+ 'Average Power Usage': '7.50 W',
+ 'Energy Usage': '18.37 J',
+ 'Location': 'Trondheim, Trøndelag, NO',
+ 'Carbon Intensity': '30.08 gCO2/kWh',
+ 'Energy': '0.000005 kWh',
+ 'CO2eq': '0.000 g'}
+ ```

carbontracker/pyproject.toml

@@ -0,0 +1,20 @@
+# File: pyproject.toml
+[build-system]
+requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fastapi-agent"
+version = "0.2.0"
+requires-python = ">=3.13"
+dependencies = [
+    "fastapi",
+    "uvicorn[standard]"
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+packages = ["fastapi_agent"]
+
+[project.scripts]
+fastapi-agent = "main:app"

carbontracker/requirements.txt

@@ -0,0 +1,2 @@
+# Carbontracker (tags: greenr)
+git+https://github.com/lfwa/carbontracker.git@greenr

carbontracker/src/fastapi_agent/main.py

@@ -0,0 +1,42 @@
+# File: main.py
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+import json
+from .utils import store_data_temporarily, delete_temporary_data, delete_logs, parse_carbontracker_stdout
+import subprocess
+
+app = FastAPI()
+
+class InputData(BaseModel):
+    data: str
+
+@app.post("/process")
+async def process(
+    data: InputData, 
+    sim_cpu: str = None, sim_cpu_tdp: str = None, sim_cpu_util: str = None,
+    sim_gpu: str = None, sim_gpu_watts: str = None, sim_gpu_util: str = None):
+    store_data_temporarily(data.data) # is stored as data.json
+    command = ["uv", "run", "carbontracker", "--simpipe", "data.json"]
+    if sim_cpu:
+        command.extend(["--sim-cpu", sim_cpu]) # Not working yet (CLI option not implemented?)
+    completed_process = subprocess.run(
+        ["uv", "run", "carbontracker", "--simpipe", "data.json"],
+        check=True,
+        capture_output=True,
+        text=True
+    )
+    delete_temporary_data()
+    parsed_output = parse_carbontracker_stdout(completed_process.stdout)
+    parsed_output_str = json.dumps(parsed_output, indent=2, ensure_ascii=False)
+    return {"stdout": completed_process.stdout, "stderr": completed_process.stderr, "parsed": parsed_output_str}
+
+@app.get("/delete-logs")
+async def delete_carbontracker_logs():
+    try:
+        delete_logs()
+    except Exception as e:
+        print(f"Error deleting temporary data: {e}")
+
+@app.get("/health")
+async def health_check():
+    return {"status": "ok"}

carbontracker/src/fastapi_agent/utils.py

@@ -0,0 +1,57 @@
+# File: agent.py
+import os
+import re
+
+
+def process_input(data: str) -> str:
+    # Minimal business logic placeholder
+    return f"Agent processed: {data}"
+
+
+def store_data_temporarily(data: str) -> str:
+    # Placeholder for storing data temporarily
+    with open("data.json", "w") as f:
+        f.write(data)
+    return "Data stored temporarily in data.json"
+
+
+def delete_temporary_data() -> str:
+    # Placeholder for deleting temporary data
+    if os.path.exists("data.json"):
+        os.remove("data.json")
+        return "Temporary data deleted"
+    return "No temporary data to delete"
+
+
+def delete_logs() -> str:
+    # Placeholder for deleting logs
+    if os.path.exists("logs"):
+        os.remove("logs/*.log")
+        return "Deleted logs"
+    return "No logs to delete"
+
+
+def parse_carbontracker_stdout(stdout: str) -> dict:
+    result = {}
+
+    # Patterns for each field
+    patterns = {
+        "Duration": r"\+ Duration:\s*([^\n]+)",
+        "CPU Model": r"\+ CPU Model:\s*([^\n]+)",
+        "CPU Power Data": r"\+ CPU Power Data:\s*([^\n]+)",
+        "Average Power Usage": r"\+ Average Power Usage:\s*([^\n]+)",
+        "Energy Usage": r"\+ Energy Usage:\s*([^\n]+)",
+        "Location": r"\+ Location:\s*([^\n]+)",
+        "Carbon Intensity": r"\+ Carbon Intensity:\s*([^\n]+)",
+        "Energy": r"\+ Energy:\s*([^\n]+)",
+        "CO2eq": r"\+ CO2eq:\s*([^\n]+)",
+    }
+
+    for key, pattern in patterns.items():
+        match = re.search(pattern, stdout)
+        if match:
+            result[key] = match.group(1).strip()
+        else:
+            result[key] = None
+
+    return result

charts/simpipe/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: simpipe
 description: SIMPIPE on Kubernetes
 type: application
-version: 0.4.8
+version: 0.4.9
 appVersion: 0.4.8
 dependencies:
   - name: cadvisor

charts/simpipe/templates/_helpers-carbontracker.tpl

@@ -0,0 +1,58 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "simpipeCarbontracker.name" -}}
+{{- default (printf "%s-carbontracker" .Chart.Name) .Values.carbontracker.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "simpipeCarbontracker.fullname" -}}
+{{- if .Values.carbontracker.fullnameOverride }}
+{{- .Values.carbontracker.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default (printf "%s-carbontracker" .Chart.Name) .Values.carbontracker.nameOverride }}
+{{- if contains $name (printf "%s-carbontracker" .Release.Name) }}
+{{- printf "%s-carbontracker" .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s-carbontracker" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "simpipeCarbontracker.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "simpipeCarbontracker.labels" -}}
+helm.sh/chart: {{ include "simpipeCarbontracker.chart" . }}
+{{ include "simpipeCarbontracker.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "simpipeCarbontracker.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "simpipeCarbontracker.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "simpipeCarbontracker.serviceAccountName" -}}
+{{- default (include "simpipeCarbontracker.fullname" .) .Values.carbontracker.serviceAccount.name }}
+{{- end }}