Merge pull request #316 from mlco2/use-cpu-load

[Version 3] Add CPU load tracking

Author: benoit-cty

.github/workflows/build.yml

@@ -165,3 +165,28 @@ jobs:
       - name: Check formatting with Prettier
         working-directory: ./webapp
         run: npx prettier . --check
+
+  TEMP_publish_release_candidate:
+    runs-on: ubuntu-latest
+    needs: [ python-test ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Check versions
+        run: |
+            pip install -U pip requests
+            python3 .github/check_version.py
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install hatch==1.13.0 hatchling==1.25.0
+      - name: Build package
+        run: hatch build -c
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/release-drafter.yml

@@ -10,6 +10,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"
-      - uses: release-drafter/release-drafter@v5.7.0
+      - uses: release-drafter/release-drafter@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -248,11 +248,15 @@ Here is the launch.json to be able to debug examples and tests:
 
 ```json
 {
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+
         {
             "name": "Python: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${file}",
             "console": "integratedTerminal",
@@ -261,13 +265,17 @@ Here is the launch.json to be able to debug examples and tests:
         },
         {
             "name": "PyTest: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "module": "pytest",
-            "args": ["${file}"],
+            "args": [
+                "-s",
+                "${file}"
+            ],
             "console": "integratedTerminal",
             "justMyCode": true,
-            "env": { "PYTHONPATH": "${workspaceRoot}" }
+            "env": { "PYTHONPATH": "${workspaceRoot}",
+            "CODECARBON_ALLOW_MULTIPLE_RUNS": "True"  }
         },
         {
             "name": "PyTest: codecarbon monitor",
@@ -279,7 +287,7 @@ Here is the launch.json to be able to debug examples and tests:
             ],
             "console": "integratedTerminal",
             "justMyCode": true,
-            "env": { "PYTHONPATH": "${workspaceRoot}" }
+            "env": { "PYTHONPATH": "${workspaceRoot}"}
         }
     ]
 }
@@ -350,7 +358,7 @@ to regenerate the html files.
 ### Release process
 
 -   Merge all PRs.
--   Create a PR bumping the version with `hatch run dev:bumpver update --patch`.
+-   Create a PR bumping the version with `hatch run dev:bumpver update --patch`. For a release candidate, use `hatch run dev:bumpver update --set-version  3.0.0_rc1`.
 -   Run `hatch run python3 .github/check_version.py` to check version consistancy.
 -   Update the dependencies with `hatch-pip-compile --upgrade --all`.
 -   [Build Documentation](#documentation) if needed with `hatch run docs:build`.

README.md

@@ -172,3 +172,9 @@ Here is a sample for BibTeX:
 # Contact 📝
 
 Maintainers are [@vict0rsch](https://github.com/vict0rsch) [@benoit-cty](https://github.com/benoit-cty) and [@SaboniAmine](https://github.com/saboniamine). Codecarbon is developed by volunteers from [**Mila**](http://mila.quebec) and the [**DataForGoodFR**](https://twitter.com/dataforgood_fr) community alongside donated professional time of engineers at [**Comet.ml**](https://comet.ml) and [**BCG GAMMA**](https://www.bcg.com/en-nl/beyond-consulting/bcg-gamma/default).
+
+## Star History
+
+Comparison of the number of stars accumulated by the different Python CO2 emissions projects:
+[![Star History Chart](https://api.star-history.com/svg?repos=mlco2/codecarbon,lfwa/carbontracker,sb-ai-lab/Eco2AI,fvaleye/tracarbon,Breakend/experiment-impact-tracker&type=Date)](https://star-history.com/#mlco2/codecarbon&lfwa/carbontracker&sb-ai-lab/Eco2AI&fvaleye/tracarbon&Breakend/experiment-impact-tracker&Date)
+

codecarbon/_version.py

@@ -1 +1 @@
-__version__ = "3.0.0-rc0"
+__version__ = "3.0.0_rc1"

codecarbon/core/api_client.py

@@ -257,8 +257,8 @@ def _create_run(self, experiment_id):
                 gpu_count=self.conf.get("gpu_count"),
                 gpu_model=self.conf.get("gpu_model"),
                 # Reduce precision for Privacy
-                longitude=round(self.conf.get("longitude"), 1),
-                latitude=round(self.conf.get("latitude"), 1),
+                longitude=round(self.conf.get("longitude", 0), 1),
+                latitude=round(self.conf.get("latitude", 0), 1),
                 region=self.conf.get("region"),
                 provider=self.conf.get("provider"),
                 ram_total_size=self.conf.get("ram_total_size"),

codecarbon/core/cpu.py

@@ -5,12 +5,14 @@
 """
 
 import os
+import re
 import shutil
 import subprocess
 import sys
 from typing import Dict, Optional, Tuple
 
 import pandas as pd
+import psutil
 from rapidfuzz import fuzz, process, utils
 
 from codecarbon.core.rapl import RAPLFile
@@ -19,6 +21,9 @@
 from codecarbon.external.logger import logger
 from codecarbon.input import DataSource
 
+# default W value per core for a CPU if no model is found in the ref csv
+DEFAULT_POWER_PER_CORE = 4
+
 
 def is_powergadget_available() -> bool:
     """
@@ -58,6 +63,21 @@ def is_rapl_available() -> bool:
         return False
 
 
+def is_psutil_available():
+    try:
+        nice = psutil.cpu_times().nice
+        if nice > 0.1:
+            return True
+        else:
+            return False
+    except Exception as e:
+        logger.debug(
+            "Not using the psutil interface, an exception occurred while instantiating "
+            + f"psutil.cpu_percent : {e}",
+        )
+        return False
+
+
 class IntelPowerGadget:
     """
     A class to interface with Intel Power Gadget for monitoring CPU power consumption on Windows and (non-Apple Silicon) macOS.
@@ -237,7 +257,7 @@ class IntelRAPL:
 
     """
 
-    def __init__(self, rapl_dir="/sys/class/powercap/intel-rapl"):
+    def __init__(self, rapl_dir="/sys/class/powercap/intel-rapl/subsystem"):
         self._lin_rapl_dir = rapl_dir
         self._system = sys.platform.lower()
         self._rapl_files = []
@@ -275,6 +295,8 @@ def _fetch_rapl_files(self) -> None:
             with open(path) as f:
                 name = f.read().strip()
                 # Fake the name used by Power Gadget
+                # We ignore "core" in name as it seems to be included in "package" for Intel CPU.
+                # TODO: Use "dram" for memory power
                 if "package" in name:
                     name = f"Processor Energy Delta_{i}(kWh)"
                     i += 1
@@ -294,7 +316,7 @@ def _fetch_rapl_files(self) -> None:
                     logger.debug("We will read Intel RAPL files at %s", rapl_file)
                 except PermissionError as e:
                     raise PermissionError(
-                        "Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
+                        "PermissionError : Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
                         + " Please view https://github.com/mlco2/codecarbon/issues/244"
                         + " for workarounds : %s",
                         e,
@@ -332,8 +354,6 @@ def get_static_cpu_details(self) -> Dict:
         """
         Return CPU details without computing them.
         """
-        logger.debug("get_static_cpu_details %s", self._cpu_details)
-
         return self._cpu_details
 
     def start(self) -> None:
@@ -426,6 +446,18 @@ def _get_matching_cpu(
         start_cpu = model_raw.find(" CPU @ ")
         if start_cpu > 0:
             model_raw = model_raw[0:start_cpu]
+        model_raw = model_raw.replace(" CPU", "")
+        model_raw = re.sub(r" @\s*\d+\.\d+GHz", "", model_raw)
+        direct_match = process.extractOne(
+            model_raw,
+            cpu_df["Name"],
+            processor=lambda s: s.lower(),
+            scorer=fuzz.ratio,
+            score_cutoff=THRESHOLD_DIRECT,
+        )
+
+        if direct_match:
+            return direct_match[0]
         indirect_matches = process.extract(
             model_raw,
             cpu_df["Name"],
@@ -467,10 +499,18 @@ def _main(self) -> Tuple[str, int]:
                 + " Please contact us.",
                 cpu_model_detected,
             )
+            if is_psutil_available():
+                # Count thread of the CPU
+                threads = psutil.cpu_count(logical=True)
+                estimated_tdp = threads * DEFAULT_POWER_PER_CORE
+                logger.warning(
+                    f"We will use the default power consumption of {DEFAULT_POWER_PER_CORE} W per thread for your {threads} CPU, so {estimated_tdp}W."
+                )
+                return cpu_model_detected, estimated_tdp
             return cpu_model_detected, None
         logger.warning(
             "We were unable to detect your CPU using the `cpuinfo` package."
-            + " Resorting to a default power consumption of 85W."
+            + " Resorting to a default power consumption."
         )
         return "Unknown", None
 

codecarbon/core/emissions.py

@@ -90,7 +90,7 @@ def get_cloud_country_iso_code(self, cloud: CloudMetadata) -> str:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find country ISO Code for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )
@@ -105,7 +105,7 @@ def get_cloud_geo_region(self, cloud: CloudMetadata) -> str:
         selected = df.loc[flags]
         if not len(selected):
             raise ValueError(
-                "Unable to find country name for "
+                "Unable to find State/City name for "
                 f"cloud_provider={cloud.provider}, "
                 f"cloud_region={cloud.region}"
             )