Merge pull request #471 from fvaleye/feature/add-gcp-and-azure-cloud-tracking

feat(cloud): add GCP and Azure carbon emission tracking using new rerentials

Author: fvaleye

README.md

@@ -37,8 +37,8 @@ pip install 'tracarbon[datadog,prometheus,kubernetes]'
 | **Cloud Provider** |                                                                                              **Description**                                                                                              |
 | ------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |
 | AWS                | ✅ Use the hardware's usage with the EC2 instances carbon emissions datasets of [cloud-carbon-coefficients](https://github.com/cloud-carbon-footprint/ccf-coefficients/blob/main/data/aws-instances.csv). |
-| GCP                |                                                                                          ❌ Not yet implemented.                                                                                          |
-| Azure              |                                                                                          ❌ Not yet implemented.                                                                                          |
+| GCP                | ✅ Use the hardware's usage with the GCP instances carbon emissions datasets of [cloud-carbon-coefficients](https://github.com/cloud-carbon-footprint/ccf-coefficients/blob/main/data/gcp-instances.csv). |
+| Azure              | ✅ Use the hardware's usage with the Azure instances carbon emissions datasets of [cloud-carbon-coefficients](https://github.com/cloud-carbon-footprint/ccf-coefficients/blob/main/data/azure-instances.csv). |
 
 ### 🎮 GPU: power tracking
 
@@ -65,6 +65,8 @@ pip install 'tracarbon[datadog,prometheus,kubernetes]'
 | Worldwide    | Get the latest co2g/kwh in near real-time using the CO2Signal or ElectricityMaps APIs. See [here](http://api.electricitymap.org/v3/zones) for the list of available zones. | [CO2Signal API](https://www.co2signal.com) or [ElectricityMaps](https://static.electricitymaps.com/api/docs/index.html)                                       |
 | Europe       |                                 Static file created from the European Environment Agency Emission for the co2g/kwh in European countries.                                  | [EEA website](https://www.eea.europa.eu/en/analysis/maps-and-charts/co2-emission-intensity-15) |
 | AWS          |                                                               Static file of the AWS Grid emissions factors.                                                               | [cloud-carbon-coefficients](https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/blob/main/data/grid-emissions-factors-aws.csv)                |
+| GCP          |                                                               Static file of the GCP Grid emissions factors (2024 yearly data).                                                               | [GoogleCloudPlatform/region-carbon-info](https://github.com/GoogleCloudPlatform/region-carbon-info/blob/main/data/yearly/2024.csv)                |
+| Azure        |                                                               Static file of the Azure Grid emissions factors.                                                             | [cloud-carbon-coefficients](https://github.com/cloud-carbon-footprint/cloud-carbon-coefficients/blob/main/data/grid-emissions-factors-azure.csv)              |
 
 ### ⚙️ Configuration
 

scripts/check_data.py

@@ -1,4 +1,5 @@
 import urllib.request
+from urllib.error import URLError
 from urllib.parse import urlparse
 
 
@@ -17,6 +18,25 @@ def check_content_length(url: str, expected_content_length: str) -> bool:
     return True
 
 
+def check_new_year_available_for_gcp(base_url: str, current_year: int) -> None:
+    if not is_valid_url(base_url):
+        raise ValueError(f"Invalid or unsafe URL scheme for URL: {base_url}")
+
+    next_year = current_year + 1
+    next_year_url = base_url.replace(str(current_year), str(next_year))
+
+    try:
+        response = urllib.request.urlopen(next_year_url, timeout=10)  # noqa: S310
+        if response.getcode() == 200:
+            raise ValueError(
+                f"New year {next_year} data is available at {next_year_url}. "
+                f"Please update the data file and code to use the new year."
+            )
+    except URLError:
+        # URL doesn't exist, which is expected if no new year is available
+        pass
+
+
 if __name__ == "__main__":
     urls = [
         {
@@ -31,6 +51,13 @@ def check_content_length(url: str, expected_content_length: str) -> bool:
             "url": "https://raw.githubusercontent.com/cloud-carbon-footprint/ccf-coefficients/main/data/grid-emissions-factors-aws.csv",
             "content_length": "1204",
         },
+        {
+            "url": "https://raw.githubusercontent.com/GoogleCloudPlatform/region-carbon-info/main/data/yearly/2024.csv",
+            "content_length": "1621",
+        },
     ]
     for url in urls:
         check_content_length(url=url["url"], expected_content_length=url["content_length"])
+
+    gcp_base_url = "https://raw.githubusercontent.com/GoogleCloudPlatform/region-carbon-info/main/data/yearly/2024.csv"
+    check_new_year_available_for_gcp(gcp_base_url, current_year=2024)

tests/hardwares/test_sensors.py

@@ -7,10 +7,16 @@
 from tracarbon import EnergyConsumption
 from tracarbon import LinuxEnergyConsumption
 from tracarbon import TracarbonException
+from tracarbon.exceptions import AzureSensorException
+from tracarbon.exceptions import GCPSensorException
 from tracarbon.hardwares import EnergyUsage
 from tracarbon.hardwares import HardwareInfo
 from tracarbon.hardwares import WindowsEnergyConsumption
 from tracarbon.hardwares.cloud_providers import AWS
+from tracarbon.hardwares.cloud_providers import GCP
+from tracarbon.hardwares.cloud_providers import Azure
+from tracarbon.hardwares.sensors import AzureEnergyConsumption
+from tracarbon.hardwares.sensors import GCPEnergyConsumption
 
 
 @pytest.mark.darwin
@@ -151,3 +157,111 @@ async def test_get_platform_should_return_the_platform_energy_consumption_window
     with pytest.raises(TracarbonException) as exception:
         await WindowsEnergyConsumption().get_energy_usage()
         assert exception.value.args[0] == "This Windows hardware is not yet supported."
+
+
+def test_is_gcp_should_return_false_on_exception():
+    assert GCP.is_gcp() is False
+
+
+def test_is_gcp_should_return_true(mocker):
+    mock_response = mocker.Mock()
+    mock_response.status_code = 200
+    mocker.patch.object(requests, "get", return_value=mock_response)
+
+    assert GCP.is_gcp() is True
+
+
+def test_gcp_from_metadata(mocker):
+    mock_machine_response = mocker.Mock()
+    mock_machine_response.text = "projects/123456/machineTypes/n2-standard-4"
+
+    mock_zone_response = mocker.Mock()
+    mock_zone_response.text = "projects/123456/zones/us-central1-a"
+
+    mocker.patch.object(
+        requests,
+        "get",
+        side_effect=[mock_machine_response, mock_zone_response],
+    )
+
+    gcp = GCP.from_metadata()
+
+    assert gcp.instance_type == "n2-standard-4"
+    assert gcp.region_name == "us-central1"
+
+
+@pytest.mark.asyncio
+async def test_gcp_sensor_should_return_energy_consumption(mocker):
+    gcp_sensor = GCPEnergyConsumption(instance_type="n2-standard-4")
+
+    assert gcp_sensor.vcpus == 4.0
+    assert gcp_sensor.memory_gb == 16.0
+    assert gcp_sensor.min_watts > 0
+    assert gcp_sensor.max_watts > gcp_sensor.min_watts
+
+    mocker.patch.object(HardwareInfo, "get_cpu_usage", return_value=50)
+    from tracarbon.hardwares.gpu import GPUInfo
+
+    mocker.patch.object(GPUInfo, "get_gpu_power_usage_or_none", return_value=None)
+
+    energy_usage = await gcp_sensor.get_energy_usage()
+
+    expected_power = gcp_sensor.min_watts + (gcp_sensor.max_watts - gcp_sensor.min_watts) * 0.5
+    assert abs(energy_usage.host_energy_usage - expected_power) < 0.01
+
+
+def test_gcp_sensor_should_return_error_when_instance_type_is_missing():
+    instance_type = "unknown-instance-type"
+
+    with pytest.raises(GCPSensorException):
+        GCPEnergyConsumption(instance_type=instance_type)
+
+
+def test_is_azure_should_return_false_on_exception():
+    assert Azure.is_azure() is False
+
+
+def test_is_azure_should_return_true(mocker):
+    mock_response = mocker.Mock()
+    mock_response.status_code = 200
+    mocker.patch.object(requests, "get", return_value=mock_response)
+
+    assert Azure.is_azure() is True
+
+
+def test_azure_from_metadata(mocker):
+    mock_response = mocker.Mock()
+    mock_response.json.return_value = {"compute": {"vmSize": "Standard_D2s_v3", "location": "eastus"}}
+    mocker.patch.object(requests, "get", return_value=mock_response)
+
+    azure = Azure.from_metadata()
+
+    assert azure.instance_type == "Standard_D2s_v3"
+    assert azure.region_name == "eastus"
+
+
+@pytest.mark.asyncio
+async def test_azure_sensor_should_return_energy_consumption(mocker):
+    azure_sensor = AzureEnergyConsumption(instance_type="D2 v3")
+
+    assert azure_sensor.vcpus == 2.0
+    assert azure_sensor.memory_gb == 8.0
+    assert azure_sensor.min_watts > 0
+    assert azure_sensor.max_watts > azure_sensor.min_watts
+
+    mocker.patch.object(HardwareInfo, "get_cpu_usage", return_value=50)
+    from tracarbon.hardwares.gpu import GPUInfo
+
+    mocker.patch.object(GPUInfo, "get_gpu_power_usage_or_none", return_value=None)
+
+    energy_usage = await azure_sensor.get_energy_usage()
+
+    expected_power = azure_sensor.min_watts + (azure_sensor.max_watts - azure_sensor.min_watts) * 0.5
+    assert abs(energy_usage.host_energy_usage - expected_power) < 0.01
+
+
+def test_azure_sensor_should_return_error_when_instance_type_is_missing():
+    instance_type = "unknown-instance-type"
+
+    with pytest.raises(AzureSensorException):
+        AzureEnergyConsumption(instance_type=instance_type)

tests/locations/test_location.py

@@ -6,6 +6,8 @@
 from tracarbon.locations import AWSLocation
 from tracarbon.locations import Country
 from tracarbon.locations import Location
+from tracarbon.locations.country import AzureLocation
+from tracarbon.locations.country import GCPLocation
 
 
 @pytest.mark.asyncio
@@ -88,3 +90,61 @@ def test_aws_location_should_return_ok_if_region_exists():
     assert location.name == "AWS(eu-west-1)"
     assert location.co2g_kwh == 316.0
     assert location.co2g_kwh_source.value == "file"
+
+
+def test_gcp_location_should_return_an_error_if_region_not_exists():
+    region_name = "unknown-region"
+
+    with pytest.raises(CloudProviderRegionIsMissing) as exception:
+        GCPLocation(region_name=region_name)
+    assert exception.value.args[0] == f"The region [{region_name}] is not in the GCP grid emissions factors file."
+
+
+def test_gcp_location_should_return_ok_if_region_exists():
+    region_name = "europe-west1"
+
+    location = GCPLocation(region_name=region_name)
+
+    assert location.name == "GCP(europe-west1)"
+    assert location.co2g_kwh > 100
+    assert location.co2g_kwh < 110
+    assert location.co2g_kwh_source.value == "file"
+
+
+def test_gcp_location_us_central1():
+    region_name = "us-central1"
+
+    location = GCPLocation(region_name=region_name)
+
+    assert location.name == "GCP(us-central1)"
+    assert location.co2g_kwh > 400
+    assert location.co2g_kwh < 420
+
+
+def test_azure_location_should_return_an_error_if_region_not_exists():
+    region_name = "unknown-region"
+
+    with pytest.raises(CloudProviderRegionIsMissing) as exception:
+        AzureLocation(region_name=region_name)
+    assert exception.value.args[0] == f"The region [{region_name}] is not in the Azure grid emissions factors file."
+
+
+def test_azure_location_should_return_ok_if_region_exists():
+    region_name = "West Europe"
+
+    location = AzureLocation(region_name=region_name)
+
+    assert location.name == "Azure(West Europe)"
+    assert location.co2g_kwh > 380
+    assert location.co2g_kwh < 400
+    assert location.co2g_kwh_source.value == "file"
+
+
+def test_azure_location_east_us():
+    region_name = "East US"
+
+    location = AzureLocation(region_name=region_name)
+
+    assert location.name == "Azure(East US)"
+    assert location.co2g_kwh > 410
+    assert location.co2g_kwh < 420

tracarbon/exceptions.py

@@ -1,3 +1,16 @@
+__all__ = [
+    "TracarbonException",
+    "CountryIsMissing",
+    "CloudProviderRegionIsMissing",
+    "AWSSensorException",
+    "GCPSensorException",
+    "AzureSensorException",
+    "HardwareRAPLException",
+    "HardwareNoGPUDetectedException",
+    "CO2SignalAPIKeyIsMissing",
+]
+
+
 class TracarbonException(Exception):
     """General Tracarbon Exception."""
 
@@ -22,6 +35,18 @@ class AWSSensorException(TracarbonException):
     pass
 
 
+class GCPSensorException(TracarbonException):
+    """Error in the GCP Sensor."""
+
+    pass
+
+
+class AzureSensorException(TracarbonException):
+    """Error in the Azure Sensor."""
+
+    pass
+
+
 class HardwareRAPLException(TracarbonException):
     """The hardware is not compatible with RAPL."""
 

tracarbon/hardwares/__init__.py

@@ -1,4 +1,5 @@
 from tracarbon.hardwares.amd_rapl import *
+from tracarbon.hardwares.cloud_providers import *
 from tracarbon.hardwares.containers import *
 from tracarbon.hardwares.energy import *
 from tracarbon.hardwares.hardware import *

tracarbon/hardwares/amd_rapl.py

@@ -15,6 +15,11 @@
 from tracarbon.hardwares.energy import EnergyUsage
 from tracarbon.hardwares.energy import Power
 
+__all__ = [
+    "AMDRAPLResult",
+    "AMDRAPL",
+]
+
 
 class AMDRAPLResult(BaseModel):
     """