From 11b951582624a8e71cb68c90546258cb7590f0d3 Mon Sep 17 00:00:00 2001 From: Dominik Baack Date: Mon, 15 Dec 2025 09:53:01 +0100 Subject: [PATCH 1/5] Added total energy to prometheus Added auto cleanup after shutdown from push gateway Renamed attributes --- codecarbon/emissions_tracker.py | 11 +++++- codecarbon/output_methods/base_output.py | 3 ++ .../output_methods/metrics/metric_docs.py | 13 +++++-- .../output_methods/metrics/prometheus.py | 38 +++++++++++++++++-- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 2bb73cbd2..f4b3c29b6 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -447,7 +447,12 @@ def _init_output_methods(self, *, api_key: str = None): self.run_id = uuid.uuid4() if self._save_to_prometheus: - self._output_handlers.append(PrometheusOutput(self._prometheus_url)) + self._output_handlers.append( + PrometheusOutput( + self._prometheus_url, + jobname=self._project_name + "_" + self._experiment_name, + ) + ) if self._save_to_logfire: self._output_handlers.append(LogfireOutput()) @@ -686,6 +691,10 @@ def stop(self) -> Optional[float]: self.final_emissions_data = emissions_data self.final_emissions = emissions_data.emissions + + for handler in self._output_handlers: + handler.exit() + return emissions_data.emissions def _persist_data( diff --git a/codecarbon/output_methods/base_output.py b/codecarbon/output_methods/base_output.py index 4b152c29b..29ca38512 100644 --- a/codecarbon/output_methods/base_output.py +++ b/codecarbon/output_methods/base_output.py @@ -22,3 +22,6 @@ def live_out(self, total: EmissionsData, delta: EmissionsData): def task_out(self, data: List[TaskEmissionsData], experiment_name: str): pass + + def exit(self): + pass diff --git a/codecarbon/output_methods/metrics/metric_docs.py b/codecarbon/output_methods/metrics/metric_docs.py index 864641ee8..d62d48f71 100644 --- a/codecarbon/output_methods/metrics/metric_docs.py +++ b/codecarbon/output_methods/metrics/metric_docs.py @@ -50,17 +50,22 @@ class MetricDocumentation: ) cpu_energy_doc = MetricDocumentation( "codecarbon_cpu_energy", - description="Energy used per CPU (kWh)", + description="Energy used per CPU since last reading (kWh)", ) gpu_energy_doc = MetricDocumentation( "codecarbon_gpu_energy", - description="Energy used per GPU (kWh)", + description="Energy used per GPU since last reading (kWh)", ) ram_energy_doc = MetricDocumentation( "codecarbon_ram_energy", - description="Energy used per RAM (kWh)", + description="Energy used per RAM since last reading (kWh)", ) energy_consumed_doc = MetricDocumentation( "codecarbon_energy_consumed", - description="Sum of cpu_energy, gpu_energy and ram_energy (kW)", + description="Sum of cpu_energy, gpu_energy and ram_energy (kWh)", +) + +energy_consumed_total_doc = MetricDocumentation( + "codecarbon_energy_total", + description="Accumulated cpu_energy, gpu_energy and ram_energy (kWh)", ) diff --git a/codecarbon/output_methods/metrics/prometheus.py b/codecarbon/output_methods/metrics/prometheus.py index 24318c6b8..f1756eafc 100644 --- a/codecarbon/output_methods/metrics/prometheus.py +++ b/codecarbon/output_methods/metrics/prometheus.py @@ -1,7 +1,14 @@ import dataclasses import os -from prometheus_client import CollectorRegistry, Gauge, push_to_gateway +from prometheus_client import ( + CollectorRegistry, + Counter, + Gauge, + delete_from_gateway, + push_to_gateway, +) + from prometheus_client.exposition import basic_auth_handler from codecarbon.external.logger import logger @@ -15,6 +22,7 @@ emissions_doc, emissions_rate_doc, energy_consumed_doc, + energy_consumed_total_doc, gpu_energy_doc, gpu_power_doc, ram_energy_doc, @@ -58,6 +66,14 @@ def generate_gauge(metric_doc: MetricDocumentation): labelnames, registry=registry, ) + +def generate_counter(metric_doc: MetricDocumentation): + return Counter( + metric_doc.name, + metric_doc.description, + labelnames, + registry=registry, + ) duration_gauge = generate_gauge(duration_doc) @@ -70,6 +86,7 @@ def generate_gauge(metric_doc: MetricDocumentation): gpu_energy_gauge = generate_gauge(gpu_energy_doc) ram_energy_gauge = generate_gauge(ram_energy_doc) energy_consumed_gauge = generate_gauge(energy_consumed_doc) +energy_consumed_total = generate_counter(energy_consumed_total_doc) class PrometheusOutput(BaseOutput): @@ -77,8 +94,18 @@ class PrometheusOutput(BaseOutput): Send emissions data to prometheus pushgateway """ - def __init__(self, prometheus_url: str): + def __init__(self, prometheus_url: str, jobname: str = "codecarbon"): self.prometheus_url = prometheus_url + self.jobname = jobname + + def exit(self): + # Cleanup metrics from pushgateway on shutdown, prometheus should already have read them + # Otherwise they will persist with their last values + try: + logger.info("Deleting metrics from Prometheus Pushgateway") + delete_from_gateway(self.prometheus_url, job=self.jobname) + except Exception as e: + logger.error(e, exc_info=True) def out(self, total: EmissionsData, delta: EmissionsData): try: @@ -120,11 +147,16 @@ def add_emission(self, carbon_emission: dict): (energy_consumed_gauge, "energy_consumed"), ]: gauge.labels(**labels).set(carbon_emission[emission_name]) + + + # Update the total energy consumed counter + # This is separate from the total values given to self.out(...) + energy_consumed_total.labels(**labels).inc(carbon_emission["energy_consumed"]) # Send the new metric values push_to_gateway( self.prometheus_url, - job="codecarbon", + job=self.jobname, registry=registry, handler=self._auth_handler, ) From e8419c3eb3e71e15a3e22ae6b81b80e03c0ce0dc Mon Sep 17 00:00:00 2001 From: Dominik Baack Date: Mon, 15 Dec 2025 12:14:43 +0100 Subject: [PATCH 2/5] Run Precommit hooks --- codecarbon/emissions_tracker.py | 4 ++-- codecarbon/output_methods/base_output.py | 2 +- codecarbon/output_methods/metrics/prometheus.py | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index f4b3c29b6..b9570198c 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -691,10 +691,10 @@ def stop(self) -> Optional[float]: self.final_emissions_data = emissions_data self.final_emissions = emissions_data.emissions - + for handler in self._output_handlers: handler.exit() - + return emissions_data.emissions def _persist_data( diff --git a/codecarbon/output_methods/base_output.py b/codecarbon/output_methods/base_output.py index 29ca38512..ff6ea1778 100644 --- a/codecarbon/output_methods/base_output.py +++ b/codecarbon/output_methods/base_output.py @@ -22,6 +22,6 @@ def live_out(self, total: EmissionsData, delta: EmissionsData): def task_out(self, data: List[TaskEmissionsData], experiment_name: str): pass - + def exit(self): pass diff --git a/codecarbon/output_methods/metrics/prometheus.py b/codecarbon/output_methods/metrics/prometheus.py index f1756eafc..c8bc3e0a1 100644 --- a/codecarbon/output_methods/metrics/prometheus.py +++ b/codecarbon/output_methods/metrics/prometheus.py @@ -8,7 +8,6 @@ delete_from_gateway, push_to_gateway, ) - from prometheus_client.exposition import basic_auth_handler from codecarbon.external.logger import logger @@ -66,7 +65,8 @@ def generate_gauge(metric_doc: MetricDocumentation): labelnames, registry=registry, ) - + + def generate_counter(metric_doc: MetricDocumentation): return Counter( metric_doc.name, @@ -97,7 +97,7 @@ class PrometheusOutput(BaseOutput): def __init__(self, prometheus_url: str, jobname: str = "codecarbon"): self.prometheus_url = prometheus_url self.jobname = jobname - + def exit(self): # Cleanup metrics from pushgateway on shutdown, prometheus should already have read them # Otherwise they will persist with their last values @@ -147,8 +147,7 @@ def add_emission(self, carbon_emission: dict): (energy_consumed_gauge, "energy_consumed"), ]: gauge.labels(**labels).set(carbon_emission[emission_name]) - - + # Update the total energy consumed counter # This is separate from the total values given to self.out(...) energy_consumed_total.labels(**labels).inc(carbon_emission["energy_consumed"]) From efff41a3d6ba8131d3e5c190f295f5d7af32a172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Courty?= <6603048+benoit-cty@users.noreply.github.com> Date: Sun, 28 Dec 2025 11:58:52 +0100 Subject: [PATCH 3/5] Apply suggestion from @cianc Co-authored-by: cianc --- codecarbon/output_methods/metrics/metric_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecarbon/output_methods/metrics/metric_docs.py b/codecarbon/output_methods/metrics/metric_docs.py index d62d48f71..83ba0e31e 100644 --- a/codecarbon/output_methods/metrics/metric_docs.py +++ b/codecarbon/output_methods/metrics/metric_docs.py @@ -67,5 +67,5 @@ class MetricDocumentation: energy_consumed_total_doc = MetricDocumentation( "codecarbon_energy_total", - description="Accumulated cpu_energy, gpu_energy and ram_energy (kWh)", + description="Accumulated cpu_energy, gpu_energy and ram_energy (kWh) since the start of the run", ) From 8d19e19017bd62bd802f11920d114c337e768297 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Sun, 28 Dec 2025 12:10:29 +0100 Subject: [PATCH 4/5] rename variable and sanitize --- codecarbon/emissions_tracker.py | 7 ++++++- codecarbon/output_methods/metrics/prometheus.py | 8 ++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index b9570198c..2ba48ff81 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -6,6 +6,7 @@ import dataclasses import os import platform +import re import time import uuid from abc import ABC, abstractmethod @@ -450,7 +451,11 @@ def _init_output_methods(self, *, api_key: str = None): self._output_handlers.append( PrometheusOutput( self._prometheus_url, - jobname=self._project_name + "_" + self._experiment_name, + job_name=re.sub( + r"[^a-zA-Z0-9_-]", + "_", + f"{self._project_name}_{self._experiment_name}", + ), ) ) diff --git a/codecarbon/output_methods/metrics/prometheus.py b/codecarbon/output_methods/metrics/prometheus.py index c8bc3e0a1..0cbd3e046 100644 --- a/codecarbon/output_methods/metrics/prometheus.py +++ b/codecarbon/output_methods/metrics/prometheus.py @@ -94,16 +94,16 @@ class PrometheusOutput(BaseOutput): Send emissions data to prometheus pushgateway """ - def __init__(self, prometheus_url: str, jobname: str = "codecarbon"): + def __init__(self, prometheus_url: str, job_name: str = "codecarbon"): self.prometheus_url = prometheus_url - self.jobname = jobname + self.job_name = job_name def exit(self): # Cleanup metrics from pushgateway on shutdown, prometheus should already have read them # Otherwise they will persist with their last values try: logger.info("Deleting metrics from Prometheus Pushgateway") - delete_from_gateway(self.prometheus_url, job=self.jobname) + delete_from_gateway(self.prometheus_url, job=self.job_name) except Exception as e: logger.error(e, exc_info=True) @@ -155,7 +155,7 @@ def add_emission(self, carbon_emission: dict): # Send the new metric values push_to_gateway( self.prometheus_url, - job=self.jobname, + job=self.job_name, registry=registry, handler=self._auth_handler, ) From 698a6fd980cbafa06de4ab86b82332591eb54eb5 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Sun, 28 Dec 2025 12:10:40 +0100 Subject: [PATCH 5/5] Add test --- tests/output_methods/metrics/test_prometheus.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/output_methods/metrics/test_prometheus.py b/tests/output_methods/metrics/test_prometheus.py index 5802bcaa4..5ee4bc74d 100644 --- a/tests/output_methods/metrics/test_prometheus.py +++ b/tests/output_methods/metrics/test_prometheus.py @@ -51,6 +51,12 @@ def test_out_method(self, mock_push_to_gateway): output = prometheus.PrometheusOutput("url") output.out(total=EMISSIONS_DATA, delta=EMISSIONS_DATA) + @patch("codecarbon.output_methods.metrics.prometheus.delete_from_gateway") + def test_exit_method(self, mock_delete): + output = prometheus.PrometheusOutput("url", job_name="custom_job") + output.exit() + mock_delete.assert_called_once_with("url", job="custom_job") + @patch( "codecarbon.output_methods.metrics.prometheus.push_to_gateway", side_effect=Exception("Test error"),