Update client metric API

Author: dmelikyan

README.md

@@ -6,10 +6,10 @@
 
 Graphsignal is an inference observability platform that helps developers accelerate and troubleshoot AI systems. With Graphsignal, developers can:
 
-* Trace LLM generations, communication, kernel launches, and more.
-* Identify top contributors to inference latency.
-* Monitor inference performance, CPU/GPU utilization, and errors.
-* Track and get alerts on errors and inefficiencies.
+* Trace and profile LLM generations, communication, CUDA kernels, batching, and more.
+* Monitor inference performance, CPU/GPU utilization, memory usage, and server metrics.
+* Track and get alerst on errors and exceptions - with contextual data, stack traces, and triggering conditions.
+* Compare performance across models, versions, hardware setups, and optimization configurations.
 
 
 [![Dashboards](https://graphsignal.com/external/screenshot-dashboard.png)](https://graphsignal.com/)

graphsignal/__init__.py

@@ -25,7 +25,7 @@ def configure(
     deployment: Optional[str] = None,
     tags: Optional[Dict[str, str]] = None,
     auto_instrument: Optional[bool] = None,
-    profiles_per_min: Optional[int] = None,
+    samples_per_min: Optional[int] = None,
     include_profiles: Optional[list] = None,
     debug_mode: Optional[bool] = None
 ) -> None:
@@ -39,7 +39,7 @@ def configure(
     api_url = read_config_param("api_url", str, api_url)
     tags = read_config_tags(tags)
     auto_instrument = read_config_param("auto_instrument", bool, auto_instrument, default_value=True)
-    profiles_per_min = read_config_param("profiles_per_min", int, profiles_per_min)
+    samples_per_min = read_config_param("samples_per_min", int, samples_per_min)
     include_profiles = read_config_param("include_profiles", list, include_profiles)
     debug_mode = read_config_param("debug_mode", bool, debug_mode, default_value=False)
 
@@ -52,7 +52,7 @@ def configure(
         api_url=api_url,
         tags=tags,
         auto_instrument=auto_instrument,
-        profiles_per_min=profiles_per_min,
+        samples_per_min=samples_per_min,
         include_profiles=include_profiles,
         debug_mode=debug_mode)
     _tracer.setup()

graphsignal/client/__init__.py

@@ -41,9 +41,9 @@
 from graphsignal.client.models.metric_type import MetricType
 from graphsignal.client.models.param import Param
 from graphsignal.client.models.profile import Profile
-from graphsignal.client.models.rate import Rate
 from graphsignal.client.models.span import Span
 from graphsignal.client.models.span_query_result import SpanQueryResult
+from graphsignal.client.models.summary import Summary
 from graphsignal.client.models.tag import Tag
 from graphsignal.client.models.validation_error import ValidationError
 from graphsignal.client.models.validation_error_loc_inner import ValidationErrorLocInner

graphsignal/client/models/__init__.py

@@ -24,9 +24,9 @@
 from graphsignal.client.models.metric_type import MetricType
 from graphsignal.client.models.param import Param
 from graphsignal.client.models.profile import Profile
-from graphsignal.client.models.rate import Rate
 from graphsignal.client.models.span import Span
 from graphsignal.client.models.span_query_result import SpanQueryResult
+from graphsignal.client.models.summary import Summary
 from graphsignal.client.models.tag import Tag
 from graphsignal.client.models.validation_error import ValidationError
 from graphsignal.client.models.validation_error_loc_inner import ValidationErrorLocInner

graphsignal/client/models/metric.py

@@ -21,7 +21,7 @@
 from typing import Any, ClassVar, Dict, List, Optional, Union
 from graphsignal.client.models.histogram import Histogram
 from graphsignal.client.models.metric_type import MetricType
-from graphsignal.client.models.rate import Rate
+from graphsignal.client.models.summary import Summary
 from graphsignal.client.models.tag import Tag
 from typing import Optional, Set
 from typing_extensions import Self
@@ -37,11 +37,11 @@ class Metric(BaseModel):
     is_time: Optional[StrictBool] = Field(default=False, description="Indicates if the metric is in nanoseconds.")
     is_size: Optional[StrictBool] = Field(default=False, description="Indicates if the metric is in bytes.")
     gauge: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="The value for gauge type metrics.")
-    counter: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="The value for counter type metrics.")
-    rate: Optional[Rate] = Field(default=None, description="The value for rate type metrics.")
+    total: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="The value for counter type metrics.")
+    summary: Optional[Summary] = Field(default=None, description="The value for summary type metrics.")
     histogram: Optional[Histogram] = Field(default=None, description="The histogram data for histogram type metrics.")
     update_ts: StrictInt = Field(description="Unix timestamp (seconds) when the metric was last updated.")
-    __properties: ClassVar[List[str]] = ["name", "tags", "type", "unit", "is_time", "is_size", "gauge", "counter", "rate", "histogram", "update_ts"]
+    __properties: ClassVar[List[str]] = ["name", "tags", "type", "unit", "is_time", "is_size", "gauge", "total", "summary", "histogram", "update_ts"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -89,9 +89,9 @@ def to_dict(self) -> Dict[str, Any]:
                 if _item_tags:
                     _items.append(_item_tags.to_dict())
             _dict['tags'] = _items
-        # override the default output from pydantic by calling `to_dict()` of rate
-        if self.rate:
-            _dict['rate'] = self.rate.to_dict()
+        # override the default output from pydantic by calling `to_dict()` of summary
+        if self.summary:
+            _dict['summary'] = self.summary.to_dict()
         # override the default output from pydantic by calling `to_dict()` of histogram
         if self.histogram:
             _dict['histogram'] = self.histogram.to_dict()
@@ -114,8 +114,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
             "is_time": obj.get("is_time") if obj.get("is_time") is not None else False,
             "is_size": obj.get("is_size") if obj.get("is_size") is not None else False,
             "gauge": obj.get("gauge"),
-            "counter": obj.get("counter"),
-            "rate": Rate.from_dict(obj["rate"]) if obj.get("rate") is not None else None,
+            "total": obj.get("total"),
+            "summary": Summary.from_dict(obj["summary"]) if obj.get("summary") is not None else None,
             "histogram": Histogram.from_dict(obj["histogram"]) if obj.get("histogram") is not None else None,
             "update_ts": obj.get("update_ts")
         })

graphsignal/client/models/metric_type.py

@@ -28,7 +28,7 @@ class MetricType(str, Enum):
     """
     GAUGE = 'gauge'
     COUNTER = 'counter'
-    RATE = 'rate'
+    SUMMARY = 'summary'
     HISTOGRAM = 'histogram'
 
     @classmethod

graphsignal/client/models/rate.py graphsignal/client/models/summary.pygraphsignal/client/models/rate.py renamed to graphsignal/client/models/summary.py

@@ -18,17 +18,18 @@
 import json
 
 from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt
-from typing import Any, ClassVar, Dict, List, Union
+from typing import Any, ClassVar, Dict, List, Optional, Union
 from typing import Optional, Set
 from typing_extensions import Self
 
-class Rate(BaseModel):
+class Summary(BaseModel):
     """
-    Rate
+    Summary
     """ # noqa: E501
-    count: Union[StrictFloat, StrictInt] = Field(description="The count value.")
-    interval: Union[StrictFloat, StrictInt] = Field(description="The the interval value.")
-    __properties: ClassVar[List[str]] = ["count", "interval"]
+    count: StrictInt = Field(description="The count value.")
+    sum: Union[StrictFloat, StrictInt] = Field(description="The sum value.")
+    sum2: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="The sum of squares value.")
+    __properties: ClassVar[List[str]] = ["count", "sum", "sum2"]
 
     model_config = ConfigDict(
         populate_by_name=True,
@@ -48,7 +49,7 @@ def to_json(self) -> str:
 
     @classmethod
     def from_json(cls, json_str: str) -> Optional[Self]:
-        """Create an instance of Rate from a JSON string"""
+        """Create an instance of Summary from a JSON string"""
         return cls.from_dict(json.loads(json_str))
 
     def to_dict(self) -> Dict[str, Any]:
@@ -73,7 +74,7 @@ def to_dict(self) -> Dict[str, Any]:
 
     @classmethod
     def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
-        """Create an instance of Rate from a dict"""
+        """Create an instance of Summary from a dict"""
         if obj is None:
             return None
 
@@ -82,7 +83,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
 
         _obj = cls.model_validate({
             "count": obj.get("count"),
-            "interval": obj.get("interval")
+            "sum": obj.get("sum"),
+            "sum2": obj.get("sum2")
         })
         return _obj
 

graphsignal/metrics.py

@@ -68,40 +68,42 @@ class CounterMetric(BaseMetric):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.type = 'counter'
-        self.counter = 0
+        self.total = 0
 
     def update(self, value, update_ts):
         with self._update_lock:
             self.touch()
-            self.counter += value
+            self.total += value
             self.update_ts = update_ts
 
     def export(self):
         with self._update_lock:
             model = super().export()
-            model.counter = self.counter
-            self.counter = 0
+            model.total = self.total
+            self.total = 0
             return model
 
 
-class RateMetric(BaseMetric):
+class SummaryMetric(BaseMetric):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.type = 'rate'
+        self.type = 'summary'
         self.count = 0
-        self.interval = 0
+        self.sum = 0
+        self.sum2 = 0
 
-    def update(self, count, interval, update_ts):
+    def update(self, count, sum_val, sum2_val, update_ts):
         with self._update_lock:
             self.touch()
             self.count += count
-            self.interval += interval
+            self.sum += sum_val
+            self.sum2 += sum2_val
             self.update_ts = update_ts
 
     def export(self):
         with self._update_lock:
             model = super().export()
-            model.rate = client.Rate(count=self.count, interval=self.interval)
+            model.summary = client.Summary(count=self.count, sum=self.sum, sum2=self.sum2)
             return model
 
 
@@ -171,21 +173,23 @@ def inc_counter(self, name, tags, value, update_ts, unit=None):
         metric.update(value, update_ts)
         return metric
 
-    def update_rate(self, name, tags, count, interval, update_ts, unit=None):
+    def update_summary(self, name, tags, count, sum_val, sum2_val, update_ts, unit=None):
         if name is None:
-            raise ValueError('Rate name cannot be None')
+            raise ValueError('Summary name cannot be None')
         if count is None:
-            raise ValueError('Rate count cannot be None')
-        if interval is None:
-            raise ValueError('Rate interval cannot be None')
+            raise ValueError('Summary count cannot be None')
+        if sum_val is None:
+            raise ValueError('Summary sum cannot be None')
+        if sum2_val is None:
+            raise ValueError('Summary sum2 cannot be None')
 
         key = self.metric_key(name, tags)
         with self._update_lock:
             if key not in self._metrics:
-                metric = self._metrics[key] = RateMetric(name, tags, unit=unit)
+                metric = self._metrics[key] = SummaryMetric(name, tags, unit=unit)
             else:
                 metric = self._metrics[key]
-        metric.update(count, interval, update_ts)
+        metric.update(count, sum_val, sum2_val, update_ts)
         return metric
 
     def update_histogram(self, name, tags, value, update_ts, unit=None, is_time=False, is_size=False):

graphsignal/recorders/nvml_recorder.py

@@ -487,7 +487,7 @@ def on_metric_update(self):
                     name='gpu.errors.xid', tags=metric_tags, 
                     value=num_xid_errors, update_ts=now)
                 for xid_error_code in device_usage.last_xid_error_codes:
-                    graphsignal._tracer().report_error(
+                    graphsignal._tracer.report_error(
                         name='gpu.errors.xid',
                         tags=metric_tags,
                         level='error',

graphsignal/recorders/otel_adapter.py

@@ -0,0 +1,59 @@
+import os
+import logging
+from opentelemetry import trace
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+from graphsignal.recorders.base_recorder import BaseRecorder
+
+logger = logging.getLogger('graphsignal')
+
+
+class LocalSpanExporter(SpanExporter):
+    def export(self, spans):
+        try:
+            self._export_callback(spans)
+        except Exception as e:
+            logger.error(f"Error during OpenTelemetry export: {e}", exc_info=True)
+        return SpanExporter.ResultCode.SUCCESS
+
+
+class OTELAdapter():
+    def __init__(self, service_name='service'):
+        self._service_name = service_name
+        self._provider = None
+        self._exporter = None
+        self._local_exporter = None
+        self._export_callback = None
+
+    def setup(self, export_callback):
+        self._export_callback = export_callback
+        try:
+            provider = trace.get_tracer_provider()
+            if isinstance(provider, TracerProvider):
+                self._local_exporter = LocalSpanExporter()
+                provider.add_span_processor(BatchSpanProcessor(self._local_exporter))
+            else:                            
+                service_name = os.environ.get("OTEL_SERVICE_NAME", self._service_name)
+                resource = Resource.create({"service.name": service_name})
+                self._provider = TracerProvider(resource=resource)
+                self._exporter = OTLPSpanExporter()
+                self._local_exporter = LocalSpanExporter()
+                self._provider.add_span_processor(BatchSpanProcessor(self._exporter))
+                self._provider.add_span_processor(BatchSpanProcessor(self._local_exporter))
+                trace.set_tracer_provider(self._provider)
+
+            logger.info(f"OpenTelemetry tracer provider configured for service: {service_name}")
+            
+        except Exception as e:
+            logger.error(f"Failed to set up OpenTelemetry tracer provider: {e}", exc_info=True)
+
+    def shutdown(self):
+        try:
+            if self._provider:
+                self._provider.force_flush()
+                logger.debug("OpenTelemetry tracer provider flushed")
+        except Exception as e:
+            logger.error(f"Error during OpenTelemetry shutdown: {e}", exc_info=True)