From 9829c33378345c2cebe000ef36a1f2b86c95e148 Mon Sep 17 00:00:00 2001 From: CasperTeirlinck Date: Wed, 29 Apr 2026 16:42:50 +0200 Subject: [PATCH] Added measurements wrapper --- .../src/checkup_conveyor/conveyor_metric.py | 14 +- .../src/checkup_dbt/metrics/base.py | 11 +- .../metrics/quality/flagged_packages.py | 6 +- .../metrics/quality/naming_convention.py | 6 +- .../metrics/quality/profile_host.py | 6 +- .../metrics/quality/supported_version.py | 9 +- .../checkup_dbt/metrics/quality/version.py | 6 +- .../metrics/test/column_test_coverage.py | 11 +- .../metrics/test/tested_columns.py | 6 +- .../checkup-git/src/checkup_git/metrics.py | 11 +- .../src/checkup_python/metrics/version.py | 7 +- .../checkup_python/metrics/version_check.py | 9 +- src/checkup/__init__.py | 7 +- src/checkup/executor/batch_executors.py | 17 +- src/checkup/executor/metric_calculator.py | 9 +- src/checkup/executor/state.py | 5 +- src/checkup/hub.py | 3 +- src/checkup/materializers/base.py | 2 +- src/checkup/materializers/console.py | 2 +- src/checkup/materializers/csv_file.py | 2 +- src/checkup/materializers/database.py | 2 +- src/checkup/materializers/html_report.py | 2 +- src/checkup/measurement.py | 96 ++++ src/checkup/metric.py | 68 +-- tests/fixtures.py | 115 ++--- tests/test_executors.py | 35 +- tests/test_graph.py | 470 ++++++++---------- tests/test_hub_execution.py | 33 +- tests/test_hub_validation.py | 7 +- tests/test_metric.py | 202 ++++---- 30 files changed, 547 insertions(+), 632 deletions(-) create mode 100644 src/checkup/measurement.py diff --git a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py index eecbf66..724a83f 100644 --- a/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py +++ b/plugins/checkup-conveyor/src/checkup_conveyor/conveyor_metric.py @@ -4,7 +4,7 @@ import requests from checkup import Context -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup_conveyor import ConveyorMetric logger = logging.getLogger(__name__) @@ -15,9 +15,7 @@ class ConveyorLastDeploymentTime(ConveyorMetric): description: ClassVar[str] = "Time of the last deployment in Conveyor" unit: ClassVar[str] = "timestamp" - def calculate( - self, context: Context, measurements: dict[type[Metric], Measurement] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: return self.measure(value=None) @@ -40,9 +38,7 @@ class ConveyorIsDirtyDeployment(ConveyorMetric): description: ClassVar[str] = "True if the last deployment was dirty" unit: ClassVar[str] = "boolean" - def calculate( - self, context: Context, measurements: dict[type[Metric], Measurement] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: return self.measure(value=None) @@ -66,9 +62,7 @@ class ConveyorLastRunStatus(ConveyorMetric): description: ClassVar[str] = "Status of the last run in Conveyor" unit: ClassVar[str] = "string" - def calculate( - self, context: Context, measurements: dict[type[Metric], Measurement] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: proj_id = self.get_conveyor_project_id(context) if proj_id is None: return self.measure(value=None) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py index 754ca8c..f46637a 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/base.py @@ -9,7 +9,8 @@ from dbt.artifacts.resources.types import NodeType from dbt.contracts.graph.manifest import Manifest -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider from checkup.types import Context from checkup_dbt.manifest_query import ManifestQuery @@ -87,9 +88,7 @@ class DbtCountMetric(DbtMetric): predicate: ClassVar[Callable[..., bool] | None] = None log_message: ClassVar[str] = "Found {value} items" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: cls = type(self) query = self.query(context).filter_by_type(cls.resource_type) @@ -128,9 +127,7 @@ class DbtDiagnosticMetric(DbtMetric): log_message: ClassVar[str] = "Found {value} items" max_diagnostic_items: ClassVar[int] = 50 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: cls = type(self) query = self.query(context).filter_by_type(cls.resource_type) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py index ccb2ee9..455d8bd 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/flagged_packages.py @@ -2,7 +2,7 @@ import yaml -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -31,9 +31,7 @@ class DbtFlaggedPackagesMetric(DbtMetric): flagged_packages: list[str] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: project_dir = self.get_project_dir(context) packages_path = project_dir / "packages.yml" diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py index 52902a0..44ca7fc 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/naming_convention.py @@ -2,7 +2,7 @@ from dbt.artifacts.resources.types import NodeType -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric, NamingConventionChecker @@ -28,9 +28,7 @@ class DbtModelsNotAdheringToNamingConventionMetric(DbtMetric): checker: NamingConventionChecker - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: manifest = self.get_manifest(context) non_adhering_models = [ diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py index 515a2d3..e470b65 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/profile_host.py @@ -2,7 +2,7 @@ import yaml -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -37,9 +37,7 @@ class DbtProfileHostMetric(DbtMetric): profile: str | None = None target: str - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: project_dir = self.get_project_dir(context) profiles_path = project_dir / "profiles.yml" diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py index f5465e9..6499d07 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/supported_version.py @@ -1,6 +1,7 @@ import logging -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.quality.version import DbtVersionMetric @@ -23,10 +24,8 @@ class DbtSupportedVersionMetric(DbtMetric): def depends_on(cls) -> list[type[Metric]]: return [DbtVersionMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - version: str = self.get_single(measurements, DbtVersionMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + version: str = measurements.get(DbtVersionMetric).value major_version = int(version.split(".")[0]) minor_version = int(version.split(".")[1]) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py index 6794092..eb7fdcb 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/quality/version.py @@ -2,7 +2,7 @@ import logging -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -18,9 +18,7 @@ class DbtVersionMetric(DbtMetric): description: str = "The dbt version used to generate the manifest" unit: str = "version" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: manifest = self.get_manifest(context) value = manifest.metadata.dbt_version return self.measure(value=value, diagnostic=f"dbt version: {value}") diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py index 0574c4c..2f1bd25 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/column_test_coverage.py @@ -1,6 +1,7 @@ import logging -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric from checkup_dbt.metrics.core.columns import DbtColumnsMetric @@ -25,11 +26,9 @@ class DbtColumnTestCoverageMetric(DbtMetric): def depends_on(cls) -> list[type[Metric]]: return [DbtTestedColumnsMetric, DbtColumnsMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - tested = self.get_single(measurements, DbtTestedColumnsMetric).value - total = self.get_single(measurements, DbtColumnsMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + tested = measurements.get(DbtTestedColumnsMetric).value + total = measurements.get(DbtColumnsMetric).value if total > 0: value = int(tested / total * 100) diff --git a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py index be20977..5463b8d 100644 --- a/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py +++ b/plugins/checkup-dbt/src/checkup_dbt/metrics/test/tested_columns.py @@ -2,7 +2,7 @@ from dbt.artifacts.resources.types import NodeType -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements from checkup.types import Context from checkup_dbt.metrics.base import DbtMetric @@ -20,9 +20,7 @@ class DbtTestedColumnsMetric(DbtMetric): description: str = "Number of columns with at least one test" unit: str = "columns" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: manifest = self.get_manifest(context) all_columns = { diff --git a/plugins/checkup-git/src/checkup_git/metrics.py b/plugins/checkup-git/src/checkup_git/metrics.py index b1216aa..8118a23 100644 --- a/plugins/checkup-git/src/checkup_git/metrics.py +++ b/plugins/checkup-git/src/checkup_git/metrics.py @@ -3,7 +3,8 @@ from datetime import UTC, datetime from fnmatch import fnmatch -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider from checkup.types import Context from checkup_git.provider import GitProvider @@ -28,9 +29,7 @@ class GitDaysSinceLastUpdateMetric(GitMetric): description: str = "Days since the last git commit" unit: str = "days" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: git_context = self.get_context(context) last_commit_date = git_context.get("git_last_commit_date") @@ -67,9 +66,7 @@ class GitTrackedFileCountMetric(GitMetric): pattern: str = "*" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: git_context = self.get_context(context) tracked_files = git_context.get("git_tracked_files", []) diff --git a/plugins/checkup-python/src/checkup_python/metrics/version.py b/plugins/checkup-python/src/checkup_python/metrics/version.py index 551364e..58e76aa 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version.py @@ -2,7 +2,8 @@ import sys from pathlib import Path -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.types import Context @@ -20,9 +21,7 @@ class PythonVersionMetric(Metric): description: str = "The Python version configured for the project" unit: str = "version" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: path = None if "path" in context: diff --git a/plugins/checkup-python/src/checkup_python/metrics/version_check.py b/plugins/checkup-python/src/checkup_python/metrics/version_check.py index 17f2960..23f80c5 100644 --- a/plugins/checkup-python/src/checkup_python/metrics/version_check.py +++ b/plugins/checkup-python/src/checkup_python/metrics/version_check.py @@ -1,4 +1,5 @@ -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.types import Context from checkup_python.metrics.utils import parse_semantic_version from checkup_python.metrics.version import PythonVersionMetric @@ -21,10 +22,8 @@ class PythonVersionCheckMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [PythonVersionMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - actual_version = self.get_single(measurements, PythonVersionMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + actual_version = measurements.get(PythonVersionMetric).value actual = parse_semantic_version(actual_version) min_ver = parse_semantic_version(self.min_version) diff --git a/src/checkup/__init__.py b/src/checkup/__init__.py index 0ca9cf5..1d7b465 100644 --- a/src/checkup/__init__.py +++ b/src/checkup/__init__.py @@ -14,18 +14,23 @@ Materializer, SQLAlchemyMaterializer, ) -from checkup.metric import ExecutorType, Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import ExecutorType, Metric from checkup.provider import Provider from checkup.providers.tags import TagProvider from checkup.types import Context from checkup.utils import suppress_subprocess_output +# Rebuild models to resolve forward references after all classes are imported +Measurement.model_rebuild() + __all__ = [ # Core "CheckHub", "MeasurementResult", "Metric", "Measurement", + "Measurements", "ExecutorType", "Provider", "TagProvider", diff --git a/src/checkup/executor/batch_executors.py b/src/checkup/executor/batch_executors.py index 1f95798..97a1942 100644 --- a/src/checkup/executor/batch_executors.py +++ b/src/checkup/executor/batch_executors.py @@ -6,7 +6,8 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed from typing import Any -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.types import Context from checkup.validators import validate_pickleable @@ -17,7 +18,7 @@ def _calculate_metric_in_process( metric: Metric, context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> Measurement: """ Calculate a single metric in a subprocess. @@ -34,7 +35,7 @@ def execute_batch_thread( batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> list[tuple[Metric, Measurement]]: """ Execute metrics using ThreadPoolExecutor. @@ -66,7 +67,7 @@ def execute_batch_process( batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> list[tuple[Metric, Measurement]]: """ Execute metrics using ProcessPoolExecutor. @@ -109,7 +110,7 @@ def execute_batch_asyncio( batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> list[tuple[Metric, Measurement]]: """ Execute metrics using asyncio. @@ -122,7 +123,7 @@ async def _execute_batch_asyncio_impl( batch: list[Metric], context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> list[tuple[Metric, Measurement]]: """ Async implementation of batch execution. @@ -139,7 +140,7 @@ async def _calculate_async_metric( metric: Metric, context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> Measurement: """ Calculate a single metric, handling both sync and async calculate methods. @@ -157,7 +158,7 @@ def _calculate_single_metric( metric: Metric, context: Context, tags: dict[str, Any], - calculated: dict[type[Metric], list[Measurement]], + calculated: Measurements, ) -> Measurement: """ Calculate a single metric (for thread executor). diff --git a/src/checkup/executor/metric_calculator.py b/src/checkup/executor/metric_calculator.py index dfe1d24..d5b5bd3 100644 --- a/src/checkup/executor/metric_calculator.py +++ b/src/checkup/executor/metric_calculator.py @@ -18,7 +18,8 @@ get_failed_dependencies, should_skip, ) -from checkup.metric import ExecutorType, Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import ExecutorType, Metric from checkup.provider import Provider logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def calculate( tags=tags, provided_classes=provided_classes, failed_providers=failed_providers or {}, - calculated=defaultdict(list), + calculated=Measurements(), ) class_to_instances = self._group_by_class(metrics, execution_order) @@ -177,7 +178,7 @@ def _record_failures( for metric in instances: measurement = create_failed_measurement(metric, state.tags, failed_deps) - state.calculated[metric_cls].append(measurement) + state.calculated.append(metric_cls, measurement) state.results.append(measurement) state.failed.add(metric_cls) @@ -204,7 +205,7 @@ def _execute_batch( results = execute_fn(batch, state.context, state.tags, state.calculated) for metric, measurement in results: - state.calculated[type(metric)].append(measurement) + state.calculated.append(type(metric), measurement) state.results.append(measurement) logger.debug( "Metric %s calculated: value=%s", metric.name, measurement.value diff --git a/src/checkup/executor/state.py b/src/checkup/executor/state.py index d54dbfe..271f941 100644 --- a/src/checkup/executor/state.py +++ b/src/checkup/executor/state.py @@ -7,7 +7,8 @@ from typing import Any from checkup.errors import ProviderError -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class CalculationState: tags: dict[str, Any] provided_classes: set[type[Provider]] failed_providers: dict[type[Provider], ProviderError] - calculated: dict[type[Metric], list[Measurement]] = field(default_factory=dict) + calculated: Measurements = field(default_factory=lambda: Measurements({})) skipped: set[type[Metric]] = field(default_factory=set) failed: set[type[Metric]] = field(default_factory=set) results: list[Measurement] = field(default_factory=list) diff --git a/src/checkup/hub.py b/src/checkup/hub.py index 7bbe72f..3364cbd 100644 --- a/src/checkup/hub.py +++ b/src/checkup/hub.py @@ -11,7 +11,8 @@ from checkup.errors import DuplicateMetricNameError, MetricPicklingError, ProviderError from checkup.executor import MetricCalculator, ProviderExecutor from checkup.graph import build_dependency_graph, topological_sort -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement +from checkup.metric import Metric from checkup.provider import Provider from checkup.validators import validate_providers, validate_unique_metric_names diff --git a/src/checkup/materializers/base.py b/src/checkup/materializers/base.py index d3ebf48..406db94 100644 --- a/src/checkup/materializers/base.py +++ b/src/checkup/materializers/base.py @@ -5,7 +5,7 @@ from pydantic import BaseModel -from checkup.metric import Measurement +from checkup.measurement import Measurement def group_measurements_by_tags( diff --git a/src/checkup/materializers/console.py b/src/checkup/materializers/console.py index 3cd7885..91e0502 100644 --- a/src/checkup/materializers/console.py +++ b/src/checkup/materializers/console.py @@ -4,7 +4,7 @@ from rich.table import Table from checkup.materializers.base import Materializer -from checkup.metric import Measurement +from checkup.measurement import Measurement class ConsoleMaterializer(Materializer): diff --git a/src/checkup/materializers/csv_file.py b/src/checkup/materializers/csv_file.py index 6aebd9d..a237d8d 100644 --- a/src/checkup/materializers/csv_file.py +++ b/src/checkup/materializers/csv_file.py @@ -4,7 +4,7 @@ from pathlib import Path from checkup.materializers.base import Materializer -from checkup.metric import Measurement +from checkup.measurement import Measurement class CSVMaterializer(Materializer): diff --git a/src/checkup/materializers/database.py b/src/checkup/materializers/database.py index 89a8b73..af252c4 100644 --- a/src/checkup/materializers/database.py +++ b/src/checkup/materializers/database.py @@ -19,7 +19,7 @@ ) from checkup.materializers.base import Materializer -from checkup.metric import Measurement +from checkup.measurement import Measurement class SQLAlchemyMaterializer(Materializer): diff --git a/src/checkup/materializers/html_report.py b/src/checkup/materializers/html_report.py index a46f5c7..0b42352 100644 --- a/src/checkup/materializers/html_report.py +++ b/src/checkup/materializers/html_report.py @@ -5,7 +5,7 @@ from jinja2 import Environment, FileSystemLoader from checkup.materializers.base import Materializer, group_measurements_hierarchical -from checkup.metric import Measurement +from checkup.measurement import Measurement class HTMLMaterializer(Materializer): diff --git a/src/checkup/measurement.py b/src/checkup/measurement.py new file mode 100644 index 0000000..1c19338 --- /dev/null +++ b/src/checkup/measurement.py @@ -0,0 +1,96 @@ +"""Measurement and Measurements classes.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterator, Mapping, Sequence +from typing import TYPE_CHECKING, Any, overload + +from pydantic import BaseModel, Field + +if TYPE_CHECKING: + from checkup.metric import Metric + + +class Measurement(BaseModel): + """ + Result of a metric calculation. + """ + + metric: Metric + value: Any = None + tags: dict = Field(default_factory=dict) + diagnostic: str = "" + + +class Measurements(Mapping[type["Metric"], Sequence["Measurement"]]): + """ + Mapping from metrics to their measurements. + + Example usage: + + # Get all measurements for a metric + all_measurements = measurements[MyMetric] + + # Get a single measurement + measurement = measurements.get(MyMetric) + + # Get a measurement by name + measurement = measurements.get(MyMetric, name="my_metric_instance") + """ + + def __init__(self, data: dict[type[Metric], Sequence[Measurement]] | None = None): + self._data: dict[type[Metric], list[Measurement]] = defaultdict(list) + if data: + for metric_cls, measurements in data.items(): + self._data[metric_cls] = list(measurements) + + def __getitem__(self, metric_cls: type[Metric]) -> Sequence[Measurement]: + return self._data.get(metric_cls, []) + + def __iter__(self) -> Iterator[type[Metric]]: + return iter(self._data) + + def __len__(self) -> int: + return len(self._data) + + def __contains__(self, metric_cls: object) -> bool: + return metric_cls in self._data and len(self._data[metric_cls]) > 0 + + @overload + def get(self, metric_cls: type[Metric]) -> Measurement | None: ... + + @overload + def get(self, metric_cls: type[Metric], *, name: str) -> Measurement | None: ... + + def get( + self, metric_cls: type[Metric], *, name: str | None = None + ) -> Measurement | None: + """ + Get a measurement, optionally filtered by metric name. + + Args: + metric_cls: The metric class to look up + name: Optional metric name to filter by + + Returns: + The matching Measurement, or None if not found. + Raises ValueError if multiple matches found. + """ + + results = self._data.get(metric_cls, []) + if name is not None: + results = [m for m in results if m.metric.name == name] + + if len(results) == 0: + return None + if len(results) > 1: + raise ValueError( + f"Multiple measurements match for {metric_cls.__name__}" + + (f" with name '{name}'" if name else "") + ) + + return results[0] + + def append(self, metric_cls: type[Metric], measurement: Measurement) -> None: + self._data[metric_cls].append(measurement) diff --git a/src/checkup/metric.py b/src/checkup/metric.py index db687cc..8bdc764 100644 --- a/src/checkup/metric.py +++ b/src/checkup/metric.py @@ -1,14 +1,17 @@ -"""Metric and Measurement classes.""" +"""Metric base class.""" + +from __future__ import annotations from abc import ABC, abstractmethod from enum import Enum from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, Field +from pydantic import BaseModel from checkup.types import Context if TYPE_CHECKING: + from checkup.measurement import Measurement, Measurements from checkup.provider import Provider @@ -53,17 +56,13 @@ def get_executor(cls) -> ExecutorType: return getattr(cls, "executor", ExecutorType.THREAD) @abstractmethod - def calculate( - self, - context: Context, - measurements: dict[type["Metric"], list["Measurement"]], - ) -> "Measurement": + def calculate(self, context: Context, measurements: Measurements) -> Measurement: """ Calculate metric and return a Measurement. Args: context: General context enriched by providers - measurements: Dict mapping Metric classes to lists of their Measurements + measurements: Wrapper for accessing calculated dependency measurements Returns: Measurement with the calculated value @@ -75,7 +74,7 @@ def measure( value: Any = None, tags: dict | None = None, diagnostic: str = "", - ) -> "Measurement": + ) -> Measurement: """ Create a Measurement for this metric. @@ -89,6 +88,8 @@ def measure( Returns: Measurement instance """ + from checkup.measurement import Measurement + return Measurement( metric=self, value=value, @@ -97,7 +98,7 @@ def measure( ) @classmethod - def depends_on(cls) -> list[type["Metric"]]: + def depends_on(cls) -> list[type[Metric]]: """ Return list of metric classes this metric depends on. @@ -107,7 +108,7 @@ def depends_on(cls) -> list[type["Metric"]]: return [] @classmethod - def providers(cls) -> list[type["Provider"]]: + def providers(cls) -> list[type[Provider]]: """ Return list of provider classes to enrich context. @@ -115,48 +116,3 @@ def providers(cls) -> list[type["Provider"]]: List of provider classes (empty by default) """ return [] - - def get_single( - self, - measurements: dict[type["Metric"], list["Measurement"]], - metric_cls: type["Metric"], - ) -> "Measurement": - """ - Get a single measurement for a dependency, erroring if not exactly one. - - Convenience method for metrics that expect exactly one instance of a dependent metric. - - Args: - measurements: The measurements dict passed to calculate() - metric_cls: The metric class to look up - - Returns: - The single Measurement - - Raises: - ValueError: If there are zero or multiple measurements for the class - """ - - results = measurements.get(metric_cls, []) - if len(results) == 0: - raise ValueError(f"No measurements found for {metric_cls.__name__}") - if len(results) > 1: - raise ValueError( - f"Expected single measurement for {metric_cls.__name__}, " - f"got {len(results)}" - ) - - return results[0] - - -class Measurement(BaseModel): - """ - Result of a metric calculation. - - Holds the metric that produced it, the calculated value, tags, and diagnostic information. - """ - - metric: Metric - value: Any = None - tags: dict = Field(default_factory=dict) - diagnostic: str = "" diff --git a/tests/fixtures.py b/tests/fixtures.py index 292503a..ceebe1f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,6 +1,7 @@ from typing import Any, ClassVar -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider from checkup.types import Context @@ -14,9 +15,7 @@ class DummyMetric(Metric): expected_value: int = 42 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: """Set value to expected_value.""" return self.measure( value=self.expected_value, @@ -36,11 +35,9 @@ def depends_on(cls) -> list[type[Metric]]: """Depends on DummyMetric.""" return [DummyMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: """Double the DummyMetric value.""" - base_value = self.get_single(measurements, DummyMetric).value + base_value = measurements.get(DummyMetric).value value = base_value * 2 return self.measure( value=value, @@ -59,10 +56,8 @@ class Level2Metric(Metric): def depends_on(cls) -> list[type[Metric]]: return [DependentDummyMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - value = self.get_single(measurements, DependentDummyMetric).value + 10 + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + value = measurements.get(DependentDummyMetric).value + 10 return self.measure( value=value, diagnostic=f"Added 10 to DependentDummyMetric value: {value}" ) @@ -79,10 +74,8 @@ class Level3Metric(Metric): def depends_on(cls) -> list[type[Metric]]: return [Level2Metric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - level2_value = self.get_single(measurements, Level2Metric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + level2_value = measurements.get(Level2Metric).value value = level2_value**2 return self.measure( value=value, @@ -101,9 +94,7 @@ class CyclicMetricA(Metric): def depends_on(cls) -> list[type[Metric]]: return [CyclicMetricB] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=1, diagnostic="CyclicMetricA calculated") @@ -118,9 +109,7 @@ class CyclicMetricB(Metric): def depends_on(cls) -> list[type[Metric]]: return [CyclicMetricA] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=1, diagnostic="CyclicMetricB calculated") @@ -132,9 +121,7 @@ class RootA(Metric): unit: str = "count" base_value: int = 10 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure( value=self.base_value, diagnostic=f"RootA calculated with base_value={self.base_value}", @@ -149,9 +136,7 @@ class RootB(Metric): unit: str = "count" base_value: int = 20 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure( value=self.base_value, diagnostic=f"RootB calculated with base_value={self.base_value}", @@ -166,9 +151,7 @@ class RootC(Metric): unit: str = "count" base_value: int = 100 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure( value=self.base_value, diagnostic=f"RootC calculated with base_value={self.base_value}", @@ -186,11 +169,9 @@ class SharedAB(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootA, RootB] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - root_a_val = self.get_single(measurements, RootA).value - root_b_val = self.get_single(measurements, RootB).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_a_val = measurements.get(RootA).value + root_b_val = measurements.get(RootB).value value = root_a_val + root_b_val return self.measure( value=value, @@ -209,10 +190,8 @@ class BranchB(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootB] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - root_b_val = self.get_single(measurements, RootB).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_b_val = measurements.get(RootB).value value = root_b_val * 3 return self.measure( value=value, diagnostic=f"Tripled RootB value: {root_b_val} * 3 = {value}" @@ -230,10 +209,8 @@ class LeafC(Metric): def depends_on(cls) -> list[type[Metric]]: return [RootC] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - root_c_val = self.get_single(measurements, RootC).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + root_c_val = measurements.get(RootC).value value = root_c_val**2 return self.measure( value=value, diagnostic=f"Squared RootC value: {root_c_val}^2 = {value}" @@ -251,10 +228,8 @@ class MidShared(Metric): def depends_on(cls) -> list[type[Metric]]: return [SharedAB] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - shared_ab_val = self.get_single(measurements, SharedAB).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + shared_ab_val = measurements.get(SharedAB).value value = shared_ab_val + 5 return self.measure( value=value, @@ -273,10 +248,8 @@ class MidBranch(Metric): def depends_on(cls) -> list[type[Metric]]: return [BranchB] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - branch_b_val = self.get_single(measurements, BranchB).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + branch_b_val = measurements.get(BranchB).value value = branch_b_val * 2 return self.measure( value=value, @@ -295,11 +268,9 @@ class LeafAB(Metric): def depends_on(cls) -> list[type[Metric]]: return [MidShared, MidBranch] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - mid_shared_val = self.get_single(measurements, MidShared).value - mid_branch_val = self.get_single(measurements, MidBranch).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + mid_shared_val = measurements.get(MidShared).value + mid_branch_val = measurements.get(MidBranch).value value = mid_shared_val * mid_branch_val return self.measure( value=value, @@ -330,9 +301,7 @@ class ProviderDummyMetric(Metric): def providers(cls) -> list[type[Provider]]: return [DummyProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: value = context[DummyProvider.name]["data"] return self.measure( value=value, diagnostic=f"Retrieved dummy_data from context: {value}" @@ -346,9 +315,7 @@ class FailingMetric(Metric): description: str = "Fails when should_fail is True" unit: str = "count" - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: if context.get("should_fail"): raise ValueError("Intentional failure") return self.measure(value=1, diagnostic="Metric calculated successfully") @@ -378,9 +345,7 @@ class IntegrationBaseMetric(Metric): def providers(cls) -> list[type[Provider]]: return [IntegrationProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: value = context[IntegrationProvider.name]["base_value"] return self.measure( value=value, @@ -400,10 +365,8 @@ class IntegrationDerivedMetric(Metric): def depends_on(cls): return [IntegrationBaseMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - base_val = self.get_single(measurements, IntegrationBaseMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_val = measurements.get(IntegrationBaseMetric).value value = base_val * self.multiplier return self.measure( value=value, @@ -435,9 +398,7 @@ class PathMetric(Metric): def providers(cls) -> list[type[Provider]]: return [PathLengthProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: path_len = context[PathLengthProvider.name]["length"] value = path_len * self.multiplier return self.measure( @@ -455,9 +416,7 @@ class OtherDummyMetric(Metric): expected_value: int = 100 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: """Set value to expected_value.""" return self.measure( value=self.expected_value, @@ -474,9 +433,7 @@ class IndirectDummyMetric(Metric): expected_value: int = 100 - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: """Set value to expected_value.""" return self.measure( value=self.expected_value, diff --git a/tests/test_executors.py b/tests/test_executors.py index 3673a2a..62770ab 100644 --- a/tests/test_executors.py +++ b/tests/test_executors.py @@ -3,7 +3,8 @@ from typing import ClassVar from checkup.hub import CheckHub -from checkup.metric import ExecutorType, Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import ExecutorType, Metric from checkup.types import Context @@ -15,9 +16,7 @@ class ThreadMetric(Metric): unit: str = "count" # executor defaults to ExecutorType.THREAD - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=10, diagnostic="Calculated in thread") @@ -29,9 +28,7 @@ class ProcessMetric(Metric): unit: str = "count" executor: ClassVar[ExecutorType] = ExecutorType.PROCESS - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=20, diagnostic="Calculated in process") @@ -43,9 +40,7 @@ class AsyncMetric(Metric): unit: str = "count" executor: ClassVar[ExecutorType] = ExecutorType.ASYNCIO - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=30, diagnostic="Calculated with asyncio") @@ -58,7 +53,7 @@ class AsyncMetricWithAsyncCalculate(Metric): executor: ClassVar[ExecutorType] = ExecutorType.ASYNCIO async def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] + self, context: Context, measurements: Measurements ) -> Measurement: import asyncio @@ -78,10 +73,8 @@ class DependentThreadMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ThreadMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - base_value = self.get_single(measurements, ThreadMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_value = measurements.get(ThreadMetric).value value = base_value * 2 return self.measure( value=value, diagnostic=f"Doubled thread metric: {base_value} -> {value}" @@ -100,10 +93,8 @@ class DependentProcessMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ThreadMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - base_value = self.get_single(measurements, ThreadMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_value = measurements.get(ThreadMetric).value value = base_value * 3 return self.measure( value=value, diagnostic=f"Tripled thread metric: {base_value} -> {value}" @@ -122,10 +113,8 @@ class DependentAsyncMetric(Metric): def depends_on(cls) -> list[type[Metric]]: return [ProcessMetric] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - base_value = self.get_single(measurements, ProcessMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_value = measurements.get(ProcessMetric).value value = base_value + 5 return self.measure( value=value, diff --git a/tests/test_graph.py b/tests/test_graph.py index c2528b3..0803828 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -22,288 +22,254 @@ ) from checkup.graph import build_dependency_graph, topological_sort -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurements +from checkup.metric import Metric -# ============================================================================= -# build_dependency_graph tests -# ============================================================================= +class TestBuildDependencyGraph: + def test_no_dependencies(self): + """Test building graph for metric with no dependencies.""" + graph = build_dependency_graph([DummyMetric]) -def test_build_graph_no_dependencies(): - """Test building graph for metric with no dependencies.""" - graph = build_dependency_graph([DummyMetric]) + assert graph == {DummyMetric: []} - assert graph == {DummyMetric: []} + def test_with_dependencies(self): + """Test building graph for metrics with dependencies.""" + graph = build_dependency_graph([DependentDummyMetric, DummyMetric]) + assert graph == {DummyMetric: [], DependentDummyMetric: [DummyMetric]} -def test_build_graph_with_dependencies(): - """Test building graph for metrics with dependencies.""" - graph = build_dependency_graph([DependentDummyMetric, DummyMetric]) + def test_auto_adds_missing_dependencies(self): + """Test that missing dependencies are automatically added.""" + graph = build_dependency_graph([DependentDummyMetric]) - assert graph == {DummyMetric: [], DependentDummyMetric: [DummyMetric]} + assert DummyMetric in graph + assert DependentDummyMetric in graph -def test_build_graph_auto_adds_missing_dependencies(): - """Test that missing dependencies are automatically added.""" - graph = build_dependency_graph([DependentDummyMetric]) +class TestTopologicalSort: + def test_no_dependencies(self): + """Test topological sort with no dependencies.""" + graph: dict[type[Metric], list[type[Metric]]] = {DummyMetric: []} - assert DummyMetric in graph - assert DependentDummyMetric in graph + result = topological_sort(graph) + assert result == [DummyMetric] -# ============================================================================= -# topological_sort tests -# ============================================================================= + def test_with_dependencies(self): + """Test topological sort with dependencies.""" + graph = {DummyMetric: [], DependentDummyMetric: [DummyMetric]} + result = topological_sort(graph) + + assert result.index(DummyMetric) < result.index(DependentDummyMetric) + assert len(result) == 2 -def test_topological_sort_no_dependencies(): - """Test topological sort with no dependencies.""" - graph: dict[type[Metric], list[type[Metric]]] = {DummyMetric: []} - - result = topological_sort(graph) - - assert result == [DummyMetric] - - -def test_topological_sort_with_dependencies(): - """Test topological sort with dependencies.""" - graph = {DummyMetric: [], DependentDummyMetric: [DummyMetric]} - - result = topological_sort(graph) - - assert result.index(DummyMetric) < result.index(DependentDummyMetric) - assert len(result) == 2 - - -def test_topological_sort_deep_chain(): - """Test topological sort with depth 3 dependency chain. + def test_deep_chain(self): + """Test topological sort with depth 3 dependency chain. + + Chain: DummyMetric -> DependentDummyMetric -> Level2Metric -> Level3Metric + """ + graph = build_dependency_graph([Level3Metric]) + + result = topological_sort(graph) + + assert len(result) == 4 + assert result.index(DummyMetric) < result.index(DependentDummyMetric) + assert result.index(DependentDummyMetric) < result.index(Level2Metric) + assert result.index(Level2Metric) < result.index(Level3Metric) + + def test_deep_chain_calculation(self, empty_context): + """Test that deep dependency chain calculates correctly. + + DummyMetric(10) -> DependentDummyMetric(20) -> Level2Metric(30) -> Level3Metric(900) + """ + graph = build_dependency_graph([Level3Metric]) + order = topological_sort(graph) + + calculated = Measurements() + + for metric_cls in order: + if metric_cls is DummyMetric: + metric: Metric = DummyMetric(expected_value=10) + else: + metric = metric_cls() # type: ignore + measurement = metric.calculate(empty_context, calculated) + calculated.append(metric_cls, measurement) + + assert calculated.get(DummyMetric).value == 10 + assert calculated.get(DependentDummyMetric).value == 20 # 10 * 2 + assert calculated.get(Level2Metric).value == 30 # 20 + 10 + assert calculated.get(Level3Metric).value == 900 # 30 ** 2 + + +class TestCycleDetection: + def test_direct_cycle(self): + """Test that topological sort detects direct cycles between two metrics.""" + graph = { + CyclicMetricA: [CyclicMetricB], + CyclicMetricB: [CyclicMetricA], + } + + with pytest.raises(CycleError): + topological_sort(graph) + + def test_via_build_graph(self): + """Test cycle detection when building graph from cyclic metrics.""" + graph = build_dependency_graph([CyclicMetricA]) + + with pytest.raises(CycleError): + topological_sort(graph) + + +class TestComplexDependencyGraph: + def test_structure(self): + """Test building graph with shared ancestors and multiple branches. + + Graph structure: + RootA RootB RootC + \\ / \\ | + \\ / \\ | + SharedAB BranchB LeafC + | | + MidShared MidBranch + \\ / + LeafAB + """ + graph = build_dependency_graph([LeafAB, LeafC]) + + # All metrics should be included + assert RootA in graph + assert RootB in graph + assert RootC in graph + assert SharedAB in graph + assert BranchB in graph + assert LeafC in graph + assert MidShared in graph + assert MidBranch in graph + assert LeafAB in graph + + # Verify dependencies + assert graph[RootA] == [] + assert graph[RootB] == [] + assert graph[RootC] == [] + assert set(graph[SharedAB]) == {RootA, RootB} + assert graph[BranchB] == [RootB] + assert graph[LeafC] == [RootC] + assert graph[MidShared] == [SharedAB] + assert graph[MidBranch] == [BranchB] + assert set(graph[LeafAB]) == {MidShared, MidBranch} + + def test_topological_order(self): + """Test topological sort respects all dependency constraints.""" + graph = build_dependency_graph([LeafAB, LeafC]) + order = topological_sort(graph) + + # Roots must come before their dependents + assert order.index(RootA) < order.index(SharedAB) + assert order.index(RootB) < order.index(SharedAB) + assert order.index(RootB) < order.index(BranchB) + assert order.index(RootC) < order.index(LeafC) + + # Mid-level must come before leaves + assert order.index(SharedAB) < order.index(MidShared) + assert order.index(BranchB) < order.index(MidBranch) + assert order.index(MidShared) < order.index(LeafAB) + assert order.index(MidBranch) < order.index(LeafAB) + + def test_calculation(self, empty_context): + """Test full calculation of complex dependency graph. + + Expected values: + - RootA: 10 + - RootB: 20 + - RootC: 100 + - SharedAB: 10 + 20 = 30 + - BranchB: 20 * 3 = 60 + - LeafC: 100 ** 2 = 10000 + - MidShared: 30 + 5 = 35 + - MidBranch: 60 * 2 = 120 + - LeafAB: 35 * 120 = 4200 + """ + graph = build_dependency_graph([LeafAB, LeafC]) + order = topological_sort(graph) + + calculated = Measurements() + + for metric_cls in order: + metric = metric_cls() # type: ignore + measurement = metric.calculate(empty_context, calculated) + calculated.append(metric_cls, measurement) - Chain: DummyMetric → DependentDummyMetric → Level2Metric → Level3Metric - """ - graph = build_dependency_graph([Level3Metric]) + # Verify all calculations + assert calculated.get(RootA).value == 10 + assert calculated.get(RootB).value == 20 + assert calculated.get(RootC).value == 100 + assert calculated.get(SharedAB).value == 30 # 10 + 20 + assert calculated.get(BranchB).value == 60 # 20 * 3 + assert calculated.get(LeafC).value == 10000 # 100 ** 2 + assert calculated.get(MidShared).value == 35 # 30 + 5 + assert calculated.get(MidBranch).value == 120 # 60 * 2 + assert calculated.get(LeafAB).value == 4200 # 35 * 120 - result = topological_sort(graph) + def test_via_checkhub(self): + """Test complex graph calculation through CheckHub.""" + from checkup import CheckHub - assert len(result) == 4 - assert result.index(DummyMetric) < result.index(DependentDummyMetric) - assert result.index(DependentDummyMetric) < result.index(Level2Metric) - assert result.index(Level2Metric) < result.index(Level3Metric) + result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() + measurements_by_name = {m.metric.name: m for m in result.measurements} -# ============================================================================= -# Deep chain calculation tests -# ============================================================================= + # All 9 metrics returned (direct and indirect) + assert len(result.measurements) == 9 + # Only requested metrics are marked as direct + assert result.direct_metric_names == {"leaf_ab", "leaf_c"} -def test_deep_chain_calculation(empty_context): - """Test that deep dependency chain calculates correctly. + # Verify calculated values + assert measurements_by_name["leaf_ab"].value == 4200 + assert measurements_by_name["leaf_c"].value == 10000 - DummyMetric(10) → DependentDummyMetric(20) → Level2Metric(30) → Level3Metric(900) - """ - from checkup.metric import Measurement + def test_shared_ancestor_calculated_once(self, empty_context): + """Test that RootB (shared by SharedAB and BranchB) is only calculated once. - graph = build_dependency_graph([Level3Metric]) - order = topological_sort(graph) + This verifies the framework doesn't re-calculate metrics that appear + multiple times in the dependency graph. + """ + graph = build_dependency_graph([LeafAB]) + order = topological_sort(graph) - calculated: dict[type[Metric], list[Measurement]] = {} + # Track how many times each metric class is calculated + calculation_counts: dict[type[Metric], int] = {} + calculated = Measurements() - for metric_cls in order: - if metric_cls is DummyMetric: - metric: Metric = DummyMetric(expected_value=10) - else: + for metric_cls in order: + calculation_counts[metric_cls] = calculation_counts.get(metric_cls, 0) + 1 metric = metric_cls() # type: ignore - measurement = metric.calculate(empty_context, calculated) - calculated[metric_cls] = [measurement] - - assert calculated[DummyMetric][0].value == 10 - assert calculated[DependentDummyMetric][0].value == 20 # 10 * 2 - assert calculated[Level2Metric][0].value == 30 # 20 + 10 - assert calculated[Level3Metric][0].value == 900 # 30 ** 2 - - -# ============================================================================= -# Cycle detection tests -# ============================================================================= - - -def test_cycle_detection_direct(): - """Test that topological sort detects direct cycles between two metrics.""" - graph = { - CyclicMetricA: [CyclicMetricB], - CyclicMetricB: [CyclicMetricA], - } - - with pytest.raises(CycleError): - topological_sort(graph) - - -def test_cycle_detection_via_build_graph(): - """Test cycle detection when building graph from cyclic metrics.""" - graph = build_dependency_graph([CyclicMetricA]) - - with pytest.raises(CycleError): - topological_sort(graph) - - -# ============================================================================= -# Complex dependency graph tests -# ============================================================================= - - -def test_complex_graph_structure(): - """Test building graph with shared ancestors and multiple branches. - - Graph structure: - RootA RootB RootC - \\ / \\ | - \\ / \\ | - SharedAB BranchB LeafC - | | - MidShared MidBranch - \\ / - LeafAB - """ - graph = build_dependency_graph([LeafAB, LeafC]) - - # All metrics should be included - assert RootA in graph - assert RootB in graph - assert RootC in graph - assert SharedAB in graph - assert BranchB in graph - assert LeafC in graph - assert MidShared in graph - assert MidBranch in graph - assert LeafAB in graph - - # Verify dependencies - assert graph[RootA] == [] - assert graph[RootB] == [] - assert graph[RootC] == [] - assert set(graph[SharedAB]) == {RootA, RootB} - assert graph[BranchB] == [RootB] - assert graph[LeafC] == [RootC] - assert graph[MidShared] == [SharedAB] - assert graph[MidBranch] == [BranchB] - assert set(graph[LeafAB]) == {MidShared, MidBranch} - - -def test_complex_graph_topological_order(): - """Test topological sort respects all dependency constraints.""" - graph = build_dependency_graph([LeafAB, LeafC]) - order = topological_sort(graph) - - # Roots must come before their dependents - assert order.index(RootA) < order.index(SharedAB) - assert order.index(RootB) < order.index(SharedAB) - assert order.index(RootB) < order.index(BranchB) - assert order.index(RootC) < order.index(LeafC) - - # Mid-level must come before leaves - assert order.index(SharedAB) < order.index(MidShared) - assert order.index(BranchB) < order.index(MidBranch) - assert order.index(MidShared) < order.index(LeafAB) - assert order.index(MidBranch) < order.index(LeafAB) - - -def test_complex_graph_calculation(empty_context): - """Test full calculation of complex dependency graph. - - Expected values: - - RootA: 10 - - RootB: 20 - - RootC: 100 - - SharedAB: 10 + 20 = 30 - - BranchB: 20 * 3 = 60 - - LeafC: 100 ** 2 = 10000 - - MidShared: 30 + 5 = 35 - - MidBranch: 60 * 2 = 120 - - LeafAB: 35 * 120 = 4200 - """ - graph = build_dependency_graph([LeafAB, LeafC]) - order = topological_sort(graph) - - calculated: dict[type[Metric], list[Measurement]] = {} - - for metric_cls in order: - metric = metric_cls() # type: ignore - measurement = metric.calculate(empty_context, calculated) - calculated[metric_cls] = [measurement] - - # Verify all calculations - assert calculated[RootA][0].value == 10 - assert calculated[RootB][0].value == 20 - assert calculated[RootC][0].value == 100 - assert calculated[SharedAB][0].value == 30 # 10 + 20 - assert calculated[BranchB][0].value == 60 # 20 * 3 - assert calculated[LeafC][0].value == 10000 # 100 ** 2 - assert calculated[MidShared][0].value == 35 # 30 + 5 - assert calculated[MidBranch][0].value == 120 # 60 * 2 - assert calculated[LeafAB][0].value == 4200 # 35 * 120 - - -def test_complex_graph_via_checkhub(): - """Test complex graph calculation through CheckHub.""" - from checkup import CheckHub - - result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() - - measurements_by_name = {m.metric.name: m for m in result.measurements} - - # All 9 metrics returned (direct and indirect) - assert len(result.measurements) == 9 - - # Only requested metrics are marked as direct - assert result.direct_metric_names == {"leaf_ab", "leaf_c"} - - # Verify calculated values - assert measurements_by_name["leaf_ab"].value == 4200 - assert measurements_by_name["leaf_c"].value == 10000 - - -def test_shared_ancestor_calculated_once(empty_context): - """Test that RootB (shared by SharedAB and BranchB) is only calculated once. - - This verifies the framework doesn't re-calculate metrics that appear - multiple times in the dependency graph. - """ - from checkup.metric import Measurement - - graph = build_dependency_graph([LeafAB]) - order = topological_sort(graph) - - # Track how many times each metric class is calculated - calculation_counts: dict[type[Metric], int] = {} - calculated: dict[type[Metric], list[Measurement]] = {} - - for metric_cls in order: - calculation_counts[metric_cls] = calculation_counts.get(metric_cls, 0) + 1 - metric = metric_cls() # type: ignore - measurement = metric.calculate(empty_context, calculated) - calculated[metric_cls] = [measurement] - - # Each metric should appear exactly once in the execution order - for metric_cls, count in calculation_counts.items(): - assert count == 1, f"{metric_cls.__name__} was calculated {count} times" + measurement = metric.calculate(empty_context, calculated) + calculated.append(metric_cls, measurement) - # Specifically verify RootB (shared ancestor) is only calculated once - assert calculation_counts[RootB] == 1 + # Each metric should appear exactly once in the execution order + for metric_cls, count in calculation_counts.items(): + assert count == 1, f"{metric_cls.__name__} was calculated {count} times" + # Specifically verify RootB (shared ancestor) is only calculated once + assert calculation_counts[RootB] == 1 -def test_independent_subgraphs(): - """Test that independent subgraphs can coexist without interference.""" - from checkup import CheckHub + def test_independent_subgraphs(self): + """Test that independent subgraphs can coexist without interference.""" + from checkup import CheckHub - # Only request LeafC (independent subgraph) - returns LeafC and RootC - result = CheckHub().with_metrics([LeafC()]).measure() + # Only request LeafC (independent subgraph) - returns LeafC and RootC + result = CheckHub().with_metrics([LeafC()]).measure() - assert len(result.measurements) == 2 - assert result.direct_metric_names == {"leaf_c"} - measurements_by_name = {m.metric.name: m for m in result.measurements} - assert measurements_by_name["leaf_c"].value == 10000 + assert len(result.measurements) == 2 + assert result.direct_metric_names == {"leaf_c"} + measurements_by_name = {m.metric.name: m for m in result.measurements} + assert measurements_by_name["leaf_c"].value == 10000 - # Request both subgraphs - returns all 9 metrics - result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() + # Request both subgraphs - returns all 9 metrics + result = CheckHub().with_metrics([LeafAB(), LeafC()]).measure() - assert len(result.measurements) == 9 - assert result.direct_metric_names == {"leaf_ab", "leaf_c"} + assert len(result.measurements) == 9 + assert result.direct_metric_names == {"leaf_ab", "leaf_c"} diff --git a/tests/test_hub_execution.py b/tests/test_hub_execution.py index 205592b..4f9afce 100644 --- a/tests/test_hub_execution.py +++ b/tests/test_hub_execution.py @@ -10,7 +10,8 @@ ) from checkup.hub import CheckHub -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider from checkup.providers.tags import TagProvider from checkup.types import Context @@ -39,9 +40,7 @@ class DataMetric(Metric): def providers(cls) -> list[type[Provider]]: return [DataProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: value = context[DataProvider.name]["value"] return self.measure(value=value) @@ -75,9 +74,7 @@ class FailingProviderMetric(Metric): def providers(cls) -> list[type[Provider]]: return [FailingProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=999) @@ -96,10 +93,8 @@ def depends_on(cls) -> list[type[Metric]]: def providers(cls) -> list[type[Provider]]: return [FailingProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: - base_val = self.get_single(measurements, FailingProviderMetric).value + def calculate(self, context: Context, measurements: Measurements) -> Measurement: + base_val = measurements.get(FailingProviderMetric).value return self.measure(value=base_val * 2) @@ -114,9 +109,7 @@ class OtherMetric(Metric): def providers(cls) -> list[type[Provider]]: return [OtherProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], list[Measurement]] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: value = context[OtherProvider.name]["other_value"] return self.measure(value=value) @@ -313,10 +306,10 @@ def test_multiple_instances_of_same_metric_class(self): assert measurements_by_name["dummy_b"].value == 20 assert measurements_by_name["dummy_c"].value == 30 - def test_multiple_instances_dependency_with_get_single_fails(self): - """Test that get_single raises when multiple instances exist for a dependency.""" + def test_multiple_instances_dependency_with_get_fails(self): + """Test that .get() raises when multiple instances exist for a dependency.""" - # DependentDummyMetric uses get_single() which should fail with multiple DummyMetric instances + # DependentDummyMetric uses .get() which should fail with multiple DummyMetric instances result = ( CheckHub() .with_metrics( @@ -329,8 +322,6 @@ def test_multiple_instances_dependency_with_get_single_fails(self): .measure() ) - # The whole provider set fails because get_single() raises during calculation + # The whole provider set fails because .get() raises during calculation assert len(result.errors) == 1 - assert "Expected single measurement for DummyMetric, got 2" in str( - result.errors[0][1] - ) + assert "Multiple measurements match for DummyMetric" in str(result.errors[0][1]) diff --git a/tests/test_hub_validation.py b/tests/test_hub_validation.py index 9614ec4..0fa87e9 100644 --- a/tests/test_hub_validation.py +++ b/tests/test_hub_validation.py @@ -2,7 +2,8 @@ from typing import Any, ClassVar -from checkup.metric import Measurement, Metric +from checkup.measurement import Measurement, Measurements +from checkup.metric import Metric from checkup.provider import Provider from checkup.types import Context from checkup.validators import validate_providers @@ -37,9 +38,7 @@ class MetricWithProvider(Metric): def providers(cls) -> list[type[Provider]]: return [RequiredProvider] - def calculate( - self, context: Context, measurements: dict[type[Metric], Measurement] - ) -> Measurement: + def calculate(self, context: Context, measurements: Measurements) -> Measurement: return self.measure(value=1) diff --git a/tests/test_metric.py b/tests/test_metric.py index d1fc290..17f74d8 100644 --- a/tests/test_metric.py +++ b/tests/test_metric.py @@ -8,149 +8,127 @@ ) from pydantic import BaseModel +from checkup.measurement import Measurements from checkup.metric import Metric -# ============================================================================= -# DummyMetric instantiation tests -# ============================================================================= +class TestDummyMetricInstantiation: + def test_with_explicit_value(self): + """Test that DummyMetric can be instantiated with config.""" + metric = DummyMetric(expected_value=42) -def test_dummy_metric_with_explicit_value(): - """Test that DummyMetric can be instantiated with config.""" - metric = DummyMetric(expected_value=42) + assert metric.name == "dummy" + assert metric.description == "Test metric" + assert metric.unit == "count" + assert metric.expected_value == 42 - assert metric.name == "dummy" - assert metric.description == "Test metric" - assert metric.unit == "count" - assert metric.expected_value == 42 + def test_with_default_value(self): + """Test that DummyMetric uses default expected_value.""" + metric = DummyMetric() + assert metric.expected_value == 42 -def test_dummy_metric_with_default_value(): - """Test that DummyMetric uses default expected_value.""" - metric = DummyMetric() - assert metric.expected_value == 42 +class TestDummyMetricCalculation: + def test_returns_measurement(self, empty_context): + """Test that DummyMetric.calculate() returns a Measurement.""" + metric = DummyMetric(expected_value=100) + measurement = metric.calculate( + context=empty_context, measurements=Measurements() + ) -# ============================================================================= -# DummyMetric calculation tests -# ============================================================================= + assert measurement.value == 100 + assert measurement.metric.name == "dummy" + def test_uses_fixture(self, dummy_metric, empty_context): + """Test calculate using pytest fixtures.""" + measurement = dummy_metric.calculate( + context=empty_context, measurements=Measurements() + ) -def test_dummy_metric_calculate_returns_measurement(empty_context): - """Test that DummyMetric.calculate() returns a Measurement.""" - metric = DummyMetric(expected_value=100) + assert measurement.value == 42 - measurement = metric.calculate(context=empty_context, measurements={}) - assert measurement.value == 100 - assert measurement.metric.name == "dummy" +class TestMetricBaseClass: + def test_default_depends_on(self): + """Test that Metric base class depends_on returns empty list.""" + assert DummyMetric.depends_on() == [] + def test_default_providers(self): + """Test that Metric base class providers returns empty list.""" + assert DummyMetric.providers() == [] -def test_dummy_metric_calculate_uses_fixture(dummy_metric, empty_context): - """Test calculate using pytest fixtures.""" - measurement = dummy_metric.calculate(context=empty_context, measurements={}) + def test_is_pydantic_model(self): + """Test that Metric is a proper Pydantic model.""" + assert issubclass(Metric, BaseModel) - assert measurement.value == 42 + def test_pydantic_model_dump(self): + """Test that we can use Pydantic features.""" + metric = DummyMetric(expected_value=50) + data = metric.model_dump() + assert data["expected_value"] == 50 + assert data["name"] == "dummy" -# ============================================================================= -# Metric base class tests -# ============================================================================= +class TestDependentDummyMetric: + def test_depends_on(self): + """Test that DependentDummyMetric declares dependencies.""" + deps = DependentDummyMetric.depends_on() -def test_metric_default_depends_on(): - """Test that Metric base class depends_on returns empty list.""" - assert DummyMetric.depends_on() == [] + assert deps == [DummyMetric] + def test_calculate(self, dummy_measurement_with_value): + """Test that DependentDummyMetric uses dependency value.""" + dependent = DependentDummyMetric() + calculated = Measurements({DummyMetric: [dummy_measurement_with_value]}) + measurement = dependent.calculate(context={}, measurements=calculated) -def test_metric_default_providers(): - """Test that Metric base class providers returns empty list.""" - assert DummyMetric.providers() == [] + assert measurement.value == 20 # 10 * 2 + def test_calculate_custom_base(self, empty_context): + """Test calculation with custom base value.""" + base_metric = DummyMetric(expected_value=25) + base_measurement = base_metric.calculate( + context=empty_context, measurements=Measurements() + ) -def test_metric_is_pydantic_model(): - """Test that Metric is a proper Pydantic model.""" - assert issubclass(Metric, BaseModel) + dependent = DependentDummyMetric() + calculated = Measurements({DummyMetric: [base_measurement]}) + measurement = dependent.calculate( + context=empty_context, measurements=calculated + ) + assert measurement.value == 50 # 25 * 2 -def test_metric_pydantic_model_dump(): - """Test that we can use Pydantic features.""" - metric = DummyMetric(expected_value=50) - data = metric.model_dump() - assert data["expected_value"] == 50 - assert data["name"] == "dummy" +class TestProviderSystem: + def test_dummy_provider_adds_data(self): + """Test DummyProvider adds data.""" + provider = DummyProvider() + result = provider.provide() + assert result == {"data": 100} + def test_dummy_provider_with_custom_data(self): + """Test DummyProvider with custom data.""" + provider = DummyProvider(data=42) + result = provider.provide() + assert result == {"data": 42} -# ============================================================================= -# DependentDummyMetric tests -# ============================================================================= + def test_provider_dummy_metric_has_provider(self): + """Test that ProviderDummyMetric declares providers.""" + providers = ProviderDummyMetric.providers() + assert providers == [DummyProvider] -def test_dependent_metric_depends_on(): - """Test that DependentDummyMetric declares dependencies.""" - deps = DependentDummyMetric.depends_on() + def test_provider_dummy_metric_calculate(self): + """Test that ProviderDummyMetric uses context from provider.""" + # Build context with namespaced provider data + provider = DummyProvider() + context = {DummyProvider.name: provider.provide()} - assert deps == [DummyMetric] + metric = ProviderDummyMetric() + measurement = metric.calculate(context=context, measurements=Measurements()) - -def test_dependent_metric_calculate(dummy_measurement_with_value): - """Test that DependentDummyMetric uses dependency value.""" - dependent = DependentDummyMetric() - measurement = dependent.calculate( - context={}, measurements={DummyMetric: [dummy_measurement_with_value]} - ) - - assert measurement.value == 20 # 10 * 2 - - -def test_dependent_metric_calculate_custom_base(empty_context): - """Test calculation with custom base value.""" - base_metric = DummyMetric(expected_value=25) - base_measurement = base_metric.calculate(context=empty_context, measurements={}) - - dependent = DependentDummyMetric() - measurement = dependent.calculate( - context=empty_context, measurements={DummyMetric: [base_measurement]} - ) - - assert measurement.value == 50 # 25 * 2 - - -# ============================================================================= -# Provider system tests -# ============================================================================= - - -def test_dummy_provider_adds_data(): - """Test DummyProvider adds data.""" - provider = DummyProvider() - result = provider.provide() - assert result == {"data": 100} - - -def test_dummy_provider_with_custom_data(): - """Test DummyProvider with custom data.""" - provider = DummyProvider(data=42) - result = provider.provide() - assert result == {"data": 42} - - -def test_provider_dummy_metric_has_provider(): - """Test that ProviderDummyMetric declares providers.""" - providers = ProviderDummyMetric.providers() - - assert providers == [DummyProvider] - - -def test_provider_dummy_metric_calculate(): - """Test that ProviderDummyMetric uses context from provider.""" - # Build context with namespaced provider data - provider = DummyProvider() - context = {DummyProvider.name: provider.provide()} - - metric = ProviderDummyMetric() - measurement = metric.calculate(context=context, measurements={}) - - assert measurement.value == 100 + assert measurement.value == 100